Intercept and assert on application data in e2e test

app/src/androidTest/java/com/stevesoltys/seedvault/KoinInstrumentationTestApp.kt

@@ -1,32 +1,39 @@
 package com.stevesoltys.seedvault
 
-import androidx.test.platform.app.InstrumentationRegistry
 import com.stevesoltys.seedvault.restore.RestoreViewModel
+import com.stevesoltys.seedvault.transport.backup.FullBackup
+import com.stevesoltys.seedvault.transport.backup.InputFactory
+import com.stevesoltys.seedvault.transport.backup.KVBackup
+import com.stevesoltys.seedvault.transport.restore.FullRestore
+import com.stevesoltys.seedvault.transport.restore.KVRestore
+import com.stevesoltys.seedvault.transport.restore.OutputFactory
 import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
 import io.mockk.spyk
+import org.koin.androidx.viewmodel.dsl.viewModel
 import org.koin.core.module.Module
 import org.koin.dsl.module
 
-private val spyBackupNotificationManager = spyk(
+internal var currentRestoreViewModel: RestoreViewModel? = null
-    BackupNotificationManager(
-        InstrumentationRegistry.getInstrumentation()
-            .targetContext.applicationContext
-    )
-)
 
 class KoinInstrumentationTestApp : App() {
 
     override fun appModules(): List<Module> {
         val testModule = module {
-            single { spyBackupNotificationManager }
+            val context = this@KoinInstrumentationTestApp
 
-            single {
+            single { spyk(BackupNotificationManager(context)) }
-                spyk(
+            single { spyk(FullBackup(get(), get(), get(), get())) }
-                    RestoreViewModel(
+            single { spyk(KVBackup(get(), get(), get(), get(), get())) }
-                        this@KoinInstrumentationTestApp,
+            single { spyk(InputFactory()) }
-                        get(), get(), get(), get(), get(), get()
+
-                    )
+            single { spyk(FullRestore(get(), get(), get(), get(), get())) }
-                )
+            single { spyk(KVRestore(get(), get(), get(), get(), get(), get())) }
+            single { spyk(OutputFactory()) }
+
+            viewModel {
+                currentRestoreViewModel =
+                    spyk(RestoreViewModel(context, get(), get(), get(), get(), get(), get()))
+                currentRestoreViewModel!!
             }
         }
 

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeBackupTestBase.kt

@@ -1,12 +1,22 @@
 package com.stevesoltys.seedvault.e2e
 
+import android.content.pm.PackageInfo
+import android.os.ParcelFileDescriptor
+import com.stevesoltys.seedvault.e2e.io.BackupDataInputIntercept
+import com.stevesoltys.seedvault.e2e.io.InputStreamIntercept
 import com.stevesoltys.seedvault.e2e.screen.impl.BackupScreen
+import com.stevesoltys.seedvault.transport.backup.FullBackup
+import com.stevesoltys.seedvault.transport.backup.InputFactory
+import com.stevesoltys.seedvault.transport.backup.KVBackup
 import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
 import io.mockk.clearMocks
+import io.mockk.coEvery
 import io.mockk.every
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeout
+import org.koin.core.component.get
+import java.io.ByteArrayOutputStream
 import java.util.concurrent.atomic.AtomicBoolean
 
 internal interface LargeBackupTestBase : LargeTestBase {
@@ -15,7 +25,13 @@ internal interface LargeBackupTestBase : LargeTestBase {
         private const val BACKUP_TIMEOUT = 360 * 1000L
     }
 
-    val spyBackupNotificationManager: BackupNotificationManager
+    val spyBackupNotificationManager: BackupNotificationManager get() = get()
+
+    val spyFullBackup: FullBackup get() = get()
+
+    val spyKVBackup: KVBackup get() = get()
+
+    val spyInputFactory: InputFactory get() = get()
 
     fun launchBackupActivity() {
         runCommand("am start -n ${targetContext.packageName}/.settings.SettingsActivity")
@@ -35,14 +51,112 @@ internal interface LargeBackupTestBase : LargeTestBase {
         }
     }
 
-    fun performBackup(expectedPackages: Set<String>) {
+    fun performBackup(): SeedvaultLargeTestResult {
-        val backupResult = spyOnBackup(expectedPackages)
+
+        val backupResult = SeedvaultLargeTestResult(
+            full = mutableMapOf(),
+            kv = mutableMapOf(),
+            userApps = packageService.userApps,
+            userNotAllowedApps = packageService.userNotAllowedApps
+        )
+
+        val completed = spyOnBackup(backupResult)
         startBackup()
-        waitForBackupResult(backupResult)
+        waitForBackupResult(completed)
+
+        return backupResult.copy(
+            backupResults = backupResult.allUserApps().associate {
+                it.packageName to spyMetadataManager.getPackageMetadata(it.packageName)
+            }.toMutableMap()
+        )
     }
 
-    private fun spyOnBackup(expectedPackages: Set<String>): AtomicBoolean {
+    private fun waitForBackupResult(completed: AtomicBoolean) {
-        val finishedBackup = AtomicBoolean(false)
+        runBlocking {
+            withTimeout(BACKUP_TIMEOUT) {
+                while (!completed.get()) {
+                    delay(100)
+                }
+            }
+        }
+    }
+
+    private fun spyOnBackup(backupResult: SeedvaultLargeTestResult): AtomicBoolean {
+        clearMocks(spyInputFactory, spyKVBackup, spyFullBackup)
+        spyOnFullBackupData(backupResult)
+        spyOnKVBackupData(backupResult)
+
+        return spyOnBackupCompletion()
+    }
+
+    private fun spyOnKVBackupData(backupResult: SeedvaultLargeTestResult) {
+        var packageName: String? = null
+        var data = mutableMapOf<String, ByteArray>()
+
+        coEvery {
+            spyKVBackup.performBackup(any(), any(), any(), any(), any())
+        } answers {
+            packageName = firstArg<PackageInfo>().packageName
+            callOriginal()
+        }
+
+        every {
+            spyInputFactory.getBackupDataInput(any())
+        } answers {
+            val fd = firstArg<ParcelFileDescriptor>().fileDescriptor
+
+            BackupDataInputIntercept(fd) { key, value ->
+                data[key] = value
+            }
+        }
+
+        coEvery {
+            spyKVBackup.finishBackup()
+        } answers {
+            backupResult.kv[packageName!!] = data
+                .mapValues { entry -> entry.value.sha256() }
+                .toMutableMap()
+
+            packageName = null
+            data = mutableMapOf()
+            callOriginal()
+        }
+    }
+
+    private fun spyOnFullBackupData(backupResult: SeedvaultLargeTestResult) {
+        var packageName: String? = null
+        var dataIntercept = ByteArrayOutputStream()
+
+        coEvery {
+            spyFullBackup.performFullBackup(any(), any(), any(), any(), any())
+        } answers {
+            packageName = firstArg<PackageInfo>().packageName
+            callOriginal()
+        }
+
+        every {
+            spyInputFactory.getInputStream(any())
+        } answers {
+            InputStreamIntercept(
+                inputStream = callOriginal(),
+                intercept = dataIntercept
+            )
+        }
+
+        every {
+            spyFullBackup.finishBackup()
+        } answers {
+            val result = callOriginal()
+            backupResult.full[packageName!!] = dataIntercept.toByteArray().sha256()
+
+            packageName = null
+            dataIntercept = ByteArrayOutputStream()
+            result
+        }
+    }
+
+    private fun spyOnBackupCompletion(): AtomicBoolean {
+        val completed = AtomicBoolean(false)
 
         clearMocks(spyBackupNotificationManager)
 
@@ -52,20 +166,10 @@ internal interface LargeBackupTestBase : LargeTestBase {
             val success = firstArg<Boolean>()
             assert(success) { "Backup failed." }
 
-            this.callOriginal()
+            callOriginal()
-            finishedBackup.set(true)
+            completed.set(true)
         }
 
-        return finishedBackup
+        return completed
-    }
-
-    private fun waitForBackupResult(finishedBackup: AtomicBoolean) {
-        runBlocking {
-            withTimeout(BACKUP_TIMEOUT) {
-                while (!finishedBackup.get()) {
-                    delay(100)
-                }
-            }
-        }
     }
 }

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeRestoreTestBase.kt

@@ -1,11 +1,24 @@
 package com.stevesoltys.seedvault.e2e
 
+import android.content.pm.PackageInfo
+import android.os.ParcelFileDescriptor
+import com.stevesoltys.seedvault.e2e.io.BackupDataOutputIntercept
+import com.stevesoltys.seedvault.e2e.io.OutputStreamIntercept
 import com.stevesoltys.seedvault.e2e.screen.impl.RecoveryCodeScreen
 import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
-import com.stevesoltys.seedvault.restore.RestoreViewModel
+import com.stevesoltys.seedvault.transport.restore.FullRestore
+import com.stevesoltys.seedvault.transport.restore.KVRestore
+import com.stevesoltys.seedvault.transport.restore.OutputFactory
+import io.mockk.clearMocks
+import io.mockk.coEvery
+import io.mockk.every
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import kotlinx.coroutines.withTimeout
+import org.koin.core.component.get
+import java.io.ByteArrayOutputStream
 
 internal interface LargeRestoreTestBase : LargeTestBase {
 
@@ -13,7 +26,11 @@ internal interface LargeRestoreTestBase : LargeTestBase {
         private const val RESTORE_TIMEOUT = 360 * 1000L
     }
 
-    val spyRestoreViewModel: RestoreViewModel
+    val spyFullRestore: FullRestore get() = get()
+
+    val spyKVRestore: KVRestore get() = get()
+
+    val spyOutputFactory: OutputFactory get() = get()
 
     fun launchRestoreActivity() {
         runCommand("am start -n ${targetContext.packageName}/.restore.RestoreActivity")
@@ -35,7 +52,17 @@ internal interface LargeRestoreTestBase : LargeTestBase {
         }
     }
 
-    fun performRestore() {
+    fun performRestore(): SeedvaultLargeTestResult {
+
+        val result = SeedvaultLargeTestResult(
+            full = mutableMapOf(),
+            kv = mutableMapOf(),
+            userApps = emptyList(), // will update everything below this after restore
+            userNotAllowedApps = emptyList()
+        )
+
+        spyOnRestoreData(result)
+
         RestoreScreen {
             backupListItem.clickAndWaitForNewWindow()
             waitUntilIdle()
@@ -48,39 +75,123 @@ internal interface LargeRestoreTestBase : LargeTestBase {
             skipButton.clickAndWaitForNewWindow()
             waitUntilIdle()
         }
+
+        return result.copy(
+            userApps = packageService.userApps,
+            userNotAllowedApps = packageService.userNotAllowedApps
+        )
+    }
+
+    private fun spyOnRestoreData(result: SeedvaultLargeTestResult) {
+        clearMocks(spyOutputFactory)
+
+        spyOnFullRestoreData(result)
+        spyOnKVRestoreData(result)
     }
 
     private fun waitForInstallResult() = runBlocking {
-        withTimeout(RESTORE_TIMEOUT) {
+
-            while (spyRestoreViewModel.installResult.value == null ||
+        withContext(Dispatchers.Main) {
-                spyRestoreViewModel.nextButtonEnabled.value == false
+            withTimeout(RESTORE_TIMEOUT) {
-            ) {
+                while (spyRestoreViewModel.installResult.value == null ||
-                delay(100)
+                    spyRestoreViewModel.nextButtonEnabled.value == false
+                ) {
+                    delay(100)
+                }
             }
+
+            val restoreResultValue = spyRestoreViewModel.installResult.value
+                ?: error("Restore APKs timed out")
+
+            assert(!restoreResultValue.hasFailed) { "Failed to install packages" }
         }
 
-        val restoreResultValue = spyRestoreViewModel.installResult.value
-            ?: error("Restore APKs timed out")
-
-        assert(!restoreResultValue.hasFailed) { "Failed to install packages" }
         waitUntilIdle()
     }
 
     private fun waitForRestoreDataResult() = runBlocking {
-        withTimeout(RESTORE_TIMEOUT) {
+        withContext(Dispatchers.Main) {
-            while (spyRestoreViewModel.restoreBackupResult.value == null) {
+            withTimeout(RESTORE_TIMEOUT) {
-                delay(100)
+                while (spyRestoreViewModel.restoreBackupResult.value == null) {
+                    delay(100)
+                }
             }
+
+            val restoreResultValue = spyRestoreViewModel.restoreBackupResult.value
+                ?: error("Restore app data timed out")
+
+            assert(!restoreResultValue.hasError()) {
+                "Restore failed: ${restoreResultValue.errorMsg}"
+            }
+
+            waitUntilIdle()
         }
-
-        val restoreResultValue = spyRestoreViewModel.restoreBackupResult.value
-            ?: error("Restore app data timed out")
-
-        assert(!restoreResultValue.hasError()) {
-            "Restore failed: ${restoreResultValue.errorMsg}"
-        }
-
-        waitUntilIdle()
     }
 
+    private fun spyOnKVRestoreData(restoreResult: SeedvaultLargeTestResult) {
+        var packageName: String? = null
+
+        clearMocks(spyKVRestore)
+
+        coEvery {
+            spyKVRestore.initializeState(any(), any(), any(), any(), any())
+        } answers {
+            packageName = arg<PackageInfo>(3).packageName
+            restoreResult.kv[packageName!!] = mutableMapOf()
+            callOriginal()
+        }
+
+        every {
+            spyOutputFactory.getBackupDataOutput(any())
+        } answers {
+            val fd = firstArg<ParcelFileDescriptor>().fileDescriptor
+
+            BackupDataOutputIntercept(fd) { key, value ->
+                restoreResult.kv[packageName!!]!![key] = value.sha256()
+            }
+        }
+    }
+
+    private fun spyOnFullRestoreData(restoreResult: SeedvaultLargeTestResult) {
+        var packageName: String? = null
+        var dataIntercept = ByteArrayOutputStream()
+
+        clearMocks(spyFullRestore)
+
+        coEvery {
+            spyFullRestore.initializeState(any(), any(), any(), any())
+        } answers {
+            packageName = arg<PackageInfo>(3).packageName
+            dataIntercept = ByteArrayOutputStream()
+
+            callOriginal()
+        }
+
+        every {
+            spyOutputFactory.getOutputStream(any())
+        } answers {
+            OutputStreamIntercept(
+                outputStream = callOriginal(),
+                intercept = dataIntercept
+            )
+        }
+
+        every {
+            spyFullRestore.abortFullRestore()
+        } answers {
+            packageName = null
+            dataIntercept = ByteArrayOutputStream()
+            callOriginal()
+        }
+
+        every {
+            spyFullRestore.finishRestore()
+        } answers {
+            restoreResult.full[packageName!!] = dataIntercept.toByteArray().sha256()
+
+            packageName = null
+            dataIntercept = ByteArrayOutputStream()
+            callOriginal()
+        }
+    }
 }

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt

@@ -2,33 +2,52 @@ package com.stevesoltys.seedvault.e2e
 
 import android.app.UiAutomation
 import android.content.Context
+import android.content.pm.PackageInfo
 import android.os.Environment
 import androidx.annotation.WorkerThread
+import androidx.preference.PreferenceManager
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.uiautomator.UiDevice
+import com.stevesoltys.seedvault.crypto.ANDROID_KEY_STORE
+import com.stevesoltys.seedvault.crypto.KEY_ALIAS_BACKUP
+import com.stevesoltys.seedvault.crypto.KEY_ALIAS_MAIN
+import com.stevesoltys.seedvault.crypto.KeyManager
+import com.stevesoltys.seedvault.currentRestoreViewModel
+import com.stevesoltys.seedvault.e2e.screen.impl.BackupScreen
 import com.stevesoltys.seedvault.e2e.screen.impl.DocumentPickerScreen
 import com.stevesoltys.seedvault.e2e.screen.impl.RecoveryCodeScreen
+import com.stevesoltys.seedvault.metadata.MetadataManager
+import com.stevesoltys.seedvault.permitDiskReads
+import com.stevesoltys.seedvault.plugins.saf.DocumentsStorage
+import com.stevesoltys.seedvault.restore.RestoreViewModel
+import com.stevesoltys.seedvault.settings.SettingsManager
+import com.stevesoltys.seedvault.transport.backup.PackageService
 import kotlinx.coroutines.DelicateCoroutinesApi
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.get
+import java.io.File
 import java.lang.Thread.sleep
+import java.security.KeyStore
+import java.security.MessageDigest
 import java.text.SimpleDateFormat
 import java.util.Calendar
 import java.util.concurrent.atomic.AtomicBoolean
 
-interface LargeTestBase {
+internal interface LargeTestBase : KoinComponent {
 
     companion object {
         private const val TEST_STORAGE_FOLDER = "seedvault_test"
         private const val TEST_VIDEO_FOLDER = "seedvault_test_videos"
     }
 
-    fun externalStorageDir(): String = Environment.getExternalStorageDirectory().absolutePath
+    val externalStorageDir: String get() = Environment.getExternalStorageDirectory().absolutePath
 
-    fun testStoragePath(): String = "${externalStorageDir()}/$TEST_STORAGE_FOLDER"
+    val testStoragePath get() = "$externalStorageDir/$TEST_STORAGE_FOLDER"
 
-    fun testVideoPath(): String = "${externalStorageDir()}/$TEST_VIDEO_FOLDER"
+    val testVideoPath get() = "$externalStorageDir/$TEST_VIDEO_FOLDER"
 
     val targetContext: Context
         get() = InstrumentationRegistry.getInstrumentation().targetContext
@@ -39,6 +58,38 @@ interface LargeTestBase {
     val device: UiDevice
         get() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
 
+    val packageService: PackageService get() = get()
+
+    val settingsManager: SettingsManager get() = get()
+
+    val keyManager: KeyManager get() = get()
+
+    val documentsStorage: DocumentsStorage get() = get()
+
+    val spyMetadataManager: MetadataManager get() = get()
+
+    val spyRestoreViewModel: RestoreViewModel
+        get() = currentRestoreViewModel ?: error("currentRestoreViewModel is null")
+
+    fun resetApplicationState() {
+        settingsManager.setNewToken(null)
+        documentsStorage.reset(null)
+
+        val sharedPreferences = permitDiskReads {
+            PreferenceManager.getDefaultSharedPreferences(targetContext)
+        }
+        sharedPreferences.edit().clear().apply()
+
+        KeyStore.getInstance(ANDROID_KEY_STORE).apply {
+            load(null)
+        }.apply {
+            deleteEntry(KEY_ALIAS_MAIN)
+            deleteEntry(KEY_ALIAS_BACKUP)
+        }
+
+        clearDocumentPickerAppData()
+    }
+
     fun waitUntilIdle() {
         device.waitForIdle()
         sleep(3000)
@@ -58,8 +109,7 @@ interface LargeTestBase {
         val timeStamp = simpleDateFormat.format(Calendar.getInstance().time)
         val fileName = "${timeStamp}_${testName.replace(" ", "_")}"
 
-        val folder = testVideoPath()
+        val folder = testVideoPath
-
         runCommand("mkdir -p $folder")
 
         // screen record automatically stops after 3 minutes
@@ -80,8 +130,8 @@ interface LargeTestBase {
         runCommand("pkill -2 screenrecord")
     }
 
-    fun uninstallPackages(packages: Set<String>) {
+    fun uninstallPackages(packages: Collection<PackageInfo>) {
-        packages.forEach { runCommand("pm uninstall $it") }
+        packages.forEach { runCommand("pm uninstall ${it.packageName}") }
     }
 
     fun clearDocumentPickerAppData() {
@@ -89,7 +139,19 @@ interface LargeTestBase {
     }
 
     fun clearTestBackups() {
-        runCommand("rm -Rf ${testStoragePath()}")
+        File(testStoragePath).deleteRecursively()
+    }
+
+    fun changeBackupLocation(
+        folderName: String = TEST_STORAGE_FOLDER,
+        exists: Boolean = false,
+    ) {
+        BackupScreen {
+            clearDocumentPickerAppData()
+            backupLocationButton.clickAndWaitForNewWindow()
+
+            chooseStorageLocation(folderName, exists)
+        }
     }
 
     fun chooseStorageLocation(
@@ -118,4 +180,10 @@ interface LargeTestBase {
             verifyCodeButton.scrollTo().click()
         }
     }
+
+    fun ByteArray.sha256(): String {
+        val data = MessageDigest.getInstance("SHA-256").digest(this)
+
+        return data.joinToString("") { "%02x".format(it) }
+    }
 }

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt

@@ -1,8 +1,6 @@
 package com.stevesoltys.seedvault.e2e
 
 import androidx.test.ext.junit.runners.AndroidJUnit4
-import com.stevesoltys.seedvault.restore.RestoreViewModel
-import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
 import kotlinx.coroutines.runBlocking
 import org.junit.After
 import org.junit.Before
@@ -10,7 +8,6 @@ import org.junit.Rule
 import org.junit.rules.TestName
 import org.junit.runner.RunWith
 import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
 import java.io.File
 import java.util.concurrent.atomic.AtomicBoolean
 
@@ -27,11 +24,9 @@ internal abstract class SeedvaultLargeTest :
         private const val RECOVERY_CODE_FILE = "recovery-code.txt"
     }
 
-    override val spyBackupNotificationManager: BackupNotificationManager by inject()
+    private val baselineBackupFolderPath get() = "$externalStorageDir/$BASELINE_BACKUP_FOLDER"
 
-    override val spyRestoreViewModel: RestoreViewModel by inject()
+    private val baselineBackupPath get() = "$baselineBackupFolderPath/.SeedVaultAndroidBackup"
-
-    private val baselineBackupFolderPath = "${this.externalStorageDir()}/$BASELINE_BACKUP_FOLDER"
 
     private val baselineRecoveryCodePath = "$baselineBackupFolderPath/$RECOVERY_CODE_FILE"
 
@@ -39,7 +34,7 @@ internal abstract class SeedvaultLargeTest :
 
     @Before
     open fun setUp() = runBlocking {
-        clearDocumentPickerAppData()
+        resetApplicationState()
         clearTestBackups()
 
         startScreenRecord(keepRecordingScreen, name.methodName)
@@ -58,14 +53,15 @@ internal abstract class SeedvaultLargeTest :
      * provisioning tests: https://github.com/seedvault-app/seedvault-test-data
      */
     private fun restoreBaselineBackup() {
-        if (File(baselineBackupFolderPath).exists()) {
+        val backupFile = File(baselineBackupPath)
+
+        if (backupFile.exists()) {
             launchRestoreActivity()
             chooseStorageLocation(folderName = BASELINE_BACKUP_FOLDER, exists = true)
             typeInRestoreCode(baselineBackupRecoveryCode())
             performRestore()
 
-            // remove baseline backup after restore
+            resetApplicationState()
-            runCommand("rm -Rf $baselineBackupFolderPath/*")
         }
     }
 

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTestResult.kt

@@ -0,0 +1,23 @@
+package com.stevesoltys.seedvault.e2e
+
+import android.content.pm.PackageInfo
+import com.stevesoltys.seedvault.metadata.PackageMetadata
+
+/**
+ * Contains maps of (package name -> SHA-256 hashes) of application data.
+ *
+ * During backups and restores, we intercept the package data and store the result here.
+ * We can use this to validate that the restored app data actually matches the backed up data.
+ *
+ * For full backups, the mapping is: Map<PackageName, SHA-256>
+ * For K/V backups, the mapping is: Map<PackageName, Map<Key, SHA-256>>
+ */
+data class SeedvaultLargeTestResult(
+    val backupResults: Map<String, PackageMetadata?> = emptyMap(),
+    val full: MutableMap<String, String>,
+    val kv: MutableMap<String, MutableMap<String, String>>,
+    val userApps: List<PackageInfo>,
+    val userNotAllowedApps: List<PackageInfo>,
+) {
+    fun allUserApps() = userApps + userNotAllowedApps
+}

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/impl/BackupRestoreTest.kt

@@ -2,40 +2,139 @@ package com.stevesoltys.seedvault.e2e.impl
 
 import androidx.test.filters.LargeTest
 import com.stevesoltys.seedvault.e2e.SeedvaultLargeTest
-import com.stevesoltys.seedvault.settings.SettingsManager
+import com.stevesoltys.seedvault.e2e.SeedvaultLargeTestResult
-import com.stevesoltys.seedvault.transport.backup.PackageService
+import com.stevesoltys.seedvault.metadata.PackageState
 import org.junit.Test
-import org.koin.core.component.inject
 
 @LargeTest
 internal class BackupRestoreTest : SeedvaultLargeTest() {
 
-    private val packageService: PackageService by inject()
-
-    private val settingsManager: SettingsManager by inject()
-
     @Test
     fun `backup and restore applications`() {
         launchBackupActivity()
 
-        if (settingsManager.getStorage() == null) {
+        if (!keyManager.hasBackupKey()) {
             confirmCode()
-            chooseStorageLocation()
         }
 
-        val eligiblePackages = getEligibleApps()
+        if (settingsManager.getStorage() == null) {
-        performBackup(eligiblePackages)
+            chooseStorageLocation()
-        uninstallPackages(eligiblePackages)
+        } else {
+            changeBackupLocation()
+        }
+
+        val backupResult = performBackup()
+        assertValidBackupMetadata(backupResult)
+
+        uninstallPackages(backupResult.allUserApps())
 
         launchRestoreActivity()
-        performRestore()
+        val restoreResult = performRestore()
 
-        // TODO: Get some real assertions in here..
+        assertValidResults(backupResult, restoreResult)
-        // val packagesAfterRestore = getEligibleApps()
-        // assert(eligiblePackages == packagesAfterRestore)
     }
 
-    private fun getEligibleApps() = packageService.userApps
+    private fun assertValidBackupMetadata(backup: SeedvaultLargeTestResult) {
-        .map { it.packageName }.toSet()
+        // Assert all user apps have metadata.
+        backup.allUserApps().forEach { app ->
+            assert(backup.backupResults.containsKey(app.packageName)) {
+                "Metadata for $app missing from backup."
+            }
+        }
 
+        // Assert all metadata has a valid state.
+        backup.backupResults.forEach { (pkg, metadata) ->
+            assert(metadata != null) { "Metadata for $pkg is null." }
+
+            assert(metadata!!.state != PackageState.UNKNOWN_ERROR) {
+                "Metadata for $pkg has an unknown state."
+            }
+        }
+    }
+
+    private fun assertValidResults(
+        backup: SeedvaultLargeTestResult,
+        restore: SeedvaultLargeTestResult,
+    ) {
+        assertAllUserAppsWereRestored(backup, restore)
+        assertValidFullData(backup, restore)
+        assertValidKeyValueData(backup, restore)
+    }
+
+    private fun assertAllUserAppsWereRestored(
+        backup: SeedvaultLargeTestResult,
+        restore: SeedvaultLargeTestResult,
+    ) {
+        val backupUserApps = backup.allUserApps()
+            .map { it.packageName }.toSet()
+
+        val restoreUserApps = restore.allUserApps()
+            .map { it.packageName }.toSet()
+
+        // Assert we re-installed all user apps.
+        assert(restoreUserApps.containsAll(backupUserApps)) {
+            val missingApps = backupUserApps
+                .minus(restoreUserApps)
+                .joinToString(", ")
+
+            "Not all user apps were restored. Missing: $missingApps"
+        }
+
+        // Assert we restored data for all user apps that had successful backups.
+        // This is expected to succeed because we are uninstalling the apps before restoring.
+        val missingFromRestore = backup.userApps
+            .map { it.packageName }
+            .filter { backup.backupResults[it]?.state == PackageState.APK_AND_DATA }
+            .filter { !restore.kv.containsKey(it) && !restore.full.containsKey(it) }
+
+        if (missingFromRestore.isNotEmpty()) {
+            val failedApps = missingFromRestore.joinToString(", ")
+
+            error("Not all user apps had their data restored. Missing: $failedApps")
+        }
+    }
+
+    private fun assertValidFullData(
+        backup: SeedvaultLargeTestResult,
+        restore: SeedvaultLargeTestResult,
+    ) {
+        // Assert all "full" restored data matches the backup data.
+        val allUserPkgs = backup.allUserApps().map { it.packageName }
+
+        restore.full.forEach { (pkg, fullData) ->
+            if (allUserPkgs.contains(pkg)) {
+                assert(backup.full.containsKey(pkg)) {
+                    "Full data for $pkg missing from restore."
+                }
+
+                if (backup.backupResults[pkg]!!.state == PackageState.APK_AND_DATA) {
+                    assert(fullData == backup.full[pkg]!!) {
+                        "Full data for $pkg does not match."
+                    }
+                }
+            }
+        }
+    }
+
+    private fun assertValidKeyValueData(
+        backup: SeedvaultLargeTestResult,
+        restore: SeedvaultLargeTestResult,
+    ) {
+        // Assert all "key/value" restored data matches the backup data.
+        restore.kv.forEach { (pkg, kvData) ->
+            assert(backup.kv.containsKey(pkg)) {
+                "KV data for $pkg missing from backup."
+            }
+
+            kvData.forEach { (key, value) ->
+                assert(backup.kv[pkg]!!.containsKey(key)) {
+                    "KV data for $pkg/$key exists in restore but is missing from backup."
+                }
+
+                assert(value.contentEquals(backup.kv[pkg]!![key]!!)) {
+                    "KV data for $pkg/$key does not match."
+                }
+            }
+        }
+    }
 }

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/io/BackupDataInputIntercept.kt

@@ -0,0 +1,24 @@
+package com.stevesoltys.seedvault.e2e.io
+
+import android.app.backup.BackupDataInput
+import java.io.FileDescriptor
+
+class BackupDataInputIntercept(
+    fileDescriptor: FileDescriptor,
+    private val callback: (String, ByteArray) -> Unit,
+) : BackupDataInput(fileDescriptor) {
+
+    var currentKey: String? = null
+
+    override fun getKey(): String? {
+        currentKey = super.getKey()
+        return currentKey
+    }
+
+    override fun readEntityData(data: ByteArray, offset: Int, size: Int): Int {
+        val result = super.readEntityData(data, offset, size)
+
+        callback(currentKey!!, data.copyOf(result))
+        return result
+    }
+}

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/io/BackupDataOutputIntercept.kt

@@ -0,0 +1,23 @@
+package com.stevesoltys.seedvault.e2e.io
+
+import android.app.backup.BackupDataOutput
+import java.io.FileDescriptor
+
+class BackupDataOutputIntercept(
+    fileDescriptor: FileDescriptor,
+    private val callback: (String, ByteArray) -> Unit,
+) : BackupDataOutput(fileDescriptor) {
+
+    private var currentKey: String? = null
+
+    override fun writeEntityHeader(key: String, dataSize: Int): Int {
+        currentKey = key
+        return super.writeEntityHeader(key, dataSize)
+    }
+
+    override fun writeEntityData(data: ByteArray, size: Int): Int {
+        callback(currentKey!!, data.copyOf())
+
+        return super.writeEntityData(data, size)
+    }
+}

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/io/InputStreamIntercept.kt

@@ -0,0 +1,26 @@
+package com.stevesoltys.seedvault.e2e.io
+
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+
+class InputStreamIntercept(
+    private val inputStream: InputStream,
+    private val intercept: ByteArrayOutputStream
+) : InputStream() {
+
+    override fun read(): Int {
+        val byte = inputStream.read()
+        if (byte != -1) {
+            intercept.write(byte)
+        }
+        return byte
+    }
+
+    override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
+        val bytesRead = inputStream.read(buffer, offset, length)
+        if (bytesRead != -1) {
+            intercept.write(buffer, offset, bytesRead)
+        }
+        return bytesRead
+    }
+}

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/io/OutputStreamIntercept.kt

@@ -0,0 +1,20 @@
+package com.stevesoltys.seedvault.e2e.io
+
+import java.io.ByteArrayOutputStream
+import java.io.OutputStream
+
+class OutputStreamIntercept(
+    private val outputStream: OutputStream,
+    private val intercept: ByteArrayOutputStream
+) : OutputStream() {
+
+    override fun write(byte: Int) {
+        intercept.write(byte)
+        outputStream.write(byte)
+    }
+
+    override fun write(buffer: ByteArray, offset: Int, length: Int) {
+        intercept.write(buffer, offset, length)
+        outputStream.write(buffer, offset, length)
+    }
+}

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/impl/BackupScreen.kt

@@ -9,4 +9,6 @@ object BackupScreen : UiDeviceScreen<BackupScreen>() {
     val backupNowButton = findObject { text("Backup now") }
 
     val backupStatusButton = findObject { text("Backup status") }
+
+    val backupLocationButton = findObject { text("Backup location") }
 }

app/src/main/java/com/stevesoltys/seedvault/crypto/CryptoModule.kt

@@ -3,7 +3,7 @@ package com.stevesoltys.seedvault.crypto
 import org.koin.dsl.module
 import java.security.KeyStore
 
-private const val ANDROID_KEY_STORE = "AndroidKeyStore"
+const val ANDROID_KEY_STORE = "AndroidKeyStore"
 
 val cryptoModule = module {
     factory<CipherFactory> { CipherFactoryImpl(get()) }

app/src/main/java/com/stevesoltys/seedvault/crypto/KeyManager.kt

@@ -14,8 +14,8 @@ import javax.crypto.spec.SecretKeySpec
 
 internal const val KEY_SIZE = 256
 internal const val KEY_SIZE_BYTES = KEY_SIZE / 8
-private const val KEY_ALIAS_BACKUP = "com.stevesoltys.seedvault"
+internal const val KEY_ALIAS_BACKUP = "com.stevesoltys.seedvault"
-private const val KEY_ALIAS_MAIN = "com.stevesoltys.seedvault.main"
+internal const val KEY_ALIAS_MAIN = "com.stevesoltys.seedvault.main"
 private const val KEY_ALGORITHM_BACKUP = "AES"
 private const val KEY_ALGORITHM_MAIN = "HmacSHA256"
 

app/src/main/java/com/stevesoltys/seedvault/settings/SettingsManager.kt

@@ -60,8 +60,17 @@ class SettingsManager(private val context: Context) {
      * Should only be called by the [BackupCoordinator]
      * to ensure that related work is performed after moving to a new token.
      */
-    fun setNewToken(newToken: Long) {
+    fun setNewToken(newToken: Long?) {
-        prefs.edit().putLong(PREF_KEY_TOKEN, newToken).apply()
+        if (newToken == null) {
+            prefs.edit()
+                .remove(PREF_KEY_TOKEN)
+                .apply()
+        } else {
+            prefs.edit()
+                .putLong(PREF_KEY_TOKEN, newToken)
+                .apply()
+        }
+
         token = newToken
     }