diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt index 11ded3de..31a7436b 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt @@ -268,7 +268,9 @@ internal class BackupCoordinator( return TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED } } - val result = kv.performBackup(packageInfo, data, flags) + val token = settingsManager.getToken() ?: error("no token in performFullBackup") + val salt = metadataManager.salt + val result = kv.performBackup(packageInfo, data, flags, token, salt) if (result == TRANSPORT_OK && packageName == MAGIC_PACKAGE_MANAGER) { // hook in here to back up APKs of apps that are otherwise not allowed for backup backUpApksOfNotBackedUpPackages() @@ -360,7 +362,7 @@ internal class BackupCoordinator( val token = settingsManager.getToken() ?: error("no token in clearBackupData") val salt = metadataManager.salt try { - kv.clearBackupData(packageInfo) + kv.clearBackupData(packageInfo, token, salt) } catch (e: IOException) { Log.w(TAG, "Error clearing K/V backup data for $packageName", e) return TRANSPORT_ERROR diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt index a3fc6457..5e119b4b 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt @@ -18,13 +18,14 @@ val backupModule = module { metadataManager = get() ) } + single { KvDbManagerImpl(androidContext()) } single { KVBackup( - plugin = get().kvBackupPlugin, + plugin = get(), settingsManager = get(), inputFactory = get(), crypto = get(), - nm = get() + dbManager = get() ) } single { diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVBackup.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVBackup.kt index 043ab1b4..a4361257 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVBackup.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVBackup.kt @@ -9,16 +9,21 @@ import android.app.backup.BackupTransport.TRANSPORT_OK import android.content.pm.PackageInfo import android.os.ParcelFileDescriptor import android.util.Log -import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER import com.stevesoltys.seedvault.crypto.Crypto -import com.stevesoltys.seedvault.encodeBase64 import com.stevesoltys.seedvault.header.VERSION import com.stevesoltys.seedvault.header.getADForKV import com.stevesoltys.seedvault.settings.SettingsManager -import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager import java.io.IOException +import java.util.zip.GZIPOutputStream -class KVBackupState(internal val packageInfo: PackageInfo) +class KVBackupState( + internal val packageInfo: PackageInfo, + val token: Long, + val name: String, + val db: KVDb +) { + var needsUpload: Boolean = false +} const val DEFAULT_QUOTA_KEY_VALUE_BACKUP = (2 * (5 * 1024 * 1024)).toLong() @@ -26,11 +31,11 @@ private val TAG = KVBackup::class.java.simpleName @Suppress("BlockingMethodInNonBlockingContext") internal class KVBackup( - private val plugin: KVBackupPlugin, + private val plugin: BackupPlugin, private val settingsManager: SettingsManager, private val inputFactory: InputFactory, private val crypto: Crypto, - private val nm: BackupNotificationManager + private val dbManager: KvDbManager ) { private var state: KVBackupState? = null @@ -39,14 +44,18 @@ internal class KVBackup( fun getCurrentPackage() = state?.packageInfo - fun getQuota(): Long { - return if (settingsManager.isQuotaUnlimited()) Long.MAX_VALUE else plugin.getQuota() + fun getQuota(): Long = if (settingsManager.isQuotaUnlimited()) { + Long.MAX_VALUE + } else { + DEFAULT_QUOTA_KEY_VALUE_BACKUP } suspend fun performBackup( packageInfo: PackageInfo, data: ParcelFileDescriptor, - flags: Int + flags: Int, + token: Long, + salt: String ): Int { val dataNotChanged = flags and FLAG_DATA_NOT_CHANGED != 0 val isIncremental = flags and FLAG_INCREMENTAL != 0 @@ -73,7 +82,9 @@ internal class KVBackup( if (state != null) { throw AssertionError("Have state for ${state.packageInfo.packageName}") } - this.state = KVBackupState(packageInfo) + val name = crypto.getNameForPackage(salt, packageName) + val db = dbManager.getDb(packageName) + this.state = KVBackupState(packageInfo, token, name, db) // no need for backup when no data has changed if (dataNotChanged) { @@ -82,12 +93,7 @@ internal class KVBackup( } // check if we have existing data for the given package - val hasDataForPackage = try { - plugin.hasDataForPackage(packageInfo) - } catch (e: IOException) { - Log.e(TAG, "Error checking for existing data for ${packageInfo.packageName}.", e) - return backupError(TRANSPORT_ERROR) - } + val hasDataForPackage = dbManager.existsDb(packageName) if (isIncremental && !hasDataForPackage) { Log.w( TAG, "Requested incremental, but transport currently stores no data" + @@ -101,80 +107,36 @@ internal class KVBackup( if (isNonIncremental && hasDataForPackage) { Log.w(TAG, "Requested non-incremental, deleting existing data.") try { - clearBackupData(packageInfo) + clearBackupData(packageInfo, token, salt) } catch (e: IOException) { Log.w(TAG, "Error clearing backup data for ${packageInfo.packageName}.", e) } } // parse and store the K/V updates - return storeRecords(packageInfo, data) + return storeRecords(data) } - private suspend fun storeRecords(packageInfo: PackageInfo, data: ParcelFileDescriptor): Int { - val backupSequence: Iterable> - val pmRecordNumber: Int? - if (packageInfo.packageName == MAGIC_PACKAGE_MANAGER) { - // Since the package manager has many small keys to store, - // and this can be slow, especially on cloud-based storage, - // we get the entire data set first, so we can show progress notifications. - val list = parseBackupStream(data).toList() - backupSequence = list - pmRecordNumber = list.size - } else { - backupSequence = parseBackupStream(data).asIterable() - pmRecordNumber = null - } + private fun storeRecords(data: ParcelFileDescriptor): Int { + val state = this.state ?: error("No state in storeRecords") // apply the delta operations - var i = 1 - for (result in backupSequence) { + for (result in parseBackupStream(data)) { if (result is Result.Error) { Log.e(TAG, "Exception reading backup input", result.exception) return backupError(TRANSPORT_ERROR) } + state.needsUpload = true val op = (result as Result.Ok).result - try { - storeRecord(packageInfo, op, i++, pmRecordNumber) - } catch (e: IOException) { - Log.e(TAG, "Unable to update base64Key file for base64Key ${op.base64Key}", e) - // Returning something more forgiving such as TRANSPORT_PACKAGE_REJECTED - // will still make the entire backup fail. - // TODO However, TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED might buy us a retry, - // we would just need to be careful not to create an infinite loop - // for permanent errors. - return backupError(TRANSPORT_ERROR) + if (op.value == null) { + Log.e(TAG, "Deleting record with key ${op.key}") + state.db.delete(op.key) + } else { + state.db.put(op.key, op.value) } } return TRANSPORT_OK } - @Throws(IOException::class) - private suspend fun storeRecord( - packageInfo: PackageInfo, - op: KVOperation, - currentNum: Int, - pmRecordNumber: Int? - ) { - // update notification for package manager backup - if (pmRecordNumber != null) { - nm.onPmKvBackup(op.key, currentNum, pmRecordNumber) - } - // check if record should get deleted - if (op.value == null) { - Log.e(TAG, "Deleting record with base64Key ${op.base64Key}") - plugin.deleteRecord(packageInfo, op.base64Key) - } else { - plugin.getOutputStreamForRecord(packageInfo, op.base64Key).use { outputStream -> - outputStream.write(ByteArray(1) { VERSION }) - val ad = getADForKV(VERSION, packageInfo.packageName) - crypto.newEncryptingStream(outputStream, ad).use { encryptedStream -> - encryptedStream.write(op.value) - encryptedStream.flush() - } - } - } - } - /** * Parses a backup stream into individual key/value operations */ @@ -194,12 +156,11 @@ internal class KVBackup( } // encode key val key = changeSet.key - val base64Key = key.encodeBase64() val dataSize = changeSet.dataSize // read value val value = if (dataSize >= 0) { - Log.v(TAG, " Delta operation key $key size $dataSize key64 $base64Key") + Log.v(TAG, " Delta operation key $key size $dataSize") val bytes = ByteArray(dataSize) val bytesRead = try { changeSet.readEntityData(bytes, 0, dataSize) @@ -213,19 +174,31 @@ internal class KVBackup( bytes } else null // add change operation to the sequence - Result.Ok(KVOperation(key, base64Key, value)) + Result.Ok(KVOperation(key, value)) } } @Throws(IOException::class) - suspend fun clearBackupData(packageInfo: PackageInfo) { - plugin.removeDataOfPackage(packageInfo) + suspend fun clearBackupData(packageInfo: PackageInfo, token: Long, salt: String) { + Log.i(TAG, "Clearing K/V data of ${packageInfo.packageName}") + val name = state?.name ?: crypto.getNameForPackage(salt, packageInfo.packageName) + plugin.removeData(token, name) + if (!dbManager.deleteDb(packageInfo.packageName)) throw IOException() } - fun finishBackup(): Int { - Log.i(TAG, "Finish K/V Backup of ${state!!.packageInfo.packageName}") - plugin.packageFinished(state!!.packageInfo) - state = null + @Throws(IOException::class) + suspend fun finishBackup(): Int { + val state = this.state ?: error("No state in finishBackup") + val packageName = state.packageInfo.packageName + Log.i(TAG, "Finish K/V Backup of $packageName") + + try { + if (state.needsUpload) uploadDb(state.token, state.name, packageName, state.db) + } catch (e: IOException) { + return TRANSPORT_ERROR + } finally { + this.state = null + } return TRANSPORT_OK } @@ -234,17 +207,43 @@ internal class KVBackup( * because [finishBackup] is not called when we don't return [TRANSPORT_OK]. */ private fun backupError(result: Int): Int { - "Resetting state because of K/V Backup error of ${state!!.packageInfo.packageName}".let { - Log.i(TAG, it) - } - plugin.packageFinished(state!!.packageInfo) - state = null + val state = this.state ?: error("No state in backupError") + val packageName = state.packageInfo.packageName + Log.i(TAG, "Resetting state because of K/V Backup error of $packageName") + + state.db.close() + + this.state = null return result } + @Throws(IOException::class) + private suspend fun uploadDb( + token: Long, + name: String, + packageName: String, + db: KVDb + ) { + db.vacuum() + db.close() + + plugin.getOutputStream(token, name).use { outputStream -> + outputStream.write(ByteArray(1) { VERSION }) + val ad = getADForKV(VERSION, packageName) + crypto.newEncryptingStream(outputStream, ad).use { encryptedStream -> + GZIPOutputStream(encryptedStream).use { gZipStream -> + dbManager.getDbInputStream(packageName).use { inputStream -> + inputStream.copyTo(gZipStream) + } + // TODO remove log + Log.d(TAG, "=> Uploaded db file for $packageName") + } + } + } + } + private class KVOperation( val key: String, - val base64Key: String, /** * value is null when this is a deletion operation */ diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVDbManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVDbManager.kt new file mode 100644 index 00000000..7689220c --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVDbManager.kt @@ -0,0 +1,125 @@ +package com.stevesoltys.seedvault.transport.backup + +import android.content.ContentValues +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import android.database.sqlite.SQLiteDatabase.CONFLICT_REPLACE +import android.database.sqlite.SQLiteOpenHelper +import android.provider.BaseColumns +import java.io.File +import java.io.FileInputStream +import java.io.InputStream + +interface KvDbManager { + fun getDb(packageName: String): KVDb + fun getDbInputStream(packageName: String): InputStream + fun existsDb(packageName: String): Boolean + fun deleteDb(packageName: String): Boolean +} + +class KvDbManagerImpl(private val context: Context) : KvDbManager { + + override fun getDb(packageName: String): KVDb { + return KVDbImpl(context, getFileName(packageName)) + } + + private fun getFileName(packageName: String) = "kv_$packageName.db" + + private fun getDbFile(packageName: String): File { + return context.getDatabasePath(getFileName(packageName)) + } + + override fun getDbInputStream(packageName: String): InputStream { + return FileInputStream(getDbFile(packageName)) + } + + override fun existsDb(packageName: String): Boolean { + return getDbFile(packageName).isFile + } + + override fun deleteDb(packageName: String): Boolean { + return getDbFile(packageName).delete() + } +} + +interface KVDb { + fun put(key: String, value: ByteArray) + fun get(key: String): ByteArray? + fun getAll(): List> + fun delete(key: String) + fun vacuum() + fun close() +} + +class KVDbImpl(context: Context, fileName: String) : + SQLiteOpenHelper(context, fileName, null, DATABASE_VERSION), KVDb { + + companion object { + private const val DATABASE_VERSION = 1 + + private object KVEntry : BaseColumns { + const val TABLE_NAME = "kv_entry" + const val COLUMN_NAME_KEY = "key" + const val COLUMN_NAME_VALUE = "value" + } + + private const val SQL_CREATE_ENTRIES = + "CREATE TABLE IF NOT EXISTS ${KVEntry.TABLE_NAME} (" + + "${KVEntry.COLUMN_NAME_KEY} TEXT PRIMARY KEY," + + "${KVEntry.COLUMN_NAME_VALUE} BLOB)" + } + + override fun onCreate(db: SQLiteDatabase) { + db.execSQL(SQL_CREATE_ENTRIES) + } + + override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + } + + override fun onDowngrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + } + + override fun vacuum() = writableDatabase.execSQL("VACUUM") + + override fun put(key: String, value: ByteArray) { + val values = ContentValues().apply { + put(KVEntry.COLUMN_NAME_KEY, key) + put(KVEntry.COLUMN_NAME_VALUE, value) + } + writableDatabase.insertWithOnConflict(KVEntry.TABLE_NAME, null, values, CONFLICT_REPLACE) + } + + override fun get(key: String): ByteArray? = readableDatabase.query( + KVEntry.TABLE_NAME, + arrayOf(KVEntry.COLUMN_NAME_VALUE), + "${KVEntry.COLUMN_NAME_KEY} = ?", + arrayOf(key), + null, + null, + null + ).use { cursor -> + if (!cursor.moveToNext()) null + else cursor.getBlob(0) + } + + override fun getAll(): List> = readableDatabase.query( + KVEntry.TABLE_NAME, + arrayOf(KVEntry.COLUMN_NAME_KEY, KVEntry.COLUMN_NAME_VALUE), + null, + null, + null, + null, + null + ).use { cursor -> + val list = ArrayList>(cursor.count) + while (cursor.moveToNext()) { + list.add(Pair(cursor.getString(0), cursor.getBlob(1))) + } + list + } + + override fun delete(key: String) { + writableDatabase.delete(KVEntry.TABLE_NAME, "${KVEntry.COLUMN_NAME_KEY} = ?", arrayOf(key)) + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/ui/notification/BackupNotificationManager.kt b/app/src/main/java/com/stevesoltys/seedvault/ui/notification/BackupNotificationManager.kt index ea8606a8..06ab6242 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/ui/notification/BackupNotificationManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/ui/notification/BackupNotificationManager.kt @@ -90,6 +90,7 @@ internal class BackupNotificationManager(private val context: Context) { /** * This is expected to get called before [onOptOutAppBackup] and [onBackupUpdate]. */ + // TODO remove? fun onPmKvBackup(packageName: String, transferred: Int, expected: Int) { val text = "@pm@ record for $packageName" if (expectedApps == null) { diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt index 7c52c3de..73b25085 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt @@ -21,13 +21,11 @@ import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA import com.stevesoltys.seedvault.transport.backup.ApkBackup import com.stevesoltys.seedvault.transport.backup.BackupCoordinator import com.stevesoltys.seedvault.transport.backup.BackupPlugin -import com.stevesoltys.seedvault.transport.backup.DEFAULT_QUOTA_FULL_BACKUP import com.stevesoltys.seedvault.transport.backup.FullBackup -import com.stevesoltys.seedvault.transport.backup.FullBackupPlugin import com.stevesoltys.seedvault.transport.backup.InputFactory import com.stevesoltys.seedvault.transport.backup.KVBackup -import com.stevesoltys.seedvault.transport.backup.KVBackupPlugin import com.stevesoltys.seedvault.transport.backup.PackageService +import com.stevesoltys.seedvault.transport.backup.TestKvDbManager import com.stevesoltys.seedvault.transport.restore.FullRestore import com.stevesoltys.seedvault.transport.restore.FullRestorePlugin import com.stevesoltys.seedvault.transport.restore.KVRestore @@ -61,23 +59,12 @@ internal class CoordinatorIntegrationTest : TransportTest() { private val cryptoImpl = CryptoImpl(keyManager, cipherFactory, headerReader) private val metadataReader = MetadataReaderImpl(cryptoImpl) private val notificationManager = mockk() + private val dbManager = TestKvDbManager() private val backupPlugin = mockk() - private val kvBackupPlugin = mockk() - private val kvBackup = KVBackup( - plugin = kvBackupPlugin, - settingsManager = settingsManager, - inputFactory = inputFactory, - crypto = cryptoImpl, - nm = notificationManager - ) - private val fullBackupPlugin = mockk() - private val fullBackup = FullBackup( - plugin = backupPlugin, - settingsManager = settingsManager, - inputFactory = inputFactory, - crypto = cryptoImpl - ) + private val kvBackup = + KVBackup(backupPlugin, settingsManager, inputFactory, cryptoImpl, dbManager) + private val fullBackup = FullBackup(backupPlugin, settingsManager, inputFactory, cryptoImpl) private val apkBackup = mockk() private val packageService: PackageService = mockk() private val backup = BackupCoordinator( @@ -121,12 +108,8 @@ internal class CoordinatorIntegrationTest : TransportTest() { private val key2 = "RestoreKey2" private val key264 = key2.encodeBase64() - init { - @Suppress("deprecation") - every { backupPlugin.kvBackupPlugin } returns kvBackupPlugin - @Suppress("deprecation") - every { backupPlugin.fullBackupPlugin } returns fullBackupPlugin - } + // as we use real crypto, we need a real name for packageInfo + private val realName = cryptoImpl.getNameForPackage(salt, packageInfo.packageName) @Test fun `test key-value backup and restore with 2 records`() = runBlocking { @@ -135,8 +118,9 @@ internal class CoordinatorIntegrationTest : TransportTest() { val bOutputStream = ByteArrayOutputStream() val bOutputStream2 = ByteArrayOutputStream() + every { settingsManager.getToken() } returns token + every { metadataManager.salt } returns salt // read one key/value record and write it to output stream - coEvery { kvBackupPlugin.hasDataForPackage(packageInfo) } returns false every { inputFactory.getBackupDataInput(fileDescriptor) } returns backupDataInput every { backupDataInput.readNextHeader() } returns true andThen true andThen false every { backupDataInput.key } returns key andThen key2 @@ -145,31 +129,13 @@ internal class CoordinatorIntegrationTest : TransportTest() { appData.copyInto(value.captured) // write the app data into the passed ByteArray appData.size } - coEvery { - kvBackupPlugin.getOutputStreamForRecord( - packageInfo, - key64 - ) - } returns bOutputStream every { backupDataInput.readEntityData(capture(value2), 0, appData2.size) } answers { appData2.copyInto(value2.captured) // write the app data into the passed ByteArray appData2.size } coEvery { - kvBackupPlugin.getOutputStreamForRecord( - packageInfo, - key264 - ) - } returns bOutputStream2 - every { kvBackupPlugin.packageFinished(packageInfo) } just Runs - coEvery { - apkBackup.backupApkIfNecessary( - packageInfo, - UNKNOWN_ERROR, - any() - ) + apkBackup.backupApkIfNecessary(packageInfo, UNKNOWN_ERROR, any()) } returns packageMetadata - every { settingsManager.getToken() } returns token coEvery { backupPlugin.getOutputStream(token, FILE_BACKUP_METADATA) } returns metadataOutputStream @@ -180,8 +146,13 @@ internal class CoordinatorIntegrationTest : TransportTest() { metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream) } just Runs - // start and finish K/V backup + // start K/V backup assertEquals(TRANSPORT_OK, backup.performIncrementalBackup(packageInfo, fileDescriptor, 0)) + + // upload DB + coEvery { backupPlugin.getOutputStream(token, realName) } returns bOutputStream + + // finish K/V backup assertEquals(TRANSPORT_OK, backup.finishBackup()) // start restore @@ -231,8 +202,9 @@ internal class CoordinatorIntegrationTest : TransportTest() { val appData = ByteArray(size).apply { Random.nextBytes(this) } val bOutputStream = ByteArrayOutputStream() + every { settingsManager.getToken() } returns token + every { metadataManager.salt } returns salt // read one key/value record and write it to output stream - coEvery { kvBackupPlugin.hasDataForPackage(packageInfo) } returns false every { inputFactory.getBackupDataInput(fileDescriptor) } returns backupDataInput every { backupDataInput.readNextHeader() } returns true andThen false every { backupDataInput.key } returns key @@ -241,13 +213,6 @@ internal class CoordinatorIntegrationTest : TransportTest() { appData.copyInto(value.captured) // write the app data into the passed ByteArray appData.size } - coEvery { - kvBackupPlugin.getOutputStreamForRecord( - packageInfo, - key64 - ) - } returns bOutputStream - every { kvBackupPlugin.packageFinished(packageInfo) } just Runs coEvery { apkBackup.backupApkIfNecessary(packageInfo, UNKNOWN_ERROR, any()) } returns null every { settingsManager.getToken() } returns token coEvery { @@ -257,8 +222,13 @@ internal class CoordinatorIntegrationTest : TransportTest() { metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream) } just Runs - // start and finish K/V backup + // start K/V backup assertEquals(TRANSPORT_OK, backup.performIncrementalBackup(packageInfo, fileDescriptor, 0)) + + // upload DB + coEvery { backupPlugin.getOutputStream(token, realName) } returns bOutputStream + + // finish K/V backup assertEquals(TRANSPORT_OK, backup.finishBackup()) // start restore @@ -297,16 +267,13 @@ internal class CoordinatorIntegrationTest : TransportTest() { val packageMetadata = metadata.packageMetadataMap[packageInfo.packageName]!! metadata.packageMetadataMap[packageInfo.packageName] = packageMetadata.copy(backupType = BackupType.FULL) - // as we use real crypto, we need a real name for packageInfo - val name = cryptoImpl.getNameForPackage(salt, packageInfo.packageName) // return streams from plugin and app data val bOutputStream = ByteArrayOutputStream() val bInputStream = ByteArrayInputStream(appData) - coEvery { backupPlugin.getOutputStream(token, name) } returns bOutputStream + coEvery { backupPlugin.getOutputStream(token, realName) } returns bOutputStream every { inputFactory.getInputStream(fileDescriptor) } returns bInputStream every { settingsManager.isQuotaUnlimited() } returns false - every { fullBackupPlugin.getQuota() } returns DEFAULT_QUOTA_FULL_BACKUP coEvery { apkBackup.backupApkIfNecessary( packageInfo, diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt index 6f0ee3dd..f4374c73 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt @@ -222,7 +222,7 @@ internal class BackupCoordinatorTest : BackupTest() { fun `clearing KV backup data throws`() = runBlocking { every { settingsManager.getToken() } returns token every { metadataManager.salt } returns salt - coEvery { kv.clearBackupData(packageInfo) } throws IOException() + coEvery { kv.clearBackupData(packageInfo, token, salt) } throws IOException() assertEquals(TRANSPORT_ERROR, backup.clearBackupData(packageInfo)) } @@ -231,7 +231,7 @@ internal class BackupCoordinatorTest : BackupTest() { fun `clearing full backup data throws`() = runBlocking { every { settingsManager.getToken() } returns token every { metadataManager.salt } returns salt - coEvery { kv.clearBackupData(packageInfo) } just Runs + coEvery { kv.clearBackupData(packageInfo, token, salt) } just Runs coEvery { full.clearBackupData(packageInfo, token, salt) } throws IOException() assertEquals(TRANSPORT_ERROR, backup.clearBackupData(packageInfo)) @@ -241,7 +241,7 @@ internal class BackupCoordinatorTest : BackupTest() { fun `clearing backup data succeeds`() = runBlocking { every { settingsManager.getToken() } returns token every { metadataManager.salt } returns salt - coEvery { kv.clearBackupData(packageInfo) } just Runs + coEvery { kv.clearBackupData(packageInfo, token, salt) } just Runs coEvery { full.clearBackupData(packageInfo, token, salt) } just Runs assertEquals(TRANSPORT_OK, backup.clearBackupData(packageInfo)) @@ -264,7 +264,7 @@ internal class BackupCoordinatorTest : BackupTest() { every { metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream) } just Runs - every { kv.finishBackup() } returns result + coEvery { kv.finishBackup() } returns result every { metadataOutputStream.close() } just Runs assertEquals(result, backup.finishBackup()) @@ -416,8 +416,12 @@ internal class BackupCoordinatorTest : BackupTest() { every { settingsManager.canDoBackupNow() } returns true every { metadataManager.isLegacyFormat } returns false + every { settingsManager.getToken() } returns token + every { metadataManager.salt } returns salt // do actual @pm@ backup - coEvery { kv.performBackup(packageInfo, fileDescriptor, 0) } returns TRANSPORT_OK + coEvery { + kv.performBackup(packageInfo, fileDescriptor, 0, token, salt) + } returns TRANSPORT_OK // now check if we have opt-out apps that we need to back up APKs for every { packageService.notBackedUpPackages } returns notAllowedPackages // update notification diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/FullBackupTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/FullBackupTest.kt index 4e44c9bb..cf311451 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/FullBackupTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/FullBackupTest.kt @@ -39,7 +39,10 @@ internal class FullBackupTest : BackupTest() { fun `checkFullBackupSize exceeds quota`() { every { settingsManager.isQuotaUnlimited() } returns false - assertEquals(TRANSPORT_QUOTA_EXCEEDED, backup.checkFullBackupSize(DEFAULT_QUOTA_FULL_BACKUP + 1)) + assertEquals( + TRANSPORT_QUOTA_EXCEEDED, + backup.checkFullBackupSize(DEFAULT_QUOTA_FULL_BACKUP + 1) + ) } @Test diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/KVBackupTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/KVBackupTest.kt index 5624328d..a4476b64 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/KVBackupTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/KVBackupTest.kt @@ -8,12 +8,10 @@ import android.app.backup.BackupTransport.TRANSPORT_ERROR import android.app.backup.BackupTransport.TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED import android.app.backup.BackupTransport.TRANSPORT_OK import android.content.pm.PackageInfo -import com.stevesoltys.seedvault.Utf8 import com.stevesoltys.seedvault.getRandomString import com.stevesoltys.seedvault.header.MAX_KEY_LENGTH_SIZE import com.stevesoltys.seedvault.header.VERSION import com.stevesoltys.seedvault.header.getADForKV -import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager import io.mockk.CapturingSlot import io.mockk.Runs import io.mockk.coEvery @@ -21,34 +19,30 @@ import io.mockk.every import io.mockk.just import io.mockk.mockk import io.mockk.verify -import io.mockk.verifyOrder import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream import java.io.IOException -import java.util.Base64 import kotlin.random.Random @Suppress("BlockingMethodInNonBlockingContext") internal class KVBackupTest : BackupTest() { - private val plugin = mockk() + private val plugin = mockk() private val dataInput = mockk() - private val notificationManager = mockk() + private val dbManager = mockk() - private val backup = KVBackup( - plugin = plugin, - settingsManager = settingsManager, - inputFactory = inputFactory, - crypto = crypto, - nm = notificationManager - ) + private val backup = KVBackup(plugin, settingsManager, inputFactory, crypto, dbManager) + private val db = mockk() + private val packageName = packageInfo.packageName private val key = getRandomString(MAX_KEY_LENGTH_SIZE) - private val key64 = Base64.getEncoder().encodeToString(key.toByteArray(Utf8)) private val dataValue = Random.nextBytes(23) + private val dbBytes = Random.nextBytes(42) + private val inputStream = ByteArrayInputStream(dbBytes) @Test fun `has no initial state`() { @@ -59,82 +53,35 @@ internal class KVBackupTest : BackupTest() { fun `simple backup with one record`() = runBlocking { singleRecordBackup() - assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0)) + assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt)) assertTrue(backup.hasState()) + assertEquals(packageInfo, backup.getCurrentPackage()) assertEquals(TRANSPORT_OK, backup.finishBackup()) assertFalse(backup.hasState()) } - @Test - fun `@pm@ backup shows notification`() = runBlocking { - // init plugin and give back two keys - initPlugin(true, pmPackageInfo) - createBackupDataInput() - every { dataInput.readNextHeader() } returnsMany listOf(true, true, false) - every { dataInput.key } returnsMany listOf("key1", "key2") - // we don't care about values, so just use the same one always - every { dataInput.dataSize } returns dataValue.size - every { dataInput.readEntityData(any(), 0, dataValue.size) } returns dataValue.size - - // store first record and show notification for it - every { notificationManager.onPmKvBackup("key1", 1, 2) } just Runs - coEvery { plugin.getOutputStreamForRecord(pmPackageInfo, "a2V5MQ") } returns outputStream - every { outputStream.write(ByteArray(1) { VERSION }) } just Runs - - // store second record and show notification for it - every { notificationManager.onPmKvBackup("key2", 2, 2) } just Runs - coEvery { plugin.getOutputStreamForRecord(pmPackageInfo, "a2V5Mg") } returns outputStream - - // encrypt to and close output stream - every { crypto.newEncryptingStream(outputStream, any()) } returns encryptedOutputStream - every { encryptedOutputStream.write(any()) } just Runs - every { encryptedOutputStream.flush() } just Runs - every { encryptedOutputStream.close() } just Runs - every { outputStream.flush() } just Runs - every { outputStream.close() } just Runs - - assertEquals(TRANSPORT_OK, backup.performBackup(pmPackageInfo, data, 0)) - assertTrue(backup.hasState()) - - every { plugin.packageFinished(pmPackageInfo) } just Runs - - assertEquals(TRANSPORT_OK, backup.finishBackup()) - assertFalse(backup.hasState()) - - // verify that notifications were shown - verifyOrder { - notificationManager.onPmKvBackup("key1", 1, 2) - notificationManager.onPmKvBackup("key2", 2, 2) - } - } - @Test fun `incremental backup with no data gets rejected`() = runBlocking { - coEvery { plugin.hasDataForPackage(packageInfo) } returns false - every { plugin.packageFinished(packageInfo) } just Runs + initPlugin(false) + every { db.close() } just Runs assertEquals( TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED, - backup.performBackup(packageInfo, data, FLAG_INCREMENTAL) + backup.performBackup(packageInfo, data, FLAG_INCREMENTAL, token, salt) ) assertFalse(backup.hasState()) } - @Test - fun `check for existing data throws exception`() = runBlocking { - coEvery { plugin.hasDataForPackage(packageInfo) } throws IOException() - every { plugin.packageFinished(packageInfo) } just Runs - - assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0)) - assertFalse(backup.hasState()) - } - @Test fun `non-incremental backup with data clears old data first`() = runBlocking { singleRecordBackup(true) - coEvery { plugin.removeDataOfPackage(packageInfo) } just Runs + coEvery { plugin.removeData(token, name) } just Runs + every { dbManager.deleteDb(packageName) } returns true - assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL)) + assertEquals( + TRANSPORT_OK, + backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL, token, salt) + ) assertTrue(backup.hasState()) assertEquals(TRANSPORT_OK, backup.finishBackup()) assertFalse(backup.hasState()) @@ -144,11 +91,11 @@ internal class KVBackupTest : BackupTest() { fun `ignoring exception when clearing data when non-incremental backup has data`() = runBlocking { singleRecordBackup(true) - coEvery { plugin.removeDataOfPackage(packageInfo) } throws IOException() + coEvery { plugin.removeData(token, name) } throws IOException() assertEquals( TRANSPORT_OK, - backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL) + backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL, token, salt) ) assertTrue(backup.hasState()) assertEquals(TRANSPORT_OK, backup.finishBackup()) @@ -157,15 +104,18 @@ internal class KVBackupTest : BackupTest() { @Test fun `package with no new data comes back ok right away`() = runBlocking { + every { crypto.getNameForPackage(salt, packageName) } returns name + every { dbManager.getDb(packageName) } returns db every { data.close() } just Runs - assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, FLAG_DATA_NOT_CHANGED)) + assertEquals( + TRANSPORT_OK, + backup.performBackup(packageInfo, data, FLAG_DATA_NOT_CHANGED, token, salt) + ) assertTrue(backup.hasState()) verify { data.close() } - every { plugin.packageFinished(packageInfo) } just Runs - assertEquals(TRANSPORT_OK, backup.finishBackup()) assertFalse(backup.hasState()) } @@ -175,9 +125,9 @@ internal class KVBackupTest : BackupTest() { initPlugin(false) createBackupDataInput() every { dataInput.readNextHeader() } throws IOException() - every { plugin.packageFinished(packageInfo) } just Runs + every { db.close() } just Runs - assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0)) + assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0, token, salt)) assertFalse(backup.hasState()) } @@ -189,9 +139,9 @@ internal class KVBackupTest : BackupTest() { every { dataInput.key } returns key every { dataInput.dataSize } returns dataValue.size every { dataInput.readEntityData(any(), 0, dataValue.size) } throws IOException() - every { plugin.packageFinished(packageInfo) } just Runs + every { db.close() } just Runs - assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0)) + assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0, token, salt)) assertFalse(backup.hasState()) } @@ -199,24 +149,46 @@ internal class KVBackupTest : BackupTest() { fun `no data records`() = runBlocking { initPlugin(false) getDataInput(listOf(false)) - every { plugin.packageFinished(packageInfo) } just Runs - assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0)) + assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt)) assertTrue(backup.hasState()) assertEquals(TRANSPORT_OK, backup.finishBackup()) assertFalse(backup.hasState()) } + @Test + fun `null data deletes key`() = runBlocking { + initPlugin(true) + createBackupDataInput() + every { dataInput.readNextHeader() } returns true andThen false + every { dataInput.key } returns key + every { dataInput.dataSize } returns -1 // just documented by example code in LocalTransport + every { db.delete(key) } just Runs + + assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt)) + assertTrue(backup.hasState()) + + uploadData() + + assertEquals(TRANSPORT_OK, backup.finishBackup()) + assertFalse(backup.hasState()) + } + @Test fun `exception while writing version`() = runBlocking { initPlugin(false) - getDataInput(listOf(true)) - coEvery { plugin.getOutputStreamForRecord(packageInfo, key64) } returns outputStream + getDataInput(listOf(true, false)) + every { db.put(key, dataValue) } just Runs + + assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt)) + assertTrue(backup.hasState()) + + every { db.vacuum() } just Runs + every { db.close() } just Runs + coEvery { plugin.getOutputStream(token, name) } returns outputStream every { outputStream.write(ByteArray(1) { VERSION }) } throws IOException() every { outputStream.close() } just Runs - every { plugin.packageFinished(packageInfo) } just Runs - - assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0)) + assertEquals(TRANSPORT_ERROR, backup.finishBackup()) assertFalse(backup.hasState()) verify { outputStream.close() } @@ -224,65 +196,41 @@ internal class KVBackupTest : BackupTest() { @Test fun `exception while writing encrypted value to output stream`() = runBlocking { - initPlugin(false) - getDataInput(listOf(true)) - writeVersionAndEncrypt() - every { encryptedOutputStream.write(dataValue) } throws IOException() - every { plugin.packageFinished(packageInfo) } just Runs - - assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0)) - assertFalse(backup.hasState()) - - verify { outputStream.close() } - } - - @Test - fun `exception while flushing output stream`() = runBlocking { - initPlugin(false) - getDataInput(listOf(true)) - writeVersionAndEncrypt() - every { encryptedOutputStream.write(dataValue) } just Runs - every { encryptedOutputStream.flush() } throws IOException() - every { encryptedOutputStream.close() } just Runs - every { outputStream.close() } just Runs - every { plugin.packageFinished(packageInfo) } just Runs - - assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0)) - assertFalse(backup.hasState()) - - verify { outputStream.close() } - } - - @Test - fun `ignoring exception while closing output stream`() = runBlocking { initPlugin(false) getDataInput(listOf(true, false)) - writeVersionAndEncrypt() - every { encryptedOutputStream.write(dataValue) } just Runs - every { encryptedOutputStream.flush() } just Runs - every { encryptedOutputStream.close() } just Runs - every { outputStream.close() } just Runs - every { plugin.packageFinished(packageInfo) } just Runs + every { db.put(key, dataValue) } just Runs - assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0)) + assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt)) assertTrue(backup.hasState()) - assertEquals(TRANSPORT_OK, backup.finishBackup()) + + every { db.vacuum() } just Runs + every { db.close() } just Runs + coEvery { plugin.getOutputStream(token, name) } returns outputStream + every { outputStream.write(ByteArray(1) { VERSION }) } just Runs + val ad = getADForKV(VERSION, packageInfo.packageName) + every { crypto.newEncryptingStream(outputStream, ad) } returns encryptedOutputStream + every { encryptedOutputStream.write(any()) } throws IOException() + + assertEquals(TRANSPORT_ERROR, backup.finishBackup()) assertFalse(backup.hasState()) + + verify { + encryptedOutputStream.close() + outputStream.close() + } } private fun singleRecordBackup(hasDataForPackage: Boolean = false) { initPlugin(hasDataForPackage) + every { db.put(key, dataValue) } just Runs getDataInput(listOf(true, false)) - writeVersionAndEncrypt() - every { encryptedOutputStream.write(dataValue) } just Runs - every { encryptedOutputStream.flush() } just Runs - every { encryptedOutputStream.close() } just Runs - every { outputStream.close() } just Runs - every { plugin.packageFinished(packageInfo) } just Runs + uploadData() } private fun initPlugin(hasDataForPackage: Boolean = false, pi: PackageInfo = packageInfo) { - coEvery { plugin.hasDataForPackage(pi) } returns hasDataForPackage + every { dbManager.existsDb(pi.packageName) } returns hasDataForPackage + every { crypto.getNameForPackage(salt, pi.packageName) } returns name + every { dbManager.getDb(pi.packageName) } returns db } private fun createBackupDataInput() { @@ -301,11 +249,19 @@ internal class KVBackupTest : BackupTest() { } } - private fun writeVersionAndEncrypt() { - coEvery { plugin.getOutputStreamForRecord(packageInfo, key64) } returns outputStream + private fun uploadData() { + every { db.vacuum() } just Runs + every { db.close() } just Runs + + coEvery { plugin.getOutputStream(token, name) } returns outputStream every { outputStream.write(ByteArray(1) { VERSION }) } just Runs val ad = getADForKV(VERSION, packageInfo.packageName) every { crypto.newEncryptingStream(outputStream, ad) } returns encryptedOutputStream + every { encryptedOutputStream.write(any()) } just Runs // gzip header + every { encryptedOutputStream.write(any(), any(), any()) } just Runs // stream copy + every { dbManager.getDbInputStream(packageName) } returns inputStream + every { encryptedOutputStream.close() } just Runs + every { outputStream.close() } just Runs } } diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/TestKvDbManager.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/TestKvDbManager.kt new file mode 100644 index 00000000..34a5e0d4 --- /dev/null +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/TestKvDbManager.kt @@ -0,0 +1,138 @@ +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.getRandomString +import com.stevesoltys.seedvault.toByteArrayFromHex +import com.stevesoltys.seedvault.toHexString +import junit.framework.Assert.assertEquals +import junit.framework.Assert.assertFalse +import junit.framework.Assert.assertNull +import junit.framework.Assert.assertTrue +import org.json.JSONObject +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream +import java.io.InputStream +import kotlin.random.Random + +class TestKvDbManager : KvDbManager { + + private var db: TestKVDb? = null + + override fun getDb(packageName: String): KVDb { + return TestKVDb().apply { db = this } + } + + override fun getDbInputStream(packageName: String): InputStream { + return ByteArrayInputStream(db!!.serialize().toByteArray()) + } + + override fun existsDb(packageName: String): Boolean { + return db != null + } + + override fun deleteDb(packageName: String): Boolean { + clearDb() + return true + } + + fun clearDb() { + this.db = null + } + + fun readDbFromStream(inputStream: InputStream) { + this.db = TestKVDb.deserialize(String(inputStream.readBytes())) + } +} + +class TestKVDb(private val json: JSONObject = JSONObject()) : KVDb { + + override fun put(key: String, value: ByteArray) { + json.put(key, value.toHexString(spacer = "")) + } + + override fun get(key: String): ByteArray? { + return json.getByteArray(key) + } + + override fun getAll(): List> { + val list = ArrayList>(json.length()) + json.keys().forEach { key -> + val bytes = json.getByteArray(key) + if (bytes != null) list.add(Pair(key, bytes)) + } + return list + } + + override fun delete(key: String) { + json.remove(key) + } + + override fun vacuum() { + } + + override fun close() { + } + + fun serialize(): String { + return json.toString() + } + + companion object { + fun deserialize(str: String): TestKVDb { + return TestKVDb(JSONObject(str)) + } + } + + private fun JSONObject.getByteArray(key: String): ByteArray? { + val str = optString(key, "") + if (str.isNullOrEmpty()) return null + return str.toByteArrayFromHex() + } + +} + +class TestKvDbManagerTest { + + private val dbManager = TestKvDbManager() + + private val key1 = getRandomString(12) + private val key2 = getRandomString(12) + private val bytes1 = Random.nextBytes(23) + private val bytes2 = Random.nextBytes(23) + + @Test + fun test() { + assertFalse(dbManager.existsDb("foo")) + + val db = dbManager.getDb("foo") + db.put(key1, bytes1) + db.put(key2, bytes2) + assertTrue(dbManager.existsDb("foo")) + + assertArrayEquals(bytes1, db.get(key1)) + assertArrayEquals(bytes2, db.get(key2)) + + val list = db.getAll() + assertEquals(2, list.size) + assertEquals(key1, list[0].first) + assertArrayEquals(bytes1, list[0].second) + assertEquals(key2, list[1].first) + assertArrayEquals(bytes2, list[1].second) + + val dbBytes = dbManager.getDbInputStream("foo").readBytes() + + assertTrue(dbManager.existsDb("foo")) + dbManager.clearDb() + assertFalse(dbManager.existsDb("foo")) + + dbManager.readDbFromStream(ByteArrayInputStream(dbBytes)) + assertTrue(dbManager.existsDb("foo")) + assertArrayEquals(bytes1, db.get(key1)) + assertArrayEquals(bytes2, db.get(key2)) + assertNull(db.get("bar")) + + db.delete(key2) + assertNull(db.get(key2)) + } + +}