mobi/seedvault
1
0
Fork 0
Code Pull requests Activity

K/V backup using single file

Browse source 
Tests are still broken until restore has also been implemented with single file approach
This commit is contained in:
Torsten Grote 2021-09-21 18:50:10 +02:00 committed by Chirayu Desai
parent 23bb385190
commit 0c915e5eb8
10 changed files with 486 additions and 290 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt
View file

@ -268,7 +268,9 @@ internal class BackupCoordinator(
return TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED
}
}
val result = kv.performBackup(packageInfo, data, flags)
val token = settingsManager.getToken() ?: error("no token in performFullBackup")
val salt = metadataManager.salt
val result = kv.performBackup(packageInfo, data, flags, token, salt)
if (result == TRANSPORT_OK && packageName == MAGIC_PACKAGE_MANAGER) {
// hook in here to back up APKs of apps that are otherwise not allowed for backup
backUpApksOfNotBackedUpPackages()
@ -360,7 +362,7 @@ internal class BackupCoordinator(
val token = settingsManager.getToken() ?: error("no token in clearBackupData")
val salt = metadataManager.salt
try {
kv.clearBackupData(packageInfo)
kv.clearBackupData(packageInfo, token, salt)
} catch (e: IOException) {
Log.w(TAG, "Error clearing K/V backup data for $packageName", e)
return TRANSPORT_ERROR

5
app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt
View file

@ -18,13 +18,14 @@ val backupModule = module {
metadataManager = get()
)
}
single<KvDbManager> { KvDbManagerImpl(androidContext()) }
single {
KVBackup(
plugin = get<BackupPlugin>().kvBackupPlugin,
plugin = get(),
settingsManager = get(),
inputFactory = get(),
crypto = get(),
nm = get()
dbManager = get()
)
}
single {

171
app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVBackup.kt
View file

@ -9,16 +9,21 @@ import android.app.backup.BackupTransport.TRANSPORT_OK
import android.content.pm.PackageInfo
import android.os.ParcelFileDescriptor
import android.util.Log
import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER
import com.stevesoltys.seedvault.crypto.Crypto
import com.stevesoltys.seedvault.encodeBase64
import com.stevesoltys.seedvault.header.VERSION
import com.stevesoltys.seedvault.header.getADForKV
import com.stevesoltys.seedvault.settings.SettingsManager
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import java.io.IOException
import java.util.zip.GZIPOutputStream
class KVBackupState(internal val packageInfo: PackageInfo)
class KVBackupState(
internal val packageInfo: PackageInfo,
val token: Long,
val name: String,
val db: KVDb
) {
var needsUpload: Boolean = false
}
const val DEFAULT_QUOTA_KEY_VALUE_BACKUP = (2 * (5 * 1024 * 1024)).toLong()
@ -26,11 +31,11 @@ private val TAG = KVBackup::class.java.simpleName
@Suppress("BlockingMethodInNonBlockingContext")
internal class KVBackup(
private val plugin: KVBackupPlugin,
private val plugin: BackupPlugin,
private val settingsManager: SettingsManager,
private val inputFactory: InputFactory,
private val crypto: Crypto,
private val nm: BackupNotificationManager
private val dbManager: KvDbManager
) {
private var state: KVBackupState? = null
@ -39,14 +44,18 @@ internal class KVBackup(
fun getCurrentPackage() = state?.packageInfo
fun getQuota(): Long {
return if (settingsManager.isQuotaUnlimited()) Long.MAX_VALUE else plugin.getQuota()
fun getQuota(): Long = if (settingsManager.isQuotaUnlimited()) {
Long.MAX_VALUE
} else {
DEFAULT_QUOTA_KEY_VALUE_BACKUP
}
suspend fun performBackup(
packageInfo: PackageInfo,
data: ParcelFileDescriptor,
flags: Int
flags: Int,
token: Long,
salt: String
): Int {
val dataNotChanged = flags and FLAG_DATA_NOT_CHANGED != 0
val isIncremental = flags and FLAG_INCREMENTAL != 0
@ -73,7 +82,9 @@ internal class KVBackup(
if (state != null) {
throw AssertionError("Have state for ${state.packageInfo.packageName}")
}
this.state = KVBackupState(packageInfo)
val name = crypto.getNameForPackage(salt, packageName)
val db = dbManager.getDb(packageName)
this.state = KVBackupState(packageInfo, token, name, db)
// no need for backup when no data has changed
if (dataNotChanged) {
@ -82,12 +93,7 @@ internal class KVBackup(
}
// check if we have existing data for the given package
val hasDataForPackage = try {
plugin.hasDataForPackage(packageInfo)
} catch (e: IOException) {
Log.e(TAG, "Error checking for existing data for ${packageInfo.packageName}.", e)
return backupError(TRANSPORT_ERROR)
}
val hasDataForPackage = dbManager.existsDb(packageName)
if (isIncremental && !hasDataForPackage) {
Log.w(
TAG, "Requested incremental, but transport currently stores no data" +
@ -101,80 +107,36 @@ internal class KVBackup(
if (isNonIncremental && hasDataForPackage) {
Log.w(TAG, "Requested non-incremental, deleting existing data.")
try {
clearBackupData(packageInfo)
clearBackupData(packageInfo, token, salt)
} catch (e: IOException) {
Log.w(TAG, "Error clearing backup data for ${packageInfo.packageName}.", e)
}
}
// parse and store the K/V updates
return storeRecords(packageInfo, data)
return storeRecords(data)
}
private suspend fun storeRecords(packageInfo: PackageInfo, data: ParcelFileDescriptor): Int {
val backupSequence: Iterable<Result<KVOperation>>
val pmRecordNumber: Int?
if (packageInfo.packageName == MAGIC_PACKAGE_MANAGER) {
// Since the package manager has many small keys to store,
// and this can be slow, especially on cloud-based storage,
// we get the entire data set first, so we can show progress notifications.
val list = parseBackupStream(data).toList()
backupSequence = list
pmRecordNumber = list.size
} else {
backupSequence = parseBackupStream(data).asIterable()
pmRecordNumber = null
}
private fun storeRecords(data: ParcelFileDescriptor): Int {
val state = this.state ?: error("No state in storeRecords")
// apply the delta operations
var i = 1
for (result in backupSequence) {
for (result in parseBackupStream(data)) {
if (result is Result.Error) {
Log.e(TAG, "Exception reading backup input", result.exception)
return backupError(TRANSPORT_ERROR)
}
state.needsUpload = true
val op = (result as Result.Ok).result
try {
storeRecord(packageInfo, op, i++, pmRecordNumber)
} catch (e: IOException) {
Log.e(TAG, "Unable to update base64Key file for base64Key ${op.base64Key}", e)
// Returning something more forgiving such as TRANSPORT_PACKAGE_REJECTED
// will still make the entire backup fail.
// TODO However, TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED might buy us a retry,
// we would just need to be careful not to create an infinite loop
// for permanent errors.
return backupError(TRANSPORT_ERROR)
if (op.value == null) {
Log.e(TAG, "Deleting record with key ${op.key}")
state.db.delete(op.key)
} else {
state.db.put(op.key, op.value)
}
}
return TRANSPORT_OK
}
@Throws(IOException::class)
private suspend fun storeRecord(
packageInfo: PackageInfo,
op: KVOperation,
currentNum: Int,
pmRecordNumber: Int?
) {
// update notification for package manager backup
if (pmRecordNumber != null) {
nm.onPmKvBackup(op.key, currentNum, pmRecordNumber)
}
// check if record should get deleted
if (op.value == null) {
Log.e(TAG, "Deleting record with base64Key ${op.base64Key}")
plugin.deleteRecord(packageInfo, op.base64Key)
} else {
plugin.getOutputStreamForRecord(packageInfo, op.base64Key).use { outputStream ->
outputStream.write(ByteArray(1) { VERSION })
val ad = getADForKV(VERSION, packageInfo.packageName)
crypto.newEncryptingStream(outputStream, ad).use { encryptedStream ->
encryptedStream.write(op.value)
encryptedStream.flush()
}
}
}
}
/**
* Parses a backup stream into individual key/value operations
*/
@ -194,12 +156,11 @@ internal class KVBackup(
}
// encode key
val key = changeSet.key
val base64Key = key.encodeBase64()
val dataSize = changeSet.dataSize
// read value
val value = if (dataSize >= 0) {
Log.v(TAG, " Delta operation key $key size $dataSize key64 $base64Key")
Log.v(TAG, " Delta operation key $key size $dataSize")
val bytes = ByteArray(dataSize)
val bytesRead = try {
changeSet.readEntityData(bytes, 0, dataSize)
@ -213,19 +174,31 @@ internal class KVBackup(
bytes
} else null
// add change operation to the sequence
Result.Ok(KVOperation(key, base64Key, value))
Result.Ok(KVOperation(key, value))
}
}
@Throws(IOException::class)
suspend fun clearBackupData(packageInfo: PackageInfo) {
plugin.removeDataOfPackage(packageInfo)
suspend fun clearBackupData(packageInfo: PackageInfo, token: Long, salt: String) {
Log.i(TAG, "Clearing K/V data of ${packageInfo.packageName}")
val name = state?.name ?: crypto.getNameForPackage(salt, packageInfo.packageName)
plugin.removeData(token, name)
if (!dbManager.deleteDb(packageInfo.packageName)) throw IOException()
}
fun finishBackup(): Int {
Log.i(TAG, "Finish K/V Backup of ${state!!.packageInfo.packageName}")
plugin.packageFinished(state!!.packageInfo)
state = null
@Throws(IOException::class)
suspend fun finishBackup(): Int {
val state = this.state ?: error("No state in finishBackup")
val packageName = state.packageInfo.packageName
Log.i(TAG, "Finish K/V Backup of $packageName")
try {
if (state.needsUpload) uploadDb(state.token, state.name, packageName, state.db)
} catch (e: IOException) {
return TRANSPORT_ERROR
} finally {
this.state = null
}
return TRANSPORT_OK
}
@ -234,17 +207,43 @@ internal class KVBackup(
* because [finishBackup] is not called when we don't return [TRANSPORT_OK].
*/
private fun backupError(result: Int): Int {
"Resetting state because of K/V Backup error of ${state!!.packageInfo.packageName}".let {
Log.i(TAG, it)
}
plugin.packageFinished(state!!.packageInfo)
state = null
val state = this.state ?: error("No state in backupError")
val packageName = state.packageInfo.packageName
Log.i(TAG, "Resetting state because of K/V Backup error of $packageName")
state.db.close()
this.state = null
return result
}
@Throws(IOException::class)
private suspend fun uploadDb(
token: Long,
name: String,
packageName: String,
db: KVDb
) {
db.vacuum()
db.close()
plugin.getOutputStream(token, name).use { outputStream ->
outputStream.write(ByteArray(1) { VERSION })
val ad = getADForKV(VERSION, packageName)
crypto.newEncryptingStream(outputStream, ad).use { encryptedStream ->
GZIPOutputStream(encryptedStream).use { gZipStream ->
dbManager.getDbInputStream(packageName).use { inputStream ->
inputStream.copyTo(gZipStream)
}
// TODO remove log
Log.d(TAG, "=> Uploaded db file for $packageName")
}
}
}
}
private class KVOperation(
val key: String,
val base64Key: String,
/**
* value is null when this is a deletion operation
*/

125
app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVDbManager.kt Normal file
View file

@ -0,0 +1,125 @@
package com.stevesoltys.seedvault.transport.backup
import android.content.ContentValues
import android.content.Context
import android.database.sqlite.SQLiteDatabase
import android.database.sqlite.SQLiteDatabase.CONFLICT_REPLACE
import android.database.sqlite.SQLiteOpenHelper
import android.provider.BaseColumns
import java.io.File
import java.io.FileInputStream
import java.io.InputStream
interface KvDbManager {
fun getDb(packageName: String): KVDb
fun getDbInputStream(packageName: String): InputStream
fun existsDb(packageName: String): Boolean
fun deleteDb(packageName: String): Boolean
}
class KvDbManagerImpl(private val context: Context) : KvDbManager {
override fun getDb(packageName: String): KVDb {
return KVDbImpl(context, getFileName(packageName))
}
private fun getFileName(packageName: String) = "kv_$packageName.db"
private fun getDbFile(packageName: String): File {
return context.getDatabasePath(getFileName(packageName))
}
override fun getDbInputStream(packageName: String): InputStream {
return FileInputStream(getDbFile(packageName))
}
override fun existsDb(packageName: String): Boolean {
return getDbFile(packageName).isFile
}
override fun deleteDb(packageName: String): Boolean {
return getDbFile(packageName).delete()
}
}
interface KVDb {
fun put(key: String, value: ByteArray)
fun get(key: String): ByteArray?
fun getAll(): List<Pair<String, ByteArray>>
fun delete(key: String)
fun vacuum()
fun close()
}
class KVDbImpl(context: Context, fileName: String) :
SQLiteOpenHelper(context, fileName, null, DATABASE_VERSION), KVDb {
companion object {
private const val DATABASE_VERSION = 1
private object KVEntry : BaseColumns {
const val TABLE_NAME = "kv_entry"
const val COLUMN_NAME_KEY = "key"
const val COLUMN_NAME_VALUE = "value"
}
private const val SQL_CREATE_ENTRIES =
"CREATE TABLE IF NOT EXISTS ${KVEntry.TABLE_NAME} (" +
"${KVEntry.COLUMN_NAME_KEY} TEXT PRIMARY KEY," +
"${KVEntry.COLUMN_NAME_VALUE} BLOB)"
}
override fun onCreate(db: SQLiteDatabase) {
db.execSQL(SQL_CREATE_ENTRIES)
}
override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
}
override fun onDowngrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
}
override fun vacuum() = writableDatabase.execSQL("VACUUM")
override fun put(key: String, value: ByteArray) {
val values = ContentValues().apply {
put(KVEntry.COLUMN_NAME_KEY, key)
put(KVEntry.COLUMN_NAME_VALUE, value)
}
writableDatabase.insertWithOnConflict(KVEntry.TABLE_NAME, null, values, CONFLICT_REPLACE)
}
override fun get(key: String): ByteArray? = readableDatabase.query(
KVEntry.TABLE_NAME,
arrayOf(KVEntry.COLUMN_NAME_VALUE),
"${KVEntry.COLUMN_NAME_KEY} = ?",
arrayOf(key),
null,
null,
null
).use { cursor ->
if (!cursor.moveToNext()) null
else cursor.getBlob(0)
}
override fun getAll(): List<Pair<String, ByteArray>> = readableDatabase.query(
KVEntry.TABLE_NAME,
arrayOf(KVEntry.COLUMN_NAME_KEY, KVEntry.COLUMN_NAME_VALUE),
null,
null,
null,
null,
null
).use { cursor ->
val list = ArrayList<Pair<String, ByteArray>>(cursor.count)
while (cursor.moveToNext()) {
list.add(Pair(cursor.getString(0), cursor.getBlob(1)))
}
list
}
override fun delete(key: String) {
writableDatabase.delete(KVEntry.TABLE_NAME, "${KVEntry.COLUMN_NAME_KEY} = ?", arrayOf(key))
}
}

1
app/src/main/java/com/stevesoltys/seedvault/ui/notification/BackupNotificationManager.kt
View file

@ -90,6 +90,7 @@ internal class BackupNotificationManager(private val context: Context) {
/**
* This is expected to get called before [onOptOutAppBackup] and [onBackupUpdate].
*/
// TODO remove?
fun onPmKvBackup(packageName: String, transferred: Int, expected: Int) {
val text = "@pm@ record for $packageName"
if (expectedApps == null) {

83
app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt
View file

@ -21,13 +21,11 @@ import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA
import com.stevesoltys.seedvault.transport.backup.ApkBackup
import com.stevesoltys.seedvault.transport.backup.BackupCoordinator
import com.stevesoltys.seedvault.transport.backup.BackupPlugin
import com.stevesoltys.seedvault.transport.backup.DEFAULT_QUOTA_FULL_BACKUP
import com.stevesoltys.seedvault.transport.backup.FullBackup
import com.stevesoltys.seedvault.transport.backup.FullBackupPlugin
import com.stevesoltys.seedvault.transport.backup.InputFactory
import com.stevesoltys.seedvault.transport.backup.KVBackup
import com.stevesoltys.seedvault.transport.backup.KVBackupPlugin
import com.stevesoltys.seedvault.transport.backup.PackageService
import com.stevesoltys.seedvault.transport.backup.TestKvDbManager
import com.stevesoltys.seedvault.transport.restore.FullRestore
import com.stevesoltys.seedvault.transport.restore.FullRestorePlugin
import com.stevesoltys.seedvault.transport.restore.KVRestore
@ -61,23 +59,12 @@ internal class CoordinatorIntegrationTest : TransportTest() {
private val cryptoImpl = CryptoImpl(keyManager, cipherFactory, headerReader)
private val metadataReader = MetadataReaderImpl(cryptoImpl)
private val notificationManager = mockk<BackupNotificationManager>()
private val dbManager = TestKvDbManager()
private val backupPlugin = mockk<BackupPlugin>()
private val kvBackupPlugin = mockk<KVBackupPlugin>()
private val kvBackup = KVBackup(
plugin = kvBackupPlugin,
settingsManager = settingsManager,
inputFactory = inputFactory,
crypto = cryptoImpl,
nm = notificationManager
)
private val fullBackupPlugin = mockk<FullBackupPlugin>()
private val fullBackup = FullBackup(
plugin = backupPlugin,
settingsManager = settingsManager,
inputFactory = inputFactory,
crypto = cryptoImpl
)
private val kvBackup =
KVBackup(backupPlugin, settingsManager, inputFactory, cryptoImpl, dbManager)
private val fullBackup = FullBackup(backupPlugin, settingsManager, inputFactory, cryptoImpl)
private val apkBackup = mockk<ApkBackup>()
private val packageService: PackageService = mockk()
private val backup = BackupCoordinator(
@ -121,12 +108,8 @@ internal class CoordinatorIntegrationTest : TransportTest() {
private val key2 = "RestoreKey2"
private val key264 = key2.encodeBase64()
init {
@Suppress("deprecation")
every { backupPlugin.kvBackupPlugin } returns kvBackupPlugin
@Suppress("deprecation")
every { backupPlugin.fullBackupPlugin } returns fullBackupPlugin
}
// as we use real crypto, we need a real name for packageInfo
private val realName = cryptoImpl.getNameForPackage(salt, packageInfo.packageName)
@Test
fun `test key-value backup and restore with 2 records`() = runBlocking {
@ -135,8 +118,9 @@ internal class CoordinatorIntegrationTest : TransportTest() {
val bOutputStream = ByteArrayOutputStream()
val bOutputStream2 = ByteArrayOutputStream()
every { settingsManager.getToken() } returns token
every { metadataManager.salt } returns salt
// read one key/value record and write it to output stream
coEvery { kvBackupPlugin.hasDataForPackage(packageInfo) } returns false
every { inputFactory.getBackupDataInput(fileDescriptor) } returns backupDataInput
every { backupDataInput.readNextHeader() } returns true andThen true andThen false
every { backupDataInput.key } returns key andThen key2
@ -145,31 +129,13 @@ internal class CoordinatorIntegrationTest : TransportTest() {
appData.copyInto(value.captured) // write the app data into the passed ByteArray
appData.size
}
coEvery {
kvBackupPlugin.getOutputStreamForRecord(
packageInfo,
key64
)
} returns bOutputStream
every { backupDataInput.readEntityData(capture(value2), 0, appData2.size) } answers {
appData2.copyInto(value2.captured) // write the app data into the passed ByteArray
appData2.size
}
coEvery {
kvBackupPlugin.getOutputStreamForRecord(
packageInfo,
key264
)
} returns bOutputStream2
every { kvBackupPlugin.packageFinished(packageInfo) } just Runs
coEvery {
apkBackup.backupApkIfNecessary(
packageInfo,
UNKNOWN_ERROR,
any()
)
apkBackup.backupApkIfNecessary(packageInfo, UNKNOWN_ERROR, any())
} returns packageMetadata
every { settingsManager.getToken() } returns token
coEvery {
backupPlugin.getOutputStream(token, FILE_BACKUP_METADATA)
} returns metadataOutputStream
@ -180,8 +146,13 @@ internal class CoordinatorIntegrationTest : TransportTest() {
metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream)
} just Runs
// start and finish K/V backup
// start K/V backup
assertEquals(TRANSPORT_OK, backup.performIncrementalBackup(packageInfo, fileDescriptor, 0))
// upload DB
coEvery { backupPlugin.getOutputStream(token, realName) } returns bOutputStream
// finish K/V backup
assertEquals(TRANSPORT_OK, backup.finishBackup())
// start restore
@ -231,8 +202,9 @@ internal class CoordinatorIntegrationTest : TransportTest() {
val appData = ByteArray(size).apply { Random.nextBytes(this) }
val bOutputStream = ByteArrayOutputStream()
every { settingsManager.getToken() } returns token
every { metadataManager.salt } returns salt
// read one key/value record and write it to output stream
coEvery { kvBackupPlugin.hasDataForPackage(packageInfo) } returns false
every { inputFactory.getBackupDataInput(fileDescriptor) } returns backupDataInput
every { backupDataInput.readNextHeader() } returns true andThen false
every { backupDataInput.key } returns key
@ -241,13 +213,6 @@ internal class CoordinatorIntegrationTest : TransportTest() {
appData.copyInto(value.captured) // write the app data into the passed ByteArray
appData.size
}
coEvery {
kvBackupPlugin.getOutputStreamForRecord(
packageInfo,
key64
)
} returns bOutputStream
every { kvBackupPlugin.packageFinished(packageInfo) } just Runs
coEvery { apkBackup.backupApkIfNecessary(packageInfo, UNKNOWN_ERROR, any()) } returns null
every { settingsManager.getToken() } returns token
coEvery {
@ -257,8 +222,13 @@ internal class CoordinatorIntegrationTest : TransportTest() {
metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream)
} just Runs
// start and finish K/V backup
// start K/V backup
assertEquals(TRANSPORT_OK, backup.performIncrementalBackup(packageInfo, fileDescriptor, 0))
// upload DB
coEvery { backupPlugin.getOutputStream(token, realName) } returns bOutputStream
// finish K/V backup
assertEquals(TRANSPORT_OK, backup.finishBackup())
// start restore
@ -297,16 +267,13 @@ internal class CoordinatorIntegrationTest : TransportTest() {
val packageMetadata = metadata.packageMetadataMap[packageInfo.packageName]!!
metadata.packageMetadataMap[packageInfo.packageName] =
packageMetadata.copy(backupType = BackupType.FULL)
// as we use real crypto, we need a real name for packageInfo
val name = cryptoImpl.getNameForPackage(salt, packageInfo.packageName)
// return streams from plugin and app data
val bOutputStream = ByteArrayOutputStream()
val bInputStream = ByteArrayInputStream(appData)
coEvery { backupPlugin.getOutputStream(token, name) } returns bOutputStream
coEvery { backupPlugin.getOutputStream(token, realName) } returns bOutputStream
every { inputFactory.getInputStream(fileDescriptor) } returns bInputStream
every { settingsManager.isQuotaUnlimited() } returns false
every { fullBackupPlugin.getQuota() } returns DEFAULT_QUOTA_FULL_BACKUP
coEvery {
apkBackup.backupApkIfNecessary(
packageInfo,

14
app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt
View file

@ -222,7 +222,7 @@ internal class BackupCoordinatorTest : BackupTest() {
fun `clearing KV backup data throws`() = runBlocking {
every { settingsManager.getToken() } returns token
every { metadataManager.salt } returns salt
coEvery { kv.clearBackupData(packageInfo) } throws IOException()
coEvery { kv.clearBackupData(packageInfo, token, salt) } throws IOException()
assertEquals(TRANSPORT_ERROR, backup.clearBackupData(packageInfo))
}
@ -231,7 +231,7 @@ internal class BackupCoordinatorTest : BackupTest() {
fun `clearing full backup data throws`() = runBlocking {
every { settingsManager.getToken() } returns token
every { metadataManager.salt } returns salt
coEvery { kv.clearBackupData(packageInfo) } just Runs
coEvery { kv.clearBackupData(packageInfo, token, salt) } just Runs
coEvery { full.clearBackupData(packageInfo, token, salt) } throws IOException()
assertEquals(TRANSPORT_ERROR, backup.clearBackupData(packageInfo))
@ -241,7 +241,7 @@ internal class BackupCoordinatorTest : BackupTest() {
fun `clearing backup data succeeds`() = runBlocking {
every { settingsManager.getToken() } returns token
every { metadataManager.salt } returns salt
coEvery { kv.clearBackupData(packageInfo) } just Runs
coEvery { kv.clearBackupData(packageInfo, token, salt) } just Runs
coEvery { full.clearBackupData(packageInfo, token, salt) } just Runs
assertEquals(TRANSPORT_OK, backup.clearBackupData(packageInfo))
@ -264,7 +264,7 @@ internal class BackupCoordinatorTest : BackupTest() {
every {
metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream)
} just Runs
every { kv.finishBackup() } returns result
coEvery { kv.finishBackup() } returns result
every { metadataOutputStream.close() } just Runs
assertEquals(result, backup.finishBackup())
@ -416,8 +416,12 @@ internal class BackupCoordinatorTest : BackupTest() {
every { settingsManager.canDoBackupNow() } returns true
every { metadataManager.isLegacyFormat } returns false
every { settingsManager.getToken() } returns token
every { metadataManager.salt } returns salt
// do actual @pm@ backup
coEvery { kv.performBackup(packageInfo, fileDescriptor, 0) } returns TRANSPORT_OK
coEvery {
kv.performBackup(packageInfo, fileDescriptor, 0, token, salt)
} returns TRANSPORT_OK
// now check if we have opt-out apps that we need to back up APKs for
every { packageService.notBackedUpPackages } returns notAllowedPackages
// update notification

5
app/src/test/java/com/stevesoltys/seedvault/transport/backup/FullBackupTest.kt
View file

@ -39,7 +39,10 @@ internal class FullBackupTest : BackupTest() {
fun `checkFullBackupSize exceeds quota`() {
every { settingsManager.isQuotaUnlimited() } returns false
assertEquals(TRANSPORT_QUOTA_EXCEEDED, backup.checkFullBackupSize(DEFAULT_QUOTA_FULL_BACKUP + 1))
assertEquals(
TRANSPORT_QUOTA_EXCEEDED,
backup.checkFullBackupSize(DEFAULT_QUOTA_FULL_BACKUP + 1)
)
}
@Test

228
app/src/test/java/com/stevesoltys/seedvault/transport/backup/KVBackupTest.kt
View file

@ -8,12 +8,10 @@ import android.app.backup.BackupTransport.TRANSPORT_ERROR
import android.app.backup.BackupTransport.TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED
import android.app.backup.BackupTransport.TRANSPORT_OK
import android.content.pm.PackageInfo
import com.stevesoltys.seedvault.Utf8
import com.stevesoltys.seedvault.getRandomString
import com.stevesoltys.seedvault.header.MAX_KEY_LENGTH_SIZE
import com.stevesoltys.seedvault.header.VERSION
import com.stevesoltys.seedvault.header.getADForKV
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import io.mockk.CapturingSlot
import io.mockk.Runs
import io.mockk.coEvery
@ -21,34 +19,30 @@ import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.verify
import io.mockk.verifyOrder
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.io.ByteArrayInputStream
import java.io.IOException
import java.util.Base64
import kotlin.random.Random
@Suppress("BlockingMethodInNonBlockingContext")
internal class KVBackupTest : BackupTest() {
private val plugin = mockk<KVBackupPlugin>()
private val plugin = mockk<BackupPlugin>()
private val dataInput = mockk<BackupDataInput>()
private val notificationManager = mockk<BackupNotificationManager>()
private val dbManager = mockk<KvDbManager>()
private val backup = KVBackup(
plugin = plugin,
settingsManager = settingsManager,
inputFactory = inputFactory,
crypto = crypto,
nm = notificationManager
)
private val backup = KVBackup(plugin, settingsManager, inputFactory, crypto, dbManager)
private val db = mockk<KVDb>()
private val packageName = packageInfo.packageName
private val key = getRandomString(MAX_KEY_LENGTH_SIZE)
private val key64 = Base64.getEncoder().encodeToString(key.toByteArray(Utf8))
private val dataValue = Random.nextBytes(23)
private val dbBytes = Random.nextBytes(42)
private val inputStream = ByteArrayInputStream(dbBytes)
@Test
fun `has no initial state`() {
@ -59,82 +53,35 @@ internal class KVBackupTest : BackupTest() {
fun `simple backup with one record`() = runBlocking {
singleRecordBackup()
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0))
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt))
assertTrue(backup.hasState())
assertEquals(packageInfo, backup.getCurrentPackage())
assertEquals(TRANSPORT_OK, backup.finishBackup())
assertFalse(backup.hasState())
}
@Test
fun `@pm@ backup shows notification`() = runBlocking {
// init plugin and give back two keys
initPlugin(true, pmPackageInfo)
createBackupDataInput()
every { dataInput.readNextHeader() } returnsMany listOf(true, true, false)
every { dataInput.key } returnsMany listOf("key1", "key2")
// we don't care about values, so just use the same one always
every { dataInput.dataSize } returns dataValue.size
every { dataInput.readEntityData(any(), 0, dataValue.size) } returns dataValue.size
// store first record and show notification for it
every { notificationManager.onPmKvBackup("key1", 1, 2) } just Runs
coEvery { plugin.getOutputStreamForRecord(pmPackageInfo, "a2V5MQ") } returns outputStream
every { outputStream.write(ByteArray(1) { VERSION }) } just Runs
// store second record and show notification for it
every { notificationManager.onPmKvBackup("key2", 2, 2) } just Runs
coEvery { plugin.getOutputStreamForRecord(pmPackageInfo, "a2V5Mg") } returns outputStream
// encrypt to and close output stream
every { crypto.newEncryptingStream(outputStream, any()) } returns encryptedOutputStream
every { encryptedOutputStream.write(any<ByteArray>()) } just Runs
every { encryptedOutputStream.flush() } just Runs
every { encryptedOutputStream.close() } just Runs
every { outputStream.flush() } just Runs
every { outputStream.close() } just Runs
assertEquals(TRANSPORT_OK, backup.performBackup(pmPackageInfo, data, 0))
assertTrue(backup.hasState())
every { plugin.packageFinished(pmPackageInfo) } just Runs
assertEquals(TRANSPORT_OK, backup.finishBackup())
assertFalse(backup.hasState())
// verify that notifications were shown
verifyOrder {
notificationManager.onPmKvBackup("key1", 1, 2)
notificationManager.onPmKvBackup("key2", 2, 2)
}
}
@Test
fun `incremental backup with no data gets rejected`() = runBlocking {
coEvery { plugin.hasDataForPackage(packageInfo) } returns false
every { plugin.packageFinished(packageInfo) } just Runs
initPlugin(false)
every { db.close() } just Runs
assertEquals(
TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED,
backup.performBackup(packageInfo, data, FLAG_INCREMENTAL)
backup.performBackup(packageInfo, data, FLAG_INCREMENTAL, token, salt)
)
assertFalse(backup.hasState())
}
@Test
fun `check for existing data throws exception`() = runBlocking {
coEvery { plugin.hasDataForPackage(packageInfo) } throws IOException()
every { plugin.packageFinished(packageInfo) } just Runs
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0))
assertFalse(backup.hasState())
}
@Test
fun `non-incremental backup with data clears old data first`() = runBlocking {
singleRecordBackup(true)
coEvery { plugin.removeDataOfPackage(packageInfo) } just Runs
coEvery { plugin.removeData(token, name) } just Runs
every { dbManager.deleteDb(packageName) } returns true
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL))
assertEquals(
TRANSPORT_OK,
backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL, token, salt)
)
assertTrue(backup.hasState())
assertEquals(TRANSPORT_OK, backup.finishBackup())
assertFalse(backup.hasState())
@ -144,11 +91,11 @@ internal class KVBackupTest : BackupTest() {
fun `ignoring exception when clearing data when non-incremental backup has data`() =
runBlocking {
singleRecordBackup(true)
coEvery { plugin.removeDataOfPackage(packageInfo) } throws IOException()
coEvery { plugin.removeData(token, name) } throws IOException()
assertEquals(
TRANSPORT_OK,
backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL)
backup.performBackup(packageInfo, data, FLAG_NON_INCREMENTAL, token, salt)
)
assertTrue(backup.hasState())
assertEquals(TRANSPORT_OK, backup.finishBackup())
@ -157,15 +104,18 @@ internal class KVBackupTest : BackupTest() {
@Test
fun `package with no new data comes back ok right away`() = runBlocking {
every { crypto.getNameForPackage(salt, packageName) } returns name
every { dbManager.getDb(packageName) } returns db
every { data.close() } just Runs
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, FLAG_DATA_NOT_CHANGED))
assertEquals(
TRANSPORT_OK,
backup.performBackup(packageInfo, data, FLAG_DATA_NOT_CHANGED, token, salt)
)
assertTrue(backup.hasState())
verify { data.close() }
every { plugin.packageFinished(packageInfo) } just Runs
assertEquals(TRANSPORT_OK, backup.finishBackup())
assertFalse(backup.hasState())
}
@ -175,9 +125,9 @@ internal class KVBackupTest : BackupTest() {
initPlugin(false)
createBackupDataInput()
every { dataInput.readNextHeader() } throws IOException()
every { plugin.packageFinished(packageInfo) } just Runs
every { db.close() } just Runs
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0))
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0, token, salt))
assertFalse(backup.hasState())
}
@ -189,9 +139,9 @@ internal class KVBackupTest : BackupTest() {
every { dataInput.key } returns key
every { dataInput.dataSize } returns dataValue.size
every { dataInput.readEntityData(any(), 0, dataValue.size) } throws IOException()
every { plugin.packageFinished(packageInfo) } just Runs
every { db.close() } just Runs
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0))
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0, token, salt))
assertFalse(backup.hasState())
}
@ -199,24 +149,46 @@ internal class KVBackupTest : BackupTest() {
fun `no data records`() = runBlocking {
initPlugin(false)
getDataInput(listOf(false))
every { plugin.packageFinished(packageInfo) } just Runs
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0))
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt))
assertTrue(backup.hasState())
assertEquals(TRANSPORT_OK, backup.finishBackup())
assertFalse(backup.hasState())
}
@Test
fun `null data deletes key`() = runBlocking {
initPlugin(true)
createBackupDataInput()
every { dataInput.readNextHeader() } returns true andThen false
every { dataInput.key } returns key
every { dataInput.dataSize } returns -1 // just documented by example code in LocalTransport
every { db.delete(key) } just Runs
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt))
assertTrue(backup.hasState())
uploadData()
assertEquals(TRANSPORT_OK, backup.finishBackup())
assertFalse(backup.hasState())
}
@Test
fun `exception while writing version`() = runBlocking {
initPlugin(false)
getDataInput(listOf(true))
coEvery { plugin.getOutputStreamForRecord(packageInfo, key64) } returns outputStream
getDataInput(listOf(true, false))
every { db.put(key, dataValue) } just Runs
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt))
assertTrue(backup.hasState())
every { db.vacuum() } just Runs
every { db.close() } just Runs
coEvery { plugin.getOutputStream(token, name) } returns outputStream
every { outputStream.write(ByteArray(1) { VERSION }) } throws IOException()
every { outputStream.close() } just Runs
every { plugin.packageFinished(packageInfo) } just Runs
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0))
assertEquals(TRANSPORT_ERROR, backup.finishBackup())
assertFalse(backup.hasState())
verify { outputStream.close() }
@ -224,65 +196,41 @@ internal class KVBackupTest : BackupTest() {
@Test
fun `exception while writing encrypted value to output stream`() = runBlocking {
initPlugin(false)
getDataInput(listOf(true))
writeVersionAndEncrypt()
every { encryptedOutputStream.write(dataValue) } throws IOException()
every { plugin.packageFinished(packageInfo) } just Runs
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0))
assertFalse(backup.hasState())
verify { outputStream.close() }
}
@Test
fun `exception while flushing output stream`() = runBlocking {
initPlugin(false)
getDataInput(listOf(true))
writeVersionAndEncrypt()
every { encryptedOutputStream.write(dataValue) } just Runs
every { encryptedOutputStream.flush() } throws IOException()
every { encryptedOutputStream.close() } just Runs
every { outputStream.close() } just Runs
every { plugin.packageFinished(packageInfo) } just Runs
assertEquals(TRANSPORT_ERROR, backup.performBackup(packageInfo, data, 0))
assertFalse(backup.hasState())
verify { outputStream.close() }
}
@Test
fun `ignoring exception while closing output stream`() = runBlocking {
initPlugin(false)
getDataInput(listOf(true, false))
writeVersionAndEncrypt()
every { encryptedOutputStream.write(dataValue) } just Runs
every { encryptedOutputStream.flush() } just Runs
every { encryptedOutputStream.close() } just Runs
every { outputStream.close() } just Runs
every { plugin.packageFinished(packageInfo) } just Runs
every { db.put(key, dataValue) } just Runs
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0))
assertEquals(TRANSPORT_OK, backup.performBackup(packageInfo, data, 0, token, salt))
assertTrue(backup.hasState())
assertEquals(TRANSPORT_OK, backup.finishBackup())
every { db.vacuum() } just Runs
every { db.close() } just Runs
coEvery { plugin.getOutputStream(token, name) } returns outputStream
every { outputStream.write(ByteArray(1) { VERSION }) } just Runs
val ad = getADForKV(VERSION, packageInfo.packageName)
every { crypto.newEncryptingStream(outputStream, ad) } returns encryptedOutputStream
every { encryptedOutputStream.write(any<ByteArray>()) } throws IOException()
assertEquals(TRANSPORT_ERROR, backup.finishBackup())
assertFalse(backup.hasState())
verify {
encryptedOutputStream.close()
outputStream.close()
}
}
private fun singleRecordBackup(hasDataForPackage: Boolean = false) {
initPlugin(hasDataForPackage)
every { db.put(key, dataValue) } just Runs
getDataInput(listOf(true, false))
writeVersionAndEncrypt()
every { encryptedOutputStream.write(dataValue) } just Runs
every { encryptedOutputStream.flush() } just Runs
every { encryptedOutputStream.close() } just Runs
every { outputStream.close() } just Runs
every { plugin.packageFinished(packageInfo) } just Runs
uploadData()
}
private fun initPlugin(hasDataForPackage: Boolean = false, pi: PackageInfo = packageInfo) {
coEvery { plugin.hasDataForPackage(pi) } returns hasDataForPackage
every { dbManager.existsDb(pi.packageName) } returns hasDataForPackage
every { crypto.getNameForPackage(salt, pi.packageName) } returns name
every { dbManager.getDb(pi.packageName) } returns db
}
private fun createBackupDataInput() {
@ -301,11 +249,19 @@ internal class KVBackupTest : BackupTest() {
}
}
private fun writeVersionAndEncrypt() {
coEvery { plugin.getOutputStreamForRecord(packageInfo, key64) } returns outputStream
private fun uploadData() {
every { db.vacuum() } just Runs
every { db.close() } just Runs
coEvery { plugin.getOutputStream(token, name) } returns outputStream
every { outputStream.write(ByteArray(1) { VERSION }) } just Runs
val ad = getADForKV(VERSION, packageInfo.packageName)
every { crypto.newEncryptingStream(outputStream, ad) } returns encryptedOutputStream
every { encryptedOutputStream.write(any<ByteArray>()) } just Runs // gzip header
every { encryptedOutputStream.write(any(), any(), any()) } just Runs // stream copy
every { dbManager.getDbInputStream(packageName) } returns inputStream
every { encryptedOutputStream.close() } just Runs
every { outputStream.close() } just Runs
}
}

138
app/src/test/java/com/stevesoltys/seedvault/transport/backup/TestKvDbManager.kt Normal file
View file

@ -0,0 +1,138 @@
package com.stevesoltys.seedvault.transport.backup
import com.stevesoltys.seedvault.getRandomString
import com.stevesoltys.seedvault.toByteArrayFromHex
import com.stevesoltys.seedvault.toHexString
import junit.framework.Assert.assertEquals
import junit.framework.Assert.assertFalse
import junit.framework.Assert.assertNull
import junit.framework.Assert.assertTrue
import org.json.JSONObject
import org.junit.jupiter.api.Assertions.assertArrayEquals
import org.junit.jupiter.api.Test
import java.io.ByteArrayInputStream
import java.io.InputStream
import kotlin.random.Random
class TestKvDbManager : KvDbManager {
private var db: TestKVDb? = null
override fun getDb(packageName: String): KVDb {
return TestKVDb().apply { db = this }
}
override fun getDbInputStream(packageName: String): InputStream {
return ByteArrayInputStream(db!!.serialize().toByteArray())
}
override fun existsDb(packageName: String): Boolean {
return db != null
}
override fun deleteDb(packageName: String): Boolean {
clearDb()
return true
}
fun clearDb() {
this.db = null
}
fun readDbFromStream(inputStream: InputStream) {
this.db = TestKVDb.deserialize(String(inputStream.readBytes()))
}
}
class TestKVDb(private val json: JSONObject = JSONObject()) : KVDb {
override fun put(key: String, value: ByteArray) {
json.put(key, value.toHexString(spacer = ""))
}
override fun get(key: String): ByteArray? {
return json.getByteArray(key)
}
override fun getAll(): List<Pair<String, ByteArray>> {
val list = ArrayList<Pair<String, ByteArray>>(json.length())
json.keys().forEach { key ->
val bytes = json.getByteArray(key)
if (bytes != null) list.add(Pair(key, bytes))
}
return list
}
override fun delete(key: String) {
json.remove(key)
}
override fun vacuum() {
}
override fun close() {
}
fun serialize(): String {
return json.toString()
}
companion object {
fun deserialize(str: String): TestKVDb {
return TestKVDb(JSONObject(str))
}
}
private fun JSONObject.getByteArray(key: String): ByteArray? {
val str = optString(key, "")
if (str.isNullOrEmpty()) return null
return str.toByteArrayFromHex()
}
}
class TestKvDbManagerTest {
private val dbManager = TestKvDbManager()
private val key1 = getRandomString(12)
private val key2 = getRandomString(12)
private val bytes1 = Random.nextBytes(23)
private val bytes2 = Random.nextBytes(23)
@Test
fun test() {
assertFalse(dbManager.existsDb("foo"))
val db = dbManager.getDb("foo")
db.put(key1, bytes1)
db.put(key2, bytes2)
assertTrue(dbManager.existsDb("foo"))
assertArrayEquals(bytes1, db.get(key1))
assertArrayEquals(bytes2, db.get(key2))
val list = db.getAll()
assertEquals(2, list.size)
assertEquals(key1, list[0].first)
assertArrayEquals(bytes1, list[0].second)
assertEquals(key2, list[1].first)
assertArrayEquals(bytes2, list[1].second)
val dbBytes = dbManager.getDbInputStream("foo").readBytes()
assertTrue(dbManager.existsDb("foo"))
dbManager.clearDb()
assertFalse(dbManager.existsDb("foo"))
dbManager.readDbFromStream(ByteArrayInputStream(dbBytes))
assertTrue(dbManager.existsDb("foo"))
assertArrayEquals(bytes1, db.get(key1))
assertArrayEquals(bytes2, db.get(key2))
assertNull(db.get("bar"))
db.delete(key2)
assertNull(db.get(key2))
}
}