mobi/seedvault
1
0
Fork 0
Code Pull requests Activity

Store size of app backups in metadata

Browse source
This commit is contained in:
Torsten Grote 2024-01-15 11:48:31 -03:00
parent 0319d733c1
commit c362da8220
No known key found for this signature in database
GPG key ID: 3E5F77D92CF891FF
13 changed files with 106 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt
View file

@ -74,6 +74,7 @@ data class PackageMetadata(
internal var time: Long = 0L,
internal var state: PackageState = UNKNOWN_ERROR,
internal var backupType: BackupType? = null,
internal var size: Long? = null,
internal val system: Boolean = false,
internal val version: Long? = null,
internal val installer: String? = null,
@ -97,6 +98,7 @@ enum class BackupType { KV, FULL }
internal const val JSON_PACKAGE_TIME = "time"
internal const val JSON_PACKAGE_BACKUP_TYPE = "backupType"
internal const val JSON_PACKAGE_STATE = "state"
internal const val JSON_PACKAGE_SIZE = "size"
internal const val JSON_PACKAGE_SYSTEM = "system"
internal const val JSON_PACKAGE_VERSION = "version"
internal const val JSON_PACKAGE_INSTALLER = "installer"

6
app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataManager.kt
View file

@ -131,6 +131,7 @@ internal class MetadataManager(
fun onPackageBackedUp(
packageInfo: PackageInfo,
type: BackupType,
size: Long?,
metadataOutputStream: OutputStream,
) {
val packageName = packageInfo.packageName
@ -143,12 +144,15 @@ internal class MetadataManager(
metadata.packageMetadataMap[packageName]!!.time = now
metadata.packageMetadataMap[packageName]!!.state = APK_AND_DATA
metadata.packageMetadataMap[packageName]!!.backupType = type
// don't override a previous K/V size, if there were no K/V changes
if (size != null) metadata.packageMetadataMap[packageName]!!.size = size
} else {
metadata.packageMetadataMap[packageName] = PackageMetadata(
time = now,
state = APK_AND_DATA,
backupType = type,
system = packageInfo.isSystemApp()
size = size,
system = packageInfo.isSystemApp(),
)
}
}

2
app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataReader.kt
View file

@ -120,6 +120,7 @@ internal class MetadataReaderImpl(private val crypto: Crypto) : MetadataReader {
// because when only backing up the APK for example, there's no type
else -> null
}
val pSize = p.optLong(JSON_PACKAGE_SIZE, -1L)
val pSystem = p.optBoolean(JSON_PACKAGE_SYSTEM, false)
val pVersion = p.optLong(JSON_PACKAGE_VERSION, 0L)
val pInstaller = p.optString(JSON_PACKAGE_INSTALLER)
@ -136,6 +137,7 @@ internal class MetadataReaderImpl(private val crypto: Crypto) : MetadataReader {
time = p.getLong(JSON_PACKAGE_TIME),
state = pState,
backupType = pBackupType,
size = if (pSize < 0L) null else pSize,
system = pSystem,
version = if (pVersion == 0L) null else pVersion,
installer = if (pInstaller == "") null else pInstaller,

3
app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataWriter.kt
View file

@ -49,6 +49,9 @@ internal class MetadataWriterImpl(private val crypto: Crypto) : MetadataWriter {
if (packageMetadata.backupType != null) {
put(JSON_PACKAGE_BACKUP_TYPE, packageMetadata.backupType!!.name)
}
if (packageMetadata.size != null) {
put(JSON_PACKAGE_SIZE, packageMetadata.size)
}
if (packageMetadata.system) {
put(JSON_PACKAGE_SYSTEM, packageMetadata.system)
}

10
app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt
View file

@ -363,6 +363,7 @@ internal class BackupCoordinator(
// getCurrentPackage() not-null because we have state, call before finishing
val packageInfo = kv.getCurrentPackage()!!
val packageName = packageInfo.packageName
val size = kv.getCurrentSize()
// tell K/V backup to finish
var result = kv.finishBackup()
if (result == TRANSPORT_OK) {
@ -370,7 +371,7 @@ internal class BackupCoordinator(
// call onPackageBackedUp for @pm@ only if we can do backups right now
if (!isPmBackup || settingsManager.canDoBackupNow()) {
try {
onPackageBackedUp(packageInfo, BackupType.KV)
onPackageBackedUp(packageInfo, BackupType.KV, size)
} catch (e: Exception) {
Log.e(TAG, "Error calling onPackageBackedUp for $packageName", e)
result = TRANSPORT_PACKAGE_REJECTED
@ -396,10 +397,11 @@ internal class BackupCoordinator(
// getCurrentPackage() not-null because we have state
val packageInfo = full.getCurrentPackage()!!
val packageName = packageInfo.packageName
val size = full.getCurrentSize()
// tell full backup to finish
var result = full.finishBackup()
try {
onPackageBackedUp(packageInfo, BackupType.FULL)
onPackageBackedUp(packageInfo, BackupType.FULL, size)
} catch (e: Exception) {
Log.e(TAG, "Error calling onPackageBackedUp for $packageName", e)
result = TRANSPORT_PACKAGE_REJECTED
@ -470,9 +472,9 @@ internal class BackupCoordinator(
}
}
private suspend fun onPackageBackedUp(packageInfo: PackageInfo, type: BackupType) {
private suspend fun onPackageBackedUp(packageInfo: PackageInfo, type: BackupType, size: Long?) {
plugin.getMetadataOutputStream().use {
metadataManager.onPackageBackedUp(packageInfo, type, it)
metadataManager.onPackageBackedUp(packageInfo, type, size, it)
}
}

4
app/src/main/java/com/stevesoltys/seedvault/transport/backup/FullBackup.kt
View file

@ -51,6 +51,8 @@ internal class FullBackup(
fun getCurrentPackage() = state?.packageInfo
fun getCurrentSize() = state?.size
fun getQuota(): Long {
return if (settingsManager.isQuotaUnlimited()) Long.MAX_VALUE else DEFAULT_QUOTA_FULL_BACKUP
}
@ -190,7 +192,7 @@ internal class FullBackup(
}
fun finishBackup(): Int {
Log.i(TAG, "Finish full backup of ${state!!.packageName}.")
Log.i(TAG, "Finish full backup of ${state!!.packageName}. Wrote ${state!!.size} bytes")
return clearState()
}

6
app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVBackup.kt
View file

@ -46,6 +46,10 @@ internal class KVBackup(
fun getCurrentPackage() = state?.packageInfo
fun getCurrentSize() = getCurrentPackage()?.let {
dbManager.getDbSize(it.packageName)
}
fun getQuota(): Long = if (settingsManager.isQuotaUnlimited()) {
Long.MAX_VALUE
} else {
@ -252,7 +256,7 @@ internal class KVBackup(
}
}
}
Log.d(TAG, "Uploaded db file for $packageName")
Log.d(TAG, "Uploaded db file for $packageName.")
}
private class KVOperation(

10
app/src/main/java/com/stevesoltys/seedvault/transport/backup/KVDbManager.kt
View file

@ -29,6 +29,11 @@ interface KvDbManager {
* Use only for backup.
*/
fun existsDb(packageName: String): Boolean
/**
* Returns the current size of the DB in bytes or null, if no DB exists.
*/
fun getDbSize(packageName: String): Long?
fun deleteDb(packageName: String, isRestore: Boolean = false): Boolean
}
@ -59,6 +64,11 @@ class KvDbManagerImpl(private val context: Context) : KvDbManager {
return getDbFile(packageName).isFile
}
override fun getDbSize(packageName: String): Long? {
val file = getDbFile(packageName)
return if (file.isFile) file.length() else null
}
override fun deleteDb(packageName: String, isRestore: Boolean): Boolean {
return getDbFile(packageName, isRestore).delete()
}

20
app/src/test/java/com/stevesoltys/seedvault/metadata/MetadataManagerTest.kt
View file

@ -249,6 +249,7 @@ class MetadataManagerTest {
time = time,
packageMetadataMap = PackageMetadataMap() // otherwise this isn't copied, but referenced
)
val size = Random.nextLong()
val packageMetadata = PackageMetadata(time)
updatedMetadata.packageMetadataMap[packageName] = packageMetadata
@ -256,10 +257,15 @@ class MetadataManagerTest {
every { clock.time() } returns time
expectModifyMetadata(initialMetadata)
manager.onPackageBackedUp(packageInfo, BackupType.FULL, storageOutputStream)
manager.onPackageBackedUp(packageInfo, BackupType.FULL, size, storageOutputStream)
assertEquals(
packageMetadata.copy(state = APK_AND_DATA, backupType = BackupType.FULL, system = true),
packageMetadata.copy(
state = APK_AND_DATA,
backupType = BackupType.FULL,
size = size,
system = true,
),
manager.getPackageMetadata(packageName)
)
assertEquals(time, manager.getLastBackupTime())
@ -270,6 +276,7 @@ class MetadataManagerTest {
cacheOutputStream.close()
}
}
@Test
fun `test onPackageBackedUp() with D2D enabled`() {
expectReadFromCache()
@ -278,7 +285,7 @@ class MetadataManagerTest {
every { settingsManager.d2dBackupsEnabled() } returns true
manager.onPackageBackedUp(packageInfo, BackupType.FULL, storageOutputStream)
manager.onPackageBackedUp(packageInfo, BackupType.FULL, 0L, storageOutputStream)
assertTrue(initialMetadata.d2dBackup)
verify {
@ -290,19 +297,20 @@ class MetadataManagerTest {
@Test
fun `test onPackageBackedUp() fails to write to storage`() {
val updateTime = time + 1
val size = Random.nextLong()
val updatedMetadata = initialMetadata.copy(
time = updateTime,
packageMetadataMap = PackageMetadataMap() // otherwise this isn't copied, but referenced
)
updatedMetadata.packageMetadataMap[packageName] =
PackageMetadata(updateTime, APK_AND_DATA, BackupType.KV)
PackageMetadata(updateTime, APK_AND_DATA, BackupType.KV, size)
expectReadFromCache()
every { clock.time() } returns updateTime
every { metadataWriter.write(updatedMetadata, storageOutputStream) } throws IOException()
try {
manager.onPackageBackedUp(packageInfo, BackupType.KV, storageOutputStream)
manager.onPackageBackedUp(packageInfo, BackupType.KV, size, storageOutputStream)
fail()
} catch (e: IOException) {
// expected
@ -335,7 +343,7 @@ class MetadataManagerTest {
every { clock.time() } returns time
expectModifyMetadata(updatedMetadata)
manager.onPackageBackedUp(packageInfo, BackupType.FULL, storageOutputStream)
manager.onPackageBackedUp(packageInfo, BackupType.FULL, 0L, storageOutputStream)
assertEquals(time, manager.getLastBackupTime())
assertEquals(PackageMetadata(time), manager.getPackageMetadata(cachedPackageName))

10
app/src/test/java/com/stevesoltys/seedvault/metadata/MetadataWriterDecoderTest.kt
View file

@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS
import kotlin.random.Random
import kotlin.random.nextLong
@TestInstance(PER_CLASS)
internal class MetadataWriterDecoderTest {
@ -81,11 +82,12 @@ internal class MetadataWriterDecoderTest {
time = Random.nextLong(),
state = QUOTA_EXCEEDED,
backupType = BackupType.FULL,
size = Random.nextLong(0..Long.MAX_VALUE),
system = Random.nextBoolean(),
version = Random.nextLong(),
installer = getRandomString(),
sha256 = getRandomString(),
signatures = listOf(getRandomString())
signatures = listOf(getRandomString()),
)
)
put(
@ -93,22 +95,24 @@ internal class MetadataWriterDecoderTest {
time = Random.nextLong(),
state = NO_DATA,
backupType = BackupType.KV,
size = null,
system = Random.nextBoolean(),
version = Random.nextLong(),
installer = getRandomString(),
sha256 = getRandomString(),
signatures = listOf(getRandomString(), getRandomString())
signatures = listOf(getRandomString(), getRandomString()),
)
)
put(
getRandomString(), PackageMetadata(
time = 0L,
state = NOT_ALLOWED,
size = 0,
system = Random.nextBoolean(),
version = Random.nextLong(),
installer = getRandomString(),
sha256 = getRandomString(),
signatures = listOf(getRandomString(), getRandomString())
signatures = listOf(getRandomString(), getRandomString()),
)
)
}

21
app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt
View file

@ -147,7 +147,12 @@ internal class CoordinatorIntegrationTest : TransportTest() {
metadataManager.onApkBackedUp(packageInfo, packageMetadata, metadataOutputStream)
} just Runs
every {
metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream)
metadataManager.onPackageBackedUp(
packageInfo = packageInfo,
type = BackupType.KV,
size = more((appData.size + appData2.size).toLong()), // more because DB overhead
metadataOutputStream = metadataOutputStream,
)
} just Runs
// start K/V backup
@ -216,7 +221,12 @@ internal class CoordinatorIntegrationTest : TransportTest() {
backupPlugin.getOutputStream(token, FILE_BACKUP_METADATA)
} returns metadataOutputStream
every {
metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream)
metadataManager.onPackageBackedUp(
packageInfo = packageInfo,
type = BackupType.KV,
size = more(size.toLong()), // more than $size, because DB overhead
metadataOutputStream = metadataOutputStream,
)
} just Runs
// start K/V backup
@ -289,7 +299,12 @@ internal class CoordinatorIntegrationTest : TransportTest() {
)
} just Runs
every {
metadataManager.onPackageBackedUp(packageInfo, BackupType.FULL, metadataOutputStream)
metadataManager.onPackageBackedUp(
packageInfo = packageInfo,
type = BackupType.FULL,
size = appData.size.toLong(),
metadataOutputStream = metadataOutputStream,
)
} just Runs
// perform backup to output stream

30
app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt
View file

@ -36,6 +36,7 @@ import org.junit.jupiter.api.Test
import java.io.IOException
import java.io.OutputStream
import kotlin.random.Random
import kotlin.random.nextLong
@Suppress("BlockingMethodInNonBlockingContext")
internal class BackupCoordinatorTest : BackupTest() {
@ -204,14 +205,22 @@ internal class BackupCoordinatorTest : BackupTest() {
@Test
fun `finish backup delegates to KV plugin if it has state`() = runBlocking {
val size = 0L
every { kv.hasState() } returns true
every { full.hasState() } returns false
every { kv.getCurrentPackage() } returns packageInfo
coEvery { kv.finishBackup() } returns TRANSPORT_OK
every { settingsManager.getToken() } returns token
coEvery { plugin.getOutputStream(token, FILE_BACKUP_METADATA) } returns metadataOutputStream
every { kv.getCurrentSize() } returns size
every {
metadataManager.onPackageBackedUp(packageInfo, BackupType.KV, metadataOutputStream)
metadataManager.onPackageBackedUp(
packageInfo = packageInfo,
type = BackupType.KV,
size = size,
metadataOutputStream = metadataOutputStream,
)
} just Runs
every { metadataOutputStream.close() } just Runs
@ -225,6 +234,7 @@ internal class BackupCoordinatorTest : BackupTest() {
every { kv.hasState() } returns true
every { full.hasState() } returns false
every { kv.getCurrentPackage() } returns pmPackageInfo
every { kv.getCurrentSize() } returns 42L
coEvery { kv.finishBackup() } returns TRANSPORT_OK
every { settingsManager.canDoBackupNow() } returns false
@ -235,6 +245,7 @@ internal class BackupCoordinatorTest : BackupTest() {
@Test
fun `finish backup delegates to full plugin if it has state`() = runBlocking {
val result = Random.nextInt()
val size: Long? = null
every { kv.hasState() } returns false
every { full.hasState() } returns true
@ -242,8 +253,14 @@ internal class BackupCoordinatorTest : BackupTest() {
every { full.finishBackup() } returns result
every { settingsManager.getToken() } returns token
coEvery { plugin.getOutputStream(token, FILE_BACKUP_METADATA) } returns metadataOutputStream
every { full.getCurrentSize() } returns size
every {
metadataManager.onPackageBackedUp(packageInfo, BackupType.FULL, metadataOutputStream)
metadataManager.onPackageBackedUp(
packageInfo = packageInfo,
type = BackupType.FULL,
size = size,
metadataOutputStream = metadataOutputStream,
)
} just Runs
every { metadataOutputStream.close() } just Runs
@ -375,6 +392,7 @@ internal class BackupCoordinatorTest : BackupTest() {
}
)
val packageMetadata: PackageMetadata = mockk()
val size = Random.nextLong(1L..Long.MAX_VALUE)
every { settingsManager.canDoBackupNow() } returns true
every { metadataManager.requiresInit } returns false
@ -394,8 +412,14 @@ internal class BackupCoordinatorTest : BackupTest() {
every { kv.hasState() } returns true
every { full.hasState() } returns false
every { kv.getCurrentPackage() } returns pmPackageInfo
every { kv.getCurrentSize() } returns size
every {
metadataManager.onPackageBackedUp(pmPackageInfo, BackupType.KV, metadataOutputStream)
metadataManager.onPackageBackedUp(
pmPackageInfo,
BackupType.KV,
size,
metadataOutputStream,
)
} just Runs
coEvery { kv.finishBackup() } returns TRANSPORT_OK

4
app/src/test/java/com/stevesoltys/seedvault/transport/backup/TestKvDbManager.kt
View file

@ -42,6 +42,10 @@ class TestKvDbManager : KvDbManager {
return db != null
}
override fun getDbSize(packageName: String): Long? {
return db?.serialize()?.toByteArray()?.size?.toLong()
}
override fun deleteDb(packageName: String, isRestore: Boolean): Boolean {
clearDb()
return true