From 952cdec55db99f8f5650f314d0ec36f908af9587 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 12 Sep 2024 15:59:35 -0300 Subject: [PATCH] Fully implement BlobCache This class is responsible for caching blobs during a backup run, so we can know that a blob for the given chunk ID already exists and does not need to be uploaded again. It builds up its cache from snapshots available on the backend and from the persistent cache that includes blobs that could not be added to a snapshot, because the backup was aborted. --- .../java/com/stevesoltys/seedvault/App.kt | 2 +- .../seedvault/backend/BackendManager.kt | 3 + .../seedvault/transport/SnapshotManager.kt | 35 ++-- .../transport/backup/AppBackupManager.kt | 48 ++++-- .../transport/backup/BackupModule.kt | 2 +- .../transport/backup/BackupReceiver.kt | 6 +- .../seedvault/transport/backup/BlobCache.kt | 163 ++++++++++++++++++ .../seedvault/transport/backup/BlobsCache.kt | 69 -------- .../seedvault/worker/AppBackupWorker.kt | 3 +- .../seedvault/worker/WorkerModule.kt | 2 +- app/src/main/res/values/strings.xml | 1 + .../restore/install/ApkBackupRestoreTest.kt | 4 +- .../transport/SnapshotManagerTest.kt | 20 +-- .../seedvault/transport/TransportTest.kt | 25 ++- .../transport/backup/BlobCacheTest.kt | 144 ++++++++++++++++ 15 files changed, 392 insertions(+), 135 deletions(-) create mode 100644 app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCache.kt delete mode 100644 app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt create mode 100644 app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCacheTest.kt diff --git a/app/src/main/java/com/stevesoltys/seedvault/App.kt b/app/src/main/java/com/stevesoltys/seedvault/App.kt index 8112fdeb..90e6b3ad 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/App.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/App.kt @@ -62,7 +62,7 @@ open class App : Application() { private val appModule = module { single { SettingsManager(this@App) } single { BackupNotificationManager(this@App) } - single { BackendManager(this@App, get(), get()) } + single { BackendManager(this@App, get(), get(), get()) } single { BackendFactory { // uses context of the device's main user to be able to access USB storage diff --git a/app/src/main/java/com/stevesoltys/seedvault/backend/BackendManager.kt b/app/src/main/java/com/stevesoltys/seedvault/backend/BackendManager.kt index 545339a5..89b2bb8f 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/backend/BackendManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/backend/BackendManager.kt @@ -12,6 +12,7 @@ import com.stevesoltys.seedvault.getStorageContext import com.stevesoltys.seedvault.permitDiskReads import com.stevesoltys.seedvault.settings.SettingsManager import com.stevesoltys.seedvault.settings.StoragePluginType +import com.stevesoltys.seedvault.transport.backup.BlobCache import org.calyxos.seedvault.core.backends.Backend import org.calyxos.seedvault.core.backends.BackendFactory import org.calyxos.seedvault.core.backends.BackendProperties @@ -20,6 +21,7 @@ import org.calyxos.seedvault.core.backends.saf.SafBackend class BackendManager( private val context: Context, private val settingsManager: SettingsManager, + private val blobCache: BlobCache, backendFactory: BackendFactory, ) { @@ -86,6 +88,7 @@ class BackendManager( settingsManager.setStorageBackend(backend) mBackend = backend mBackendProperties = storageProperties + blobCache.clearLocalCache() } /** diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt index a6480dd3..e413667d 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt @@ -16,7 +16,6 @@ import okio.Buffer import okio.buffer import okio.sink import org.calyxos.seedvault.core.backends.AppBackupFileType -import org.calyxos.seedvault.core.backends.TopLevelFolder internal class SnapshotManager( private val crypto: Crypto, @@ -27,39 +26,25 @@ internal class SnapshotManager( private val log = KotlinLogging.logger {} /** - * The latest [Snapshot]. May be stale if [loadSnapshots] has not returned + * The latest [Snapshot]. May be stale if [onSnapshotsLoaded] has not returned * or wasn't called since new snapshots have been created. */ var latestSnapshot: Snapshot? = null private set - suspend fun loadSnapshots(callback: (Snapshot) -> Unit) { - log.info { "Loading snapshots..." } - val handles = mutableListOf() - backendManager.backend.list( - topLevelFolder = TopLevelFolder(crypto.repoId), - AppBackupFileType.Snapshot::class, - ) { fileInfo -> - fileInfo.fileHandle as AppBackupFileType.Snapshot - handles.add(fileInfo.fileHandle as AppBackupFileType.Snapshot) - } - handles.forEach { fileHandle -> + suspend fun onSnapshotsLoaded(handles: List): List { + return handles.map { snapshotHandle -> + // TODO set up local snapshot cache, so we don't need to download those all the time // TODO is it a fatal error when one snapshot is corrupted or couldn't get loaded? - val snapshot = onSnapshotFound(fileHandle) - callback(snapshot) + val snapshot = loader.loadFile(snapshotHandle).use { inputStream -> + Snapshot.parseFrom(inputStream) + } + // update latest snapshot if this one is more recent + if (snapshot.token > (latestSnapshot?.token ?: 0)) latestSnapshot = snapshot + snapshot } } - private suspend fun onSnapshotFound(snapshotHandle: AppBackupFileType.Snapshot): Snapshot { - // TODO set up local snapshot cache, so we don't need to download those all the time - val snapshot = loader.loadFile(snapshotHandle).use { inputStream -> - Snapshot.parseFrom(inputStream) - } - // update latest snapshot if this one is more recent - if (snapshot.token > (latestSnapshot?.token ?: 0)) latestSnapshot = snapshot - return snapshot - } - suspend fun saveSnapshot(snapshot: Snapshot) { val buffer = Buffer() val bufferStream = buffer.outputStream() diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt index 4330d387..bc47d6fa 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt @@ -5,13 +5,21 @@ package com.stevesoltys.seedvault.transport.backup +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto import com.stevesoltys.seedvault.settings.SettingsManager import com.stevesoltys.seedvault.transport.SnapshotManager import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.delay +import org.calyxos.seedvault.core.backends.AppBackupFileType.Blob +import org.calyxos.seedvault.core.backends.AppBackupFileType.Snapshot +import org.calyxos.seedvault.core.backends.FileInfo +import org.calyxos.seedvault.core.backends.TopLevelFolder internal class AppBackupManager( - private val blobsCache: BlobsCache, + private val crypto: Crypto, + private val blobCache: BlobCache, + private val backendManager: BackendManager, private val settingsManager: SettingsManager, private val snapshotManager: SnapshotManager, private val snapshotCreatorFactory: SnapshotCreatorFactory, @@ -22,22 +30,42 @@ internal class AppBackupManager( private set suspend fun beforeBackup() { - log.info { "Before backup" } + log.info { "Loading existing snapshots and blobs..." } + val blobInfos = mutableListOf() + val snapshotHandles = mutableListOf() + backendManager.backend.list( + topLevelFolder = TopLevelFolder(crypto.repoId), + Blob::class, Snapshot::class, + ) { fileInfo -> + when (fileInfo.fileHandle) { + is Blob -> blobInfos.add(fileInfo) + is Snapshot -> snapshotHandles.add(fileInfo.fileHandle as Snapshot) + else -> error("Unexpected FileHandle: $fileInfo") + } + } snapshotCreator = snapshotCreatorFactory.createSnapshotCreator() - blobsCache.populateCache() + val snapshots = snapshotManager.onSnapshotsLoaded(snapshotHandles) + blobCache.populateCache(blobInfos, snapshots) } suspend fun afterBackupFinished(success: Boolean) { log.info { "After backup finished. Success: $success" } - blobsCache.clear() - if (success) { - val snapshot = snapshotCreator?.finalizeSnapshot() ?: error("Had no snapshotCreator") - keepTrying { - snapshotManager.saveSnapshot(snapshot) + // free up memory by clearing blobs cache + blobCache.clear() + try { + if (success) { + val snapshot = + snapshotCreator?.finalizeSnapshot() ?: error("Had no snapshotCreator") + keepTrying { + snapshotManager.saveSnapshot(snapshot) + } + settingsManager.token = snapshot.token + // after snapshot was written, we can clear local cache as its info is in snapshot + blobCache.clearLocalCache() } - settingsManager.token = snapshot.token + } finally { + snapshotCreator = null } - snapshotCreator = null } private suspend fun keepTrying(n: Int = 3, block: suspend () -> Unit) { diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt index fcd312f7..4bb15123 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt @@ -12,7 +12,7 @@ import org.koin.dsl.module val backupModule = module { single { BackupInitializer(get()) } single { BackupReceiver(get(), get(), get()) } - single { BlobsCache(get(), get(), get()) } + single { BlobCache(androidContext()) } single { BlobCreator(get(), get()) } single { SnapshotManager(get(), get(), get()) } single { SnapshotCreatorFactory(androidContext(), get(), get(), get()) } diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt index cbe7d68c..1a1f4596 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt @@ -21,7 +21,7 @@ data class BackupData( } internal class BackupReceiver( - private val blobsCache: BlobsCache, + private val blobCache: BlobCache, private val blobCreator: BlobCreator, private val crypto: Crypto, private val replaceableChunker: Chunker? = null, @@ -89,11 +89,11 @@ internal class BackupReceiver( private suspend fun onNewChunk(chunk: Chunk) { chunks.add(chunk.hash) - val existingBlob = blobsCache.getBlob(chunk.hash) + val existingBlob = blobCache[chunk.hash] if (existingBlob == null) { val blob = blobCreator.createNewBlob(chunk) chunkMap[chunk.hash] = blob - blobsCache.saveNewBlob(chunk.hash, blob) + blobCache.saveNewBlob(chunk.hash, blob) } else { chunkMap[chunk.hash] = existingBlob } diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCache.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCache.kt new file mode 100644 index 00000000..5ee83a9c --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCache.kt @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import android.content.Context +import android.content.Context.MODE_APPEND +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import io.github.oshai.kotlinlogging.KotlinLogging +import org.calyxos.seedvault.core.backends.FileInfo +import org.calyxos.seedvault.core.toByteArrayFromHex +import org.calyxos.seedvault.core.toHexString +import java.io.FileNotFoundException +import java.io.IOException + +private const val CACHE_FILE_NAME = "blobsCache" + +/** + * Responsible for caching blobs during a backup run, + * so we can know that a blob for the given chunk ID already exists + * and does not need to be uploaded again. + * + * It builds up its cache from snapshots available on the backend + * and from the persistent cache that includes blobs that could not be added to a snapshot, + * because the backup was aborted. + */ +class BlobCache( + private val context: Context, +) { + + private val log = KotlinLogging.logger {} + private val blobMap = mutableMapOf() + + /** + * This must be called before saving files to the backend to avoid uploading duplicate blobs. + */ + @Throws(IOException::class) + fun populateCache(blobs: List, snapshots: List) { + log.info { "Getting all blobs from backend..." } + blobMap.clear() + // create map of blobId to size of blob on backend + val blobIds = blobs.associate { + Pair(it.fileHandle.name, it.size.toInt()) + } + // load local blob cache and include only blobs on backend + loadPersistentBlobCache(blobIds) + // build up mapping from chunkId to blob from available snapshots + snapshots.forEach { snapshot -> + onSnapshotLoaded(snapshot, blobIds) + } + } + + /** + * Should only be called after [populateCache] has returned. + */ + operator fun get(chunkId: String): Blob? = blobMap[chunkId] + + /** + * Should get called for all new blobs as soon as they've been saved to the backend. + */ + fun saveNewBlob(chunkId: String, blob: Blob) { + val previous = blobMap.put(chunkId, blob) + if (previous == null) { + // persist this new blob locally in case backup gets interrupted + context.openFileOutput(CACHE_FILE_NAME, MODE_APPEND).use { outputStream -> + outputStream.write(chunkId.toByteArrayFromHex()) + blob.writeDelimitedTo(outputStream) + } + } + } + + /** + * Clears the cached blob mapping. + * Should be called after a backup run to free up memory. + */ + fun clear() { + log.info { "Clearing cache..." } + blobMap.clear() + } + + /** + * Clears the local cache. + * Should get called after + * * changing to a different backup to prevent usage of blobs that don't exist there + * * uploading a new snapshot to prevent the persistent cache from growing indefinitely + */ + fun clearLocalCache() { + log.info { "Clearing local cache..." } + context.deleteFile(CACHE_FILE_NAME) + } + + /** + * Loads persistent cache from disk and adds blobs to [blobMap] + * if available in [allowedBlobIds] with the right size. + */ + private fun loadPersistentBlobCache(allowedBlobIds: Map) { + try { + context.openFileInput(CACHE_FILE_NAME).use { inputStream -> + val chunkIdBytes = ByteArray(32) + while (true) { + val bytesRead = inputStream.read(chunkIdBytes) + if (bytesRead != 32) break + val chunkId = chunkIdBytes.toHexString() + // parse blob + val blob = Blob.parseDelimitedFrom(inputStream) + val blobId = blob.id.hexFromProto() + // include blob only if size is equal to size on backend + val sizeOnBackend = allowedBlobIds[blobId] + if (sizeOnBackend == blob.length) { + blobMap[chunkId] = blob + } else log.warn { + if (sizeOnBackend == null) { + "Cached blob $blobId is missing from backend." + } else { + "Cached blob $blobId had different size on backend: $sizeOnBackend" + } + } + } + } + } catch (e: Exception) { + if (e is FileNotFoundException) log.info { "No local blob cache found." } + else { + // If the local cache is corrupted, that's not the end of the world. + // We can still continue normally, + // but may be writing out duplicated blobs we can't re-use. + // Those will get deleted again when pruning. + // So swallow the exception. + log.error(e) { "Error loading blobs cache: " } + } + } + } + + /** + * Used for populating local [blobMap] cache. + * Adds mapping from chunkId to [Blob], if it exists on backend, i.e. part of [blobIds] + * and its size matches the one on backend, i.e. value of [blobIds]. + */ + private fun onSnapshotLoaded(snapshot: Snapshot, blobIds: Map) { + snapshot.blobsMap.forEach { (chunkId, blob) -> + // check if referenced blob still exists on backend + val blobId = blob.id.hexFromProto() + val sizeOnBackend = blobIds[blobId] + if (sizeOnBackend == blob.length) { + // only add blob to our mapping, if it still exists + blobMap.putIfAbsent(chunkId, blob)?.let { previous -> + if (previous.id != blob.id) log.warn { + "Chunk ID ${chunkId.substring(0..5)} had more than one blob." + } + } + } else log.warn { + if (sizeOnBackend == null) { + "Blob $blobId in snapshot ${snapshot.token} is missing." + } else { + "Blob $blobId has unexpected size: $sizeOnBackend" + } + } + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt deleted file mode 100644 index 9d7f715e..00000000 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2024 The Calyx Institute - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.stevesoltys.seedvault.transport.backup - -import com.stevesoltys.seedvault.backend.BackendManager -import com.stevesoltys.seedvault.crypto.Crypto -import com.stevesoltys.seedvault.proto.Snapshot.Blob -import com.stevesoltys.seedvault.transport.SnapshotManager -import io.github.oshai.kotlinlogging.KotlinLogging -import org.calyxos.seedvault.core.backends.AppBackupFileType -import org.calyxos.seedvault.core.backends.TopLevelFolder - -internal class BlobsCache( - private val crypto: Crypto, - private val backendManager: BackendManager, - private val snapshotManager: SnapshotManager, -) { - - private val log = KotlinLogging.logger {} - private val blobMap = mutableMapOf() - - /** - * This must be called before saving files to the backend to avoid uploading duplicate blobs. - */ - suspend fun populateCache() { - log.info { "Getting all blobs from backend..." } - blobMap.clear() - val blobs = mutableSetOf() - backendManager.backend.list( - topLevelFolder = TopLevelFolder(crypto.repoId), - AppBackupFileType.Blob::class, - ) { fileInfo -> - fileInfo.fileHandle as AppBackupFileType.Blob - // TODO we could save size info here and later check it is as expected - blobs.add(fileInfo.fileHandle.name) - } - snapshotManager.loadSnapshots { snapshot -> - snapshot.blobsMap.forEach { (chunkId, blob) -> - // check if referenced blob still exists on backend - if (blobs.contains(blob.id.hexFromProto())) { - // only add blob to our mapping, if it still exists - blobMap.putIfAbsent(chunkId, blob)?.let { previous -> - if (previous.id != blob.id) log.warn { - "Chunk ID ${chunkId.substring(0..5)} had more than one blob" - } - } - } else log.warn { - "Blob ${blob.id.hexFromProto()} referenced in snapshot ${snapshot.token}" - } - } - } - } - - fun getBlob(hash: String): Blob? = blobMap[hash] - - fun saveNewBlob(chunkId: String, blob: Blob) { - blobMap[chunkId] = blob - // TODO persist this blob locally in case backup gets interrupted - } - - fun clear() { - log.info { "Clearing cache..." } - blobMap.clear() - } - -} diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt index e97ff922..72d7c693 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/AppBackupWorker.kt @@ -22,6 +22,7 @@ import androidx.work.OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST import androidx.work.PeriodicWorkRequestBuilder import androidx.work.WorkManager import androidx.work.WorkerParameters +import com.stevesoltys.seedvault.R import com.stevesoltys.seedvault.backend.BackendManager import com.stevesoltys.seedvault.settings.SettingsManager import com.stevesoltys.seedvault.transport.backup.AppBackupManager @@ -168,7 +169,7 @@ class AppBackupWorker( private fun createForegroundInfo() = ForegroundInfo( NOTIFICATION_ID_OBSERVER, - nm.getBackupNotification(""), + nm.getBackupNotification(applicationContext.getString(R.string.notification_init_text)), FOREGROUND_SERVICE_TYPE_DATA_SYNC, ) } diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt index 45f7e6dc..2a74cf3a 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt @@ -27,7 +27,7 @@ val workerModule = module { appBackupManager = get(), ) } - single { AppBackupManager(get(), get(), get(), get()) } + single { AppBackupManager(get(), get(), get(), get(), get(), get()) } single { ApkBackup( pm = androidContext().packageManager, diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a8230c15..44df4714 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -152,6 +152,7 @@ Backup notification Success notification Backup running + Preparing existing backup data for re-use… Backing up APK of %s Saving list of apps we can not back up. Backup already in progress diff --git a/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt b/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt index 83ee8864..84d4edcc 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/restore/install/ApkBackupRestoreTest.kt @@ -162,8 +162,8 @@ internal class ApkBackupRestoreTest : TransportTest() { val apkPath = slot() val cacheFiles = slot>() val repoId = getRandomString() - val apkHandle = AppBackupFileType.Blob(repoId, apkBlob.id.hexFromProto()) - val splitHandle = AppBackupFileType.Blob(repoId, splitBlob.id.hexFromProto()) + val apkHandle = AppBackupFileType.Blob(repoId, blob1.id.hexFromProto()) + val splitHandle = AppBackupFileType.Blob(repoId, blob2.id.hexFromProto()) every { backend.providerPackageName } returns storageProviderPackageName every { installRestriction.isAllowedToInstallApks() } returns true diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt index 61b8cee7..a4f4d54e 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt @@ -6,7 +6,6 @@ package com.stevesoltys.seedvault.transport import com.stevesoltys.seedvault.backend.BackendManager -import com.stevesoltys.seedvault.proto.Snapshot import com.stevesoltys.seedvault.transport.restore.Loader import io.mockk.coEvery import io.mockk.every @@ -15,8 +14,6 @@ import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.seedvault.core.backends.AppBackupFileType import org.calyxos.seedvault.core.backends.Backend -import org.calyxos.seedvault.core.backends.FileInfo -import org.calyxos.seedvault.core.backends.TopLevelFolder import org.calyxos.seedvault.core.toByteArrayFromHex import org.calyxos.seedvault.core.toHexString import org.junit.jupiter.api.Assertions.assertEquals @@ -63,18 +60,8 @@ internal class SnapshotManagerTest : TransportTest() { snapshotHandle.captured.hash, ) - val fileInfo = FileInfo(snapshotHandle.captured, Random.nextLong()) assertTrue(outputStream.size() > 0) val inputStream = ByteArrayInputStream(outputStream.toByteArray()) - coEvery { - backend.list( - topLevelFolder = TopLevelFolder(repoId), - AppBackupFileType.Snapshot::class, - callback = captureLambda<(FileInfo) -> Unit>() - ) - } answers { - lambda<(FileInfo) -> Unit>().captured.invoke(fileInfo) - } coEvery { backend.load(snapshotHandle.captured) } returns inputStream every { crypto.sha256(outputStream.toByteArray()) @@ -83,8 +70,9 @@ internal class SnapshotManagerTest : TransportTest() { passThroughInputStream.captured } - var loadedSnapshot: Snapshot? = null - snapshotManager.loadSnapshots { loadedSnapshot = it } - assertEquals(snapshot, loadedSnapshot) + snapshotManager.onSnapshotsLoaded(listOf(snapshotHandle.captured)).let { snapshots -> + assertEquals(1, snapshots.size) + assertEquals(snapshot, snapshots[0]) + } } } diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt index 34276218..72fe9e31 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt @@ -39,6 +39,7 @@ import io.mockk.mockk import io.mockk.mockkStatic import io.mockk.slot import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.FileInfo import org.calyxos.seedvault.core.backends.LegacyAppBackupFile import org.calyxos.seedvault.core.toHexString import org.junit.jupiter.api.TestInstance @@ -80,17 +81,29 @@ internal abstract class TransportTest { protected val splitBytes = byteArrayOf(0x07, 0x08, 0x09) protected val chunkId1 = Random.nextBytes(32).toHexString() protected val chunkId2 = Random.nextBytes(32).toHexString() - protected val apkBlob = blob { + protected val blob1 = blob { id = ByteString.copyFrom(Random.nextBytes(32)) + length = Random.nextInt(0, Int.MAX_VALUE) + uncompressedLength = Random.nextInt(0, Int.MAX_VALUE) } - protected val splitBlob = blob { + protected val blob2 = blob { id = ByteString.copyFrom(Random.nextBytes(32)) + length = Random.nextInt(0, Int.MAX_VALUE) + uncompressedLength = Random.nextInt(0, Int.MAX_VALUE) } - protected val blobHandle1 = AppBackupFileType.Blob(repoId, apkBlob.id.hexFromProto()) - protected val blobHandle2 = AppBackupFileType.Blob(repoId, splitBlob.id.hexFromProto()) - protected val apkBackupData = BackupData(listOf(chunkId1), mapOf(chunkId1 to apkBlob)) + protected val blobHandle1 = AppBackupFileType.Blob(repoId, blob1.id.hexFromProto()) + protected val blobHandle2 = AppBackupFileType.Blob(repoId, blob2.id.hexFromProto()) + protected val fileInfo1 = FileInfo( + fileHandle = blobHandle1, + size = blob1.length.toLong(), + ) + protected val fileInfo2 = FileInfo( + fileHandle = blobHandle2, + size = blob2.length.toLong(), + ) + protected val apkBackupData = BackupData(listOf(chunkId1), mapOf(chunkId1 to blob1)) protected val splitBackupData = - BackupData(listOf(chunkId2), mapOf(chunkId2 to splitBlob)) + BackupData(listOf(chunkId2), mapOf(chunkId2 to blob2)) protected val chunkMap = apkBackupData.chunkMap + splitBackupData.chunkMap protected val baseSplit = split { name = BASE_SPLIT diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCacheTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCacheTest.kt new file mode 100644 index 00000000..5e489316 --- /dev/null +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCacheTest.kt @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import android.content.Context +import com.stevesoltys.seedvault.transport.TransportTest +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Path + +internal class BlobCacheTest : TransportTest() { + + private val strictContext: Context = mockk() + + private val blobCache = BlobCache(context) + + @Test + fun `write to and read from cache`(@TempDir tmpDir: Path) { + val file = File(tmpDir.toString(), "tmpCache") + BlobCache(strictContext).saveTwoBlobsToCache(file) + + BlobCache(strictContext).let { cache -> + // old blobs are not yet in new cache + assertNull(cache[chunkId1]) + assertNull(cache[chunkId2]) + + // read saved blobs from cache + every { strictContext.openFileInput(any()) } returns file.inputStream() + cache.populateCache(listOf(fileInfo1, fileInfo2), emptyList()) + + // now both blobs are in the map + assertEquals(blob1, cache[chunkId1]) + assertEquals(blob2, cache[chunkId2]) + + // after clearing, blobs are gone + cache.clear() + assertNull(cache[chunkId1]) + assertNull(cache[chunkId2]) + } + } + + @Test + fun `cached blob gets only used if on backend`(@TempDir tmpDir: Path) { + val file = File(tmpDir.toString(), "tmpCache") + BlobCache(strictContext).saveTwoBlobsToCache(file) + + BlobCache(strictContext).let { cache -> + // read saved blobs from cache + every { strictContext.openFileInput(any()) } returns file.inputStream() + cache.populateCache(listOf(fileInfo2), emptyList()) // fileInfo1 is missing + + // now only blob2 gets used, because blob1 wasn't on backend + assertNull(cache[chunkId1]) + assertEquals(blob2, cache[chunkId2]) + } + } + + @Test + fun `cached blob gets only used if same size on backend`(@TempDir tmpDir: Path) { + val file = File(tmpDir.toString(), "tmpCache") + BlobCache(strictContext).saveTwoBlobsToCache(file) + + val info = fileInfo1.copy(size = fileInfo1.size - 1) + + BlobCache(strictContext).let { cache -> + // read saved blobs from cache + every { strictContext.openFileInput(any()) } returns file.inputStream() + cache.populateCache(listOf(info, fileInfo2), emptyList()) // info has different size now + + // now only blob2 gets used, because blob1 wasn't on backend + assertNull(cache[chunkId1]) + assertEquals(blob2, cache[chunkId2]) + } + } + + @Test + fun `blobs from snapshot get added to cache`() { + assertEquals(blob1, snapshot.blobsMap[chunkId1]) + assertEquals(blob2, snapshot.blobsMap[chunkId2]) + + // before populating cache, the blobs are not in + assertNull(blobCache[chunkId1]) + assertNull(blobCache[chunkId2]) + + blobCache.populateCache(listOf(fileInfo1, fileInfo2), listOf(snapshot)) + + // after populating cache, the blobs are in + assertEquals(blob1, blobCache[chunkId1]) + assertEquals(blob2, blobCache[chunkId2]) + + // clearing cache removes blobs + blobCache.clear() + assertNull(blobCache[chunkId1]) + assertNull(blobCache[chunkId2]) + } + + @Test + fun `blobs from snapshot get added to cache only if on backend`() { + blobCache.populateCache(listOf(fileInfo2), listOf(snapshot)) + + // after populating cache, only second blob is in + assertNull(blobCache[chunkId1]) + assertEquals(blob2, blobCache[chunkId2]) + } + + @Test + fun `blobs from snapshot get added to cache only if same size on backend`() { + val info = fileInfo1.copy(size = fileInfo1.size - 1) // same blob, different size + blobCache.populateCache(listOf(info, fileInfo2), listOf(snapshot)) + + // after populating cache, only second blob is in + assertNull(blobCache[chunkId1]) + assertEquals(blob2, blobCache[chunkId2]) + } + + @Test + fun `test clearing loading cache`() { + // clearing the local cache, deletes the cache file + every { strictContext.deleteFile(any()) } returns true + blobCache.clearLocalCache() + } + + private fun BlobCache.saveTwoBlobsToCache(file: File) { + every { strictContext.openFileOutput(any(), any()) } answers { + FileOutputStream(file, true) + } + + // save new blobs (using a new output stream for each as it gets closed) + saveNewBlob(chunkId1, blob1) + saveNewBlob(chunkId2, blob2) + + // clearing cache should affect persisted blobs + clear() + } +}