From 1b08e30a4ab81ad283cad1a412f6c6fbb4cd2730 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Fri, 18 Jun 2021 17:55:27 -0300 Subject: [PATCH] Allow using the same storage location on different devices or user profiles Previously, we would put our files directly in the root of the storage location and delete any existing backups there. When used by different devices or user profiles, these would keep deleting each other's backups. --- .../seedvault/restore/RestoreViewModel.kt | 2 + .../storagebackuptester/MainViewModel.kt | 3 +- storage/doc/design.md | 1 - .../calyxos/backup/storage/api/Snapshot.kt | 17 +- .../backup/storage/api/StorageBackup.kt | 24 +-- .../backup/storage/api/StoragePlugin.kt | 24 ++- .../storage/backup/ChunksCacheRepopulater.kt | 10 +- .../storage/plugin/SnapshotRetriever.kt | 6 +- .../backup/storage/plugin/saf/SafCache.kt | 32 ++++ .../storage/plugin/saf/SafStoragePlugin.kt | 165 +++++++++++++----- .../calyxos/backup/storage/prune/Pruner.kt | 26 +-- .../backup/storage/prune/RetentionManager.kt | 31 ++-- .../storage/restore/AbstractChunkRestore.kt | 4 +- .../storage/restore/MultiChunkRestore.kt | 15 +- .../calyxos/backup/storage/restore/Restore.kt | 44 +++-- .../backup/storage/restore/RestoreService.kt | 18 +- .../storage/restore/SingleChunkRestore.kt | 4 +- .../backup/storage/restore/ZipChunkRestore.kt | 4 +- .../backup/storage/BackupRestoreTest.kt | 55 ++++-- .../backup/ChunksCacheRepopulaterTest.kt | 11 +- .../backup/storage/prune/PrunerTest.kt | 15 +- .../storage/prune/RetentionManagerTest.kt | 30 ++-- 22 files changed, 375 insertions(+), 166 deletions(-) create mode 100644 storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt index 6fb3dea8..2b4e398f 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt @@ -62,6 +62,7 @@ import kotlinx.coroutines.launch import org.calyxos.backup.storage.api.SnapshotItem import org.calyxos.backup.storage.api.StorageBackup import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START +import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID import org.calyxos.backup.storage.ui.restore.SnapshotViewModel import java.util.LinkedList import kotlin.coroutines.Continuation @@ -392,6 +393,7 @@ internal class RestoreViewModel( @UiThread internal fun startFilesRestore(item: SnapshotItem) { val i = Intent(app, StorageRestoreService::class.java) + i.putExtra(EXTRA_USER_ID, item.storedSnapshot.userId) i.putExtra(EXTRA_TIMESTAMP_START, item.time) app.startForegroundService(i) mDisplayFragment.setEvent(RESTORE_FILES_STARTED) diff --git a/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt b/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt index bd55b1e4..d68d76cf 100644 --- a/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt +++ b/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt @@ -123,6 +123,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati // example for how to do restore via foreground service // app.startForegroundService(Intent(app, DemoRestoreService::class.java).apply { +// putExtra(EXTRA_USER_ID, item.storedSnapshot.userId) // putExtra(EXTRA_TIMESTAMP_START, snapshot.timeStart) // }) @@ -130,7 +131,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati _restoreProgressVisible.value = true val restoreObserver = RestoreStats(app, _restoreLog) viewModelScope.launch { - storageBackup.restoreBackupSnapshot(snapshot, restoreObserver) + storageBackup.restoreBackupSnapshot(item.storedSnapshot, snapshot, restoreObserver) _restoreProgressVisible.value = false } } diff --git a/storage/doc/design.md b/storage/doc/design.md index 332bb571..f43c51f4 100644 --- a/storage/doc/design.md +++ b/storage/doc/design.md @@ -388,7 +388,6 @@ but are considered out-of-scope of the current design for time and budget reason * using a rolling hash to produce chunks in order to increase likelihood of obtaining same chunks even if file contents change slightly or shift * external secret-less corruption checks that would use checksums over encrypted data -* supporting different backup clients backing up to the same storage * concealing file sizes (though zip chunks helps a bit here) * implementing different storage plugins diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt index d8c11bd8..4e936664 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt @@ -3,15 +3,28 @@ package org.calyxos.backup.storage.api import org.calyxos.backup.storage.backup.BackupSnapshot public data class SnapshotItem( - public val time: Long, + public val storedSnapshot: StoredSnapshot, public val snapshot: BackupSnapshot?, -) +) { + val time: Long get() = storedSnapshot.timestamp +} public sealed class SnapshotResult { public data class Success(val snapshots: List) : SnapshotResult() public data class Error(val e: Exception) : SnapshotResult() } +public data class StoredSnapshot( + /** + * The unique ID of the current device/user combination chosen by the [StoragePlugin]. + */ + public val userId: String, + /** + * The timestamp identifying a snapshot of the [userId]. + */ + public val timestamp: Long, +) + /** * Defines which backup snapshots should be retained when pruning backups. * diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt index 969c9851..1bb556a6 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt @@ -103,10 +103,12 @@ public class StorageBackup( * Run this on a new storage location to ensure that there are no old snapshots * (potentially encrypted with an old key) laying around. * Using a storage location with existing data is not supported. + * Using the same root folder for storage on different devices or user profiles is fine though + * as the [StoragePlugin] should isolate storage per [StoredSnapshot.userId]. */ public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) { try { - plugin.getAvailableBackupSnapshots().forEach { + plugin.getCurrentBackupSnapshots().forEach { try { plugin.deleteBackupSnapshot(it) } catch (e: IOException) { @@ -183,15 +185,16 @@ public class StorageBackup( } public suspend fun restoreBackupSnapshot( - snapshot: BackupSnapshot, - restoreObserver: RestoreObserver? = null + storedSnapshot: StoredSnapshot, + snapshot: BackupSnapshot? = null, + restoreObserver: RestoreObserver? = null, ): Boolean = withContext(dispatcher) { if (restoreRunning.getAndSet(true)) { Log.w(TAG, "Restore already running, not starting a new one") return@withContext false } try { - restore.restoreBackupSnapshot(snapshot, restoreObserver) + restore.restoreBackupSnapshot(storedSnapshot, snapshot, restoreObserver) true } catch (e: Exception) { Log.e(TAG, "Error during restore", e) @@ -201,17 +204,4 @@ public class StorageBackup( } } - public suspend fun restoreBackupSnapshot( - timestamp: Long, - restoreObserver: RestoreObserver? = null - ): Boolean = withContext(dispatcher) { - try { - restore.restoreBackupSnapshot(timestamp, restoreObserver) - true - } catch (e: Exception) { - Log.e(TAG, "Error during restore", e) - false - } - } - } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt index 465d9c39..aa1be4d6 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt @@ -35,22 +35,36 @@ public interface StoragePlugin { /* Restore */ /** - * Returns the timestamps representing a backup snapshot that are available on storage. + * Returns *all* [StoredSnapshot]s that are available on storage + * independent of user ID and whether they can be decrypted + * with the key returned by [getMasterKey]. */ @Throws(IOException::class) - public suspend fun getAvailableBackupSnapshots(): List + public suspend fun getBackupSnapshotsForRestore(): List @Throws(IOException::class) - public suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream + public suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream @Throws(IOException::class) - public suspend fun getChunkInputStream(chunkId: String): InputStream + public suspend fun getChunkInputStream(snapshot: StoredSnapshot, chunkId: String): InputStream /* Pruning */ + /** + * Returns [StoredSnapshot]s for the currently active user ID. + */ @Throws(IOException::class) - public suspend fun deleteBackupSnapshot(timestamp: Long) + public suspend fun getCurrentBackupSnapshots(): List + /** + * Deletes the given [StoredSnapshot]. + */ + @Throws(IOException::class) + public suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) + + /** + * Deletes the given chunks of the *current* user ID only. + */ @Throws(IOException::class) public suspend fun deleteChunks(chunkIds: List) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt index cc9caf94..5e7afc06 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt @@ -7,6 +7,7 @@ import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.plugin.SnapshotRetriever import java.io.IOException +import java.security.GeneralSecurityException import java.util.concurrent.TimeUnit.MILLISECONDS import kotlin.time.ExperimentalTime import kotlin.time.toDuration @@ -37,8 +38,13 @@ internal class ChunksCacheRepopulater( availableChunkIds: HashSet ) { val start = System.currentTimeMillis() - val snapshots = storagePlugin.getAvailableBackupSnapshots().map { timestamp -> - snapshotRetriever.getSnapshot(streamKey, timestamp) + val snapshots = storagePlugin.getCurrentBackupSnapshots().mapNotNull { storedSnapshot -> + try { + snapshotRetriever.getSnapshot(streamKey, storedSnapshot) + } catch (e: GeneralSecurityException) { + Log.w(TAG, "Error fetching snapshot $storedSnapshot", e) + null + } } val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS) Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration") diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt index 050b3be9..38fdcc14 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt @@ -2,6 +2,7 @@ package org.calyxos.backup.storage.plugin import com.google.protobuf.InvalidProtocolBufferException import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.restore.readVersion @@ -19,9 +20,10 @@ internal class SnapshotRetriever( GeneralSecurityException::class, InvalidProtocolBufferException::class, ) - suspend fun getSnapshot(streamKey: ByteArray, timestamp: Long): BackupSnapshot { - return storagePlugin.getBackupSnapshotInputStream(timestamp).use { inputStream -> + suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot { + return storagePlugin.getBackupSnapshotInputStream(storedSnapshot).use { inputStream -> val version = inputStream.readVersion() + val timestamp = storedSnapshot.timestamp val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte()) streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream -> BackupSnapshot.parseFrom(decryptedStream) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt new file mode 100644 index 00000000..0480a1eb --- /dev/null +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt @@ -0,0 +1,32 @@ +package org.calyxos.backup.storage.plugin.saf + +import androidx.documentfile.provider.DocumentFile +import org.calyxos.backup.storage.api.StoredSnapshot + +/** + * Accessing files and attributes via SAF is costly. + * This class caches them to speed up SAF related operations. + */ +internal class SafCache { + + /** + * The folder for the current user ID (here "${ANDROID_ID}.sv"). + */ + var currentFolder: DocumentFile? = null + + /** + * Folders containing chunks for backups of the current user ID. + */ + val backupChunkFolders = HashMap(CHUNK_FOLDER_COUNT) + + /** + * Folders containing chunks for restore of a chosen [StoredSnapshot]. + */ + val restoreChunkFolders = HashMap(CHUNK_FOLDER_COUNT) + + /** + * Files for each [StoredSnapshot]. + */ + val snapshotFiles = HashMap() + +} diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt index 93337c6f..9d4b3168 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt @@ -1,9 +1,13 @@ package org.calyxos.backup.storage.plugin.saf +import android.annotation.SuppressLint import android.content.Context +import android.provider.Settings +import android.provider.Settings.Secure.ANDROID_ID import android.util.Log import androidx.documentfile.provider.DocumentFile import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createDirectoryOrThrow import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createFileOrThrow @@ -15,11 +19,12 @@ import java.io.IOException import java.io.InputStream import java.io.OutputStream +private val folderRegex = Regex("^[a-f0-9]{16}\\.sv$") private val chunkFolderRegex = Regex("[a-f0-9]{2}") private val chunkRegex = Regex("[a-f0-9]{64}") private val snapshotRegex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286 -private const val CHUNK_FOLDER_COUNT = 256 private const val MIME_TYPE: String = "application/octet-stream" +internal const val CHUNK_FOLDER_COUNT = 256 private const val TAG = "SafStoragePlugin" @@ -28,12 +33,30 @@ public abstract class SafStoragePlugin( private val context: Context, ) : StoragePlugin { + private val cache = SafCache() protected abstract val root: DocumentFile? - private val contentResolver = context.contentResolver + private val folder: DocumentFile? + get() { + val root = this.root ?: return null + if (cache.currentFolder != null) return cache.currentFolder - private val chunkFolders = HashMap(CHUNK_FOLDER_COUNT) - private val snapshotFiles = HashMap() + @SuppressLint("HardwareIds") + // this is unique to each combination of app-signing key, user, and device + // so we don't leak anything by not hashing this and can use it as is + val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID) + // the folder name is our user ID + val folderName = "$androidId.sv" + cache.currentFolder = try { + root.findFile(folderName) ?: root.createDirectoryOrThrow(folderName) + } catch (e: IOException) { + Log.e(TAG, "Error creating storage folder $folderName") + null + } + return cache.currentFolder + } + + private val contentResolver = context.contentResolver private fun timestampToSnapshot(timestamp: Long): String { return "$timestamp.SeedSnap" @@ -41,27 +64,33 @@ public abstract class SafStoragePlugin( @Throws(IOException::class) override suspend fun getAvailableChunkIds(): List { - val root = root ?: return emptyList() + val folder = folder ?: return emptyList() val chunkIds = ArrayList() - populateChunkFolders(root) { file, name -> + populateChunkFolders(folder, cache.backupChunkFolders) { file, name -> if (chunkFolderRegex.matches(name)) { chunkIds.addAll(getChunksFromFolder(file)) } } - Log.e(TAG, "Got ${chunkIds.size} available chunks") + Log.i(TAG, "Got ${chunkIds.size} available chunks") return chunkIds } + /** + * Goes through all files in the given [folder] and performs the optional [fileOp] on them. + * Afterwards, it creates missing chunk folders, as needed. + * Chunk folders will get cached in the given [chunkFolders] for faster access. + */ @Throws(IOException::class) private suspend fun populateChunkFolders( - root: DocumentFile, + folder: DocumentFile, + chunkFolders: HashMap, fileOp: ((DocumentFile, String) -> Unit)? = null ) { val expectedChunkFolders = (0x00..0xff).map { Integer.toHexString(it).padStart(2, '0') }.toHashSet() val duration = measure { - for (file in root.listFilesBlocking(context)) { + for (file in folder.listFilesBlocking(context)) { val name = file.name ?: continue if (chunkFolderRegex.matches(name)) { chunkFolders[name] = file @@ -70,8 +99,8 @@ public abstract class SafStoragePlugin( fileOp?.invoke(file, name) } } - Log.e(TAG, "Retrieving chunk folders took $duration") - createMissingChunkFolders(root, expectedChunkFolders) + Log.i(TAG, "Retrieving chunk folders took $duration") + createMissingChunkFolders(folder, chunkFolders, expectedChunkFolders) } @Throws(IOException::class) @@ -89,7 +118,11 @@ public abstract class SafStoragePlugin( } @Throws(IOException::class) - private fun createMissingChunkFolders(root: DocumentFile, expectedChunkFolders: Set) { + private fun createMissingChunkFolders( + root: DocumentFile, + chunkFolders: HashMap, + expectedChunkFolders: Set + ) { val s = expectedChunkFolders.size val duration = measure { for ((i, chunkFolderName) in expectedChunkFolders.withIndex()) { @@ -101,13 +134,14 @@ public abstract class SafStoragePlugin( throw IOException("Only have ${chunkFolders.size} chunk folders.") } } - if (s > 0) Log.e(TAG, "Creating $s missing chunk folders took $duration") + if (s > 0) Log.i(TAG, "Creating $s missing chunk folders took $duration") } @Throws(IOException::class) override fun getChunkOutputStream(chunkId: String): OutputStream { val chunkFolderName = chunkId.substring(0, 2) - val chunkFolder = chunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId") + val chunkFolder = + cache.backupChunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId") // TODO should we check if it exists first? val chunkFile = chunkFolder.createFileOrThrow(chunkId, MIME_TYPE) return chunkFile.getOutputStream(context.contentResolver) @@ -115,48 +149,58 @@ public abstract class SafStoragePlugin( @Throws(IOException::class) override fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream { - val root = root ?: throw IOException() + val folder = folder ?: throw IOException() val name = timestampToSnapshot(timestamp) // TODO should we check if it exists first? - val snapshotFile = root.createFileOrThrow(name, MIME_TYPE) + val snapshotFile = folder.createFileOrThrow(name, MIME_TYPE) return snapshotFile.getOutputStream(contentResolver) } /************************* Restore *******************************/ @Throws(IOException::class) - override suspend fun getAvailableBackupSnapshots(): List { - val root = root ?: return emptyList() - val snapshots = ArrayList() + override suspend fun getBackupSnapshotsForRestore(): List { + val snapshots = ArrayList() - populateChunkFolders(root) { file, name -> - val match = snapshotRegex.matchEntire(name) - if (match != null) { - val timestamp = match.groupValues[1].toLong() - snapshots.add(timestamp) - snapshotFiles[timestamp] = file + root?.listFilesBlocking(context)?.forEach { folder -> + val folderName = folder.name ?: "" + if (!folderRegex.matches(folderName)) return@forEach + + Log.i(TAG, "Checking $folderName for snapshots...") + for (file in folder.listFilesBlocking(context)) { + val name = file.name ?: continue + val match = snapshotRegex.matchEntire(name) + if (match != null) { + val timestamp = match.groupValues[1].toLong() + val storedSnapshot = StoredSnapshot(folderName, timestamp) + snapshots.add(storedSnapshot) + cache.snapshotFiles[storedSnapshot] = file + } } } - Log.e(TAG, "Got ${snapshots.size} snapshots while populating chunk folders") + Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders") return snapshots } @Throws(IOException::class) - override suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream { - val snapshotFile = snapshotFiles.getOrElse(timestamp) { - root?.findFileBlocking(context, timestampToSnapshot(timestamp)) + override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream { + val timestamp = storedSnapshot.timestamp + val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) { + getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp)) } ?: throw IOException("Could not get file for snapshot $timestamp") return snapshotFile.getInputStream(contentResolver) } @Throws(IOException::class) - override suspend fun getChunkInputStream(chunkId: String): InputStream { - if (chunkFolders.size < CHUNK_FOLDER_COUNT) { - val root = root ?: throw IOException("Could not get root") - populateChunkFolders(root) + override suspend fun getChunkInputStream( + snapshot: StoredSnapshot, + chunkId: String + ): InputStream { + if (cache.restoreChunkFolders.size < CHUNK_FOLDER_COUNT) { + populateChunkFolders(getFolder(snapshot), cache.restoreChunkFolders) } val chunkFolderName = chunkId.substring(0, 2) - val chunkFolder = chunkFolders[chunkFolderName] + val chunkFolder = cache.restoreChunkFolders[chunkFolderName] ?: throw IOException("No folder for chunk $chunkId") val chunkFile = chunkFolder.findFileBlocking(context, chunkId) ?: throw IOException("No chunk $chunkId") @@ -164,23 +208,56 @@ public abstract class SafStoragePlugin( } @Throws(IOException::class) - override suspend fun deleteBackupSnapshot(timestamp: Long) { - Log.d(TAG, "Deleting snapshot $timestamp") - val snapshotFile = snapshotFiles.getOrElse(timestamp) { - root?.findFileBlocking(context, timestampToSnapshot(timestamp)) - } ?: throw IOException("Could not get file for snapshot $timestamp") - if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp") + private suspend fun getFolder(storedSnapshot: StoredSnapshot): DocumentFile { + // not cached, because used in several places only once and + // [getBackupSnapshotInputStream] uses snapshot files cache and + // [getChunkInputStream] uses restore chunk folders cache + return root?.findFileBlocking(context, storedSnapshot.userId) + ?: throw IOException("Could not find snapshot $storedSnapshot") } + /************************* Pruning *******************************/ + + @Throws(IOException::class) + override suspend fun getCurrentBackupSnapshots(): List { + val folder = folder ?: return emptyList() + val folderName = folder.name ?: error("Folder suddenly has no more name") + val snapshots = ArrayList() + + populateChunkFolders(folder, cache.backupChunkFolders) { file, name -> + val match = snapshotRegex.matchEntire(name) + if (match != null) { + val timestamp = match.groupValues[1].toLong() + val storedSnapshot = StoredSnapshot(folderName, timestamp) + snapshots.add(storedSnapshot) + cache.snapshotFiles[storedSnapshot] = file + } + } + Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders") + return snapshots + } + + @Throws(IOException::class) + override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) { + val timestamp = storedSnapshot.timestamp + Log.d(TAG, "Deleting snapshot $timestamp") + val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) { + getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp)) + } ?: throw IOException("Could not get file for snapshot $timestamp") + if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp") + cache.snapshotFiles.remove(storedSnapshot) + } + + @Throws(IOException::class) override suspend fun deleteChunks(chunkIds: List) { - if (chunkFolders.size < CHUNK_FOLDER_COUNT) { - val root = root ?: throw IOException("Could not get root") - populateChunkFolders(root) + if (cache.backupChunkFolders.size < CHUNK_FOLDER_COUNT) { + val folder = folder ?: throw IOException("Could not get current folder in root") + populateChunkFolders(folder, cache.backupChunkFolders) } for (chunkId in chunkIds) { Log.d(TAG, "Deleting chunk $chunkId") val chunkFolderName = chunkId.substring(0, 2) - val chunkFolder = chunkFolders[chunkFolderName] + val chunkFolder = cache.backupChunkFolders[chunkFolderName] ?: throw IOException("No folder for chunk $chunkId") val chunkFile = chunkFolder.findFileBlocking(context, chunkId) if (chunkFile == null) { diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt index 23082746..9426cf5e 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt @@ -3,6 +3,7 @@ package org.calyxos.backup.storage.prune import android.util.Log import org.calyxos.backup.storage.api.BackupObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.measure @@ -33,15 +34,15 @@ internal class Pruner( @Throws(IOException::class) suspend fun prune(backupObserver: BackupObserver?) { val duration = measure { - val timestamps = storagePlugin.getAvailableBackupSnapshots() - val toDelete = retentionManager.getSnapshotsToDelete(timestamps) - backupObserver?.onPruneStart(toDelete) - for (timestamp in toDelete) { + val storedSnapshots = storagePlugin.getCurrentBackupSnapshots() + val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots) + backupObserver?.onPruneStart(toDelete.map { it.timestamp }) + for (snapshot in toDelete) { try { - pruneSnapshot(timestamp, backupObserver) + pruneSnapshot(snapshot, backupObserver) } catch (e: Exception) { - Log.e(TAG, "Error pruning $timestamp", e) - backupObserver?.onPruneError(timestamp, e) + Log.e(TAG, "Error pruning $snapshot", e) + backupObserver?.onPruneError(snapshot.timestamp, e) } } } @@ -50,12 +51,15 @@ internal class Pruner( } @Throws(IOException::class, GeneralSecurityException::class) - private suspend fun pruneSnapshot(timestamp: Long, backupObserver: BackupObserver?) { - val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp) + private suspend fun pruneSnapshot( + storedSnapshot: StoredSnapshot, + backupObserver: BackupObserver? + ) { + val snapshot = snapshotRetriever.getSnapshot(streamKey, storedSnapshot) val chunks = HashSet() snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) } snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) } - storagePlugin.deleteBackupSnapshot(timestamp) + storagePlugin.deleteBackupSnapshot(storedSnapshot) db.applyInParts(chunks) { chunksCache.decrementRefCount(it) } @@ -68,7 +72,7 @@ internal class Pruner( size += it.size it.id } - backupObserver?.onPruneSnapshot(timestamp, chunkIdsToDelete.size, size) + backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size) storagePlugin.deleteChunks(chunkIdsToDelete) chunksCache.deleteChunks(cachedChunksToDelete) } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt index b1efeb02..8f2097ef 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt @@ -2,6 +2,7 @@ package org.calyxos.backup.storage.prune import android.content.Context import org.calyxos.backup.storage.api.SnapshotRetention +import org.calyxos.backup.storage.api.StoredSnapshot import java.io.IOException import java.time.LocalDate import java.time.temporal.ChronoField @@ -48,37 +49,37 @@ internal class RetentionManager(private val context: Context) { } /** - * Takes a list of snapshot timestamps and returns a list of those + * Takes a list of [StoredSnapshot]s and returns a list of those * that can be deleted according to the current snapshot retention policy. */ @Throws(IOException::class) - fun getSnapshotsToDelete(snapshotTimestamps: List): List { + fun getSnapshotsToDelete(storedSnapshots: List): List { val retention = getSnapshotRetention() - val dates = snapshotTimestamps.sortedDescending().map { - Pair(it, LocalDate.ofEpochDay(it / 1000 / 60 / 60 / 24)) + val datePairs = storedSnapshots.sortedByDescending { it.timestamp }.map { s -> + Pair(s, LocalDate.ofEpochDay(s.timestamp / 1000 / 60 / 60 / 24)) } - val toKeep = HashSet() - toKeep += getToKeep(dates, retention.daily) - toKeep += getToKeep(dates, retention.weekly) { temporal: Temporal -> + val toKeep = HashSet() + toKeep += getToKeep(datePairs, retention.daily) + toKeep += getToKeep(datePairs, retention.weekly) { temporal: Temporal -> temporal.with(ChronoField.DAY_OF_WEEK, 1) } - toKeep += getToKeep(dates, retention.monthly, firstDayOfMonth()) - toKeep += getToKeep(dates, retention.yearly, firstDayOfYear()) - return snapshotTimestamps - toKeep + toKeep += getToKeep(datePairs, retention.monthly, firstDayOfMonth()) + toKeep += getToKeep(datePairs, retention.yearly, firstDayOfYear()) + return storedSnapshots - toKeep } private fun getToKeep( - pairs: List>, + pairs: List>, keep: Int, temporalAdjuster: TemporalAdjuster? = null, - ): List { - val toKeep = ArrayList() + ): List { + val toKeep = ArrayList() if (keep == 0) return toKeep var last: LocalDate? = null - for ((timestamp, date) in pairs) { + for ((snapshot, date) in pairs) { val period = if (temporalAdjuster == null) date else date.with(temporalAdjuster) if (period != last) { - toKeep.add(timestamp) + toKeep.add(snapshot) if (toKeep.size >= keep) break last = period } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt index f7111c2e..6b22c5a8 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt @@ -2,6 +2,7 @@ package org.calyxos.backup.storage.restore import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import java.io.IOException import java.io.InputStream @@ -19,10 +20,11 @@ internal abstract class AbstractChunkRestore( @Throws(IOException::class, GeneralSecurityException::class) protected suspend fun getAndDecryptChunk( version: Int, + storedSnapshot: StoredSnapshot, chunkId: String, streamReader: suspend (InputStream) -> Unit, ) { - storagePlugin.getChunkInputStream(chunkId).use { inputStream -> + storagePlugin.getChunkInputStream(storedSnapshot, chunkId).use { inputStream -> inputStream.readVersion(version) val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte()) streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream -> diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt index 83b7f2cc..372f91d7 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt @@ -4,6 +4,7 @@ import android.content.Context import android.util.Log import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import java.io.File import java.io.FileInputStream @@ -26,6 +27,7 @@ internal class MultiChunkRestore( suspend fun restore( version: Int, + storedSnapshot: StoredSnapshot, chunkMap: Map, files: Collection, observer: RestoreObserver?, @@ -34,7 +36,7 @@ internal class MultiChunkRestore( files.forEach { file -> try { restoreFile(file, observer, "L") { outputStream -> - writeChunks(version, file, chunkMap, outputStream) + writeChunks(version, storedSnapshot, file, chunkMap, outputStream) } restoredFiles++ } catch (e: Exception) { @@ -49,6 +51,7 @@ internal class MultiChunkRestore( @Throws(IOException::class, GeneralSecurityException::class) private suspend fun writeChunks( version: Int, + storedSnapshot: StoredSnapshot, file: RestorableFile, chunkMap: Map, outputStream: OutputStream, @@ -61,8 +64,11 @@ internal class MultiChunkRestore( bytes += decryptedStream.copyTo(outputStream) } val isCached = isCached(chunkId) - if (isCached || otherFiles.size > 1) getAndCacheChunk(version, chunkId, chunkWriter) - else getAndDecryptChunk(version, chunkId, chunkWriter) + if (isCached || otherFiles.size > 1) { + getAndCacheChunk(version, storedSnapshot, chunkId, chunkWriter) + } else { + getAndDecryptChunk(version, storedSnapshot, chunkId, chunkWriter) + } otherFiles.remove(file) if (isCached && otherFiles.isEmpty()) removeCachedChunk(chunkId) @@ -77,13 +83,14 @@ internal class MultiChunkRestore( @Throws(IOException::class, GeneralSecurityException::class) private suspend fun getAndCacheChunk( version: Int, + storedSnapshot: StoredSnapshot, chunkId: String, streamReader: suspend (InputStream) -> Unit, ) { val file = getChunkCacheFile(chunkId) if (!file.isFile) { FileOutputStream(file).use { outputStream -> - getAndDecryptChunk(version, chunkId) { decryptedStream -> + getAndDecryptChunk(version, storedSnapshot, chunkId) { decryptedStream -> decryptedStream.copyTo(outputStream) } } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt index 70fa18ed..819058d3 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt @@ -8,6 +8,7 @@ import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.SnapshotItem import org.calyxos.backup.storage.api.SnapshotResult import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto @@ -46,8 +47,10 @@ internal class Restore( val numSnapshots: Int val time = measure { val list = try { - storagePlugin.getAvailableBackupSnapshots().sortedDescending().map { - SnapshotItem(it, null) + storagePlugin.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot -> + storedSnapshot.timestamp + }.map { storedSnapshot -> + SnapshotItem(storedSnapshot, null) }.toMutableList() } catch (e: Exception) { Log.e("TAG", "Error retrieving snapshots", e) @@ -61,7 +64,9 @@ internal class Restore( while (iterator.hasNext()) { val oldItem = iterator.next() val item = try { - oldItem.copy(snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.time)) + oldItem.copy( + snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.storedSnapshot) + ) } catch (e: Exception) { Log.e("TAG", "Error retrieving snapshot ${oldItem.time}", e) continue @@ -73,15 +78,15 @@ internal class Restore( Log.e(TAG, "Decrypting and parsing $numSnapshots snapshots took $time") } - @Throws(IOException::class, GeneralSecurityException::class) - suspend fun restoreBackupSnapshot(timestamp: Long, observer: RestoreObserver?) { - val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp) - restoreBackupSnapshot(snapshot, observer) - } - @OptIn(ExperimentalTime::class) - @Throws(IOException::class) - suspend fun restoreBackupSnapshot(snapshot: BackupSnapshot, observer: RestoreObserver?) { + @Throws(IOException::class, GeneralSecurityException::class) + suspend fun restoreBackupSnapshot( + storedSnapshot: StoredSnapshot, + optionalSnapshot: BackupSnapshot? = null, + observer: RestoreObserver? = null, + ) { + val snapshot = optionalSnapshot ?: snapshotRetriever.getSnapshot(streamKey, storedSnapshot) + val filesTotal = snapshot.mediaFilesList.size + snapshot.documentFilesList.size val totalSize = snapshot.mediaFilesList.sumOf { it.size } + snapshot.documentFilesList.sumOf { it.size } @@ -91,19 +96,30 @@ internal class Restore( val version = snapshot.version var restoredFiles = 0 val smallFilesDuration = measure { - restoredFiles += zipChunkRestore.restore(version, split.zipChunks, observer) + restoredFiles += zipChunkRestore.restore( + version, + storedSnapshot, + split.zipChunks, + observer, + ) } Log.e(TAG, "Restoring ${split.zipChunks.size} zip chunks took $smallFilesDuration.") val singleChunkDuration = measure { - restoredFiles += singleChunkRestore.restore(version, split.singleChunks, observer) + restoredFiles += singleChunkRestore.restore( + version, + storedSnapshot, + split.singleChunks, + observer, + ) } Log.e(TAG, "Restoring ${split.singleChunks.size} single chunks took $singleChunkDuration.") val multiChunkDuration = measure { restoredFiles += multiChunkRestore.restore( version, + storedSnapshot, split.multiChunkMap, split.multiChunkFiles, - observer + observer, ) } Log.e(TAG, "Restoring ${split.multiChunkFiles.size} multi chunks took $multiChunkDuration.") diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt index 568864ae..1917e420 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt @@ -8,20 +8,24 @@ import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StorageBackup -import org.calyxos.backup.storage.backup.BackupSnapshot +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START +import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID import org.calyxos.backup.storage.ui.NOTIFICATION_ID_RESTORE import org.calyxos.backup.storage.ui.Notifications /** * Start to trigger restore as a foreground service. Ensure that you provide the snapshot - * to be restored with [Intent.putExtra] as a [Long] in [EXTRA_TIMESTAMP_START]. - * See [BackupSnapshot.getTimeStart]. + * to be restored with [Intent.putExtra]: + * * the user ID of the snapshot as a [String] in [EXTRA_USER_ID] + * * the snapshot's timestamp as a [Long] in [EXTRA_TIMESTAMP_START]. + * See [BackupSnapshot.getTimeStart]. */ public abstract class RestoreService : Service() { public companion object { private const val TAG = "RestoreService" + public const val EXTRA_USER_ID: String = "userId" public const val EXTRA_TIMESTAMP_START: String = "timestamp" } @@ -31,13 +35,15 @@ public abstract class RestoreService : Service() { override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Log.d(TAG, "onStartCommand $intent $flags $startId") - val timestamp = intent?.getLongExtra(EXTRA_TIMESTAMP_START, -1) - if (timestamp == null || timestamp < 0) error("No timestamp in intent: $intent") + val userId = intent?.getStringExtra(EXTRA_USER_ID) ?: error("No user ID in intent: $intent") + val timestamp = intent.getLongExtra(EXTRA_TIMESTAMP_START, -1) + if (timestamp < 0) error("No timestamp in intent: $intent") + val storedSnapshot = StoredSnapshot(userId, timestamp) startForeground(NOTIFICATION_ID_RESTORE, n.getRestoreNotification()) GlobalScope.launch { // TODO offer a way to try again if failed, or do an automatic retry here - storageBackup.restoreBackupSnapshot(timestamp, restoreObserver) + storageBackup.restoreBackupSnapshot(storedSnapshot, null, restoreObserver) stopSelf(startId) } return START_STICKY_COMPATIBILITY diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt index 0d04ccf6..f6e23806 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt @@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore import android.util.Log import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto private const val TAG = "SingleChunkRestore" @@ -17,6 +18,7 @@ internal class SingleChunkRestore( suspend fun restore( version: Int, + storedSnapshot: StoredSnapshot, chunks: Collection, observer: RestoreObserver? ): Int { @@ -25,7 +27,7 @@ internal class SingleChunkRestore( check(chunk.files.size == 1) val file = chunk.files[0] try { - getAndDecryptChunk(version, chunk.chunkId) { decryptedStream -> + getAndDecryptChunk(version, storedSnapshot, chunk.chunkId) { decryptedStream -> restoreFile(file, observer, "M") { outputStream -> decryptedStream.copyTo(outputStream) } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt index e27982bd..b937bc85 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt @@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore import android.util.Log import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import java.io.IOException import java.io.InputStream @@ -24,13 +25,14 @@ internal class ZipChunkRestore( */ suspend fun restore( version: Int, + storedSnapshot: StoredSnapshot, zipChunks: Collection, observer: RestoreObserver? ): Int { var restoredFiles = 0 zipChunks.forEach { zipChunk -> try { - getAndDecryptChunk(version, zipChunk.chunkId) { decryptedStream -> + getAndDecryptChunk(version, storedSnapshot, zipChunk.chunkId) { decryptedStream -> restoredFiles += restoreZipChunk(zipChunk, decryptedStream, observer) } } catch (e: Exception) { diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt index 9597f5cd..78ef34a6 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt @@ -19,6 +19,7 @@ import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.SnapshotResult import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX import org.calyxos.backup.storage.backup.Backup.Companion.SMALL_FILE_SIZE_MAX @@ -173,14 +174,16 @@ internal class BackupRestoreTest { // RESTORE + val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured) + val smallFileMOutputStream = ByteArrayOutputStream() val smallFileDOutputStream = ByteArrayOutputStream() val fileMOutputStream = ByteArrayOutputStream() val fileDOutputStream = ByteArrayOutputStream() - coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured) + coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot) coEvery { - plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured) + plugin.getBackupSnapshotInputStream(storedSnapshot) } returns ByteArrayInputStream(snapshotOutputStream.toByteArray()) // retrieve snapshots @@ -197,14 +200,14 @@ internal class BackupRestoreTest { // pipe chunks back in coEvery { - plugin.getChunkInputStream(cachedFiles[0].chunks[0]) + plugin.getChunkInputStream(storedSnapshot, cachedFiles[0].chunks[0]) } returns ByteArrayInputStream(zipChunkOutputStream.toByteArray()) // cachedFiles[0].chunks[1] is in previous zipChunk coEvery { - plugin.getChunkInputStream(cachedFiles[2].chunks[0]) + plugin.getChunkInputStream(storedSnapshot, cachedFiles[2].chunks[0]) } returns ByteArrayInputStream(mOutputStream.toByteArray()) coEvery { - plugin.getChunkInputStream(cachedFiles[3].chunks[0]) + plugin.getChunkInputStream(storedSnapshot, cachedFiles[3].chunks[0]) } returns ByteArrayInputStream(dOutputStream.toByteArray()) // provide file output streams for restore @@ -217,7 +220,7 @@ internal class BackupRestoreTest { val fileDRestorable = getRestorableFileD(fileD, snapshot) expectRestoreFile(fileDRestorable, fileDOutputStream) - restore.restoreBackupSnapshot(snapshot, null) + restore.restoreBackupSnapshot(storedSnapshot, snapshot, null) // restored files match backed up files exactly assertArrayEquals(smallFileMBytes, smallFileMOutputStream.toByteArray()) @@ -337,9 +340,11 @@ internal class BackupRestoreTest { // RESTORE - coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured) + val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured) + + coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot) coEvery { - plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured) + plugin.getBackupSnapshotInputStream(storedSnapshot) } returns ByteArrayInputStream(snapshotOutputStream.toByteArray()) // retrieve snapshots @@ -354,19 +359,27 @@ internal class BackupRestoreTest { // pipe chunks back in coEvery { plugin.getChunkInputStream( - "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3") + storedSnapshot, + "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" + ) } returns ByteArrayInputStream(id040f32.toByteArray()) coEvery { plugin.getChunkInputStream( - "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29") + storedSnapshot, + "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" + ) } returns ByteArrayInputStream(id901fbc.toByteArray()) coEvery { plugin.getChunkInputStream( - "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d") + storedSnapshot, + "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" + ) } returns ByteArrayInputStream(id5adea3.toByteArray()) coEvery { plugin.getChunkInputStream( - "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67") + storedSnapshot, + "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" + ) } returns ByteArrayInputStream(id40d00c.toByteArray()) // provide file output streams for restore @@ -375,7 +388,7 @@ internal class BackupRestoreTest { val file2Restorable = getRestorableFileD(file2, snapshot) expectRestoreFile(file2Restorable, file2OutputStream) - restore.restoreBackupSnapshot(snapshot, null) + restore.restoreBackupSnapshot(storedSnapshot, snapshot, null) // restored files match backed up files exactly assertArrayEquals(file1Bytes, file1OutputStream.toByteArray()) @@ -384,13 +397,21 @@ internal class BackupRestoreTest { // chunks were only read from storage once coVerify(exactly = 1) { plugin.getChunkInputStream( - "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3") + storedSnapshot, + "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" + ) plugin.getChunkInputStream( - "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29") + storedSnapshot, + "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" + ) plugin.getChunkInputStream( - "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d") + storedSnapshot, + "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" + ) plugin.getChunkInputStream( - "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67") + storedSnapshot, + "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" + ) } } diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt index 64fbac94..b1fb556a 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt @@ -9,6 +9,7 @@ import io.mockk.mockk import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.db.CachedChunk import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.db.Db @@ -56,7 +57,9 @@ internal class ChunksCacheRepopulaterTest { .addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4)) .addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk5)) .build() - val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart) + val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart) + val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart) + val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2) val cachedChunks = listOf( CachedChunk(chunk1, 2, 0), CachedChunk(chunk2, 2, 0), @@ -64,12 +67,12 @@ internal class ChunksCacheRepopulaterTest { ) // chunk3 is not referenced and should get deleted val cachedChunksSlot = slot>() - coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps + coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots coEvery { - snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart) + snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1 coEvery { - snapshotRetriever.getSnapshot(streamKey, snapshot2.timeStart) + snapshotRetriever.getSnapshot(streamKey, storedSnapshot2) } returns snapshot2 every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt index 56cb71fb..3c359cf6 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt @@ -8,6 +8,7 @@ import io.mockk.mockk import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.BackupDocumentFile import org.calyxos.backup.storage.backup.BackupMediaFile import org.calyxos.backup.storage.backup.BackupSnapshot @@ -66,18 +67,20 @@ internal class PrunerTest { .addMediaFiles(BackupMediaFile.newBuilder().addChunkIds(chunk2)) .addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4)) .build() - val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart) + val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart) + val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart) + val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2) val expectedChunks = listOf(chunk1, chunk2, chunk3) val actualChunks = slot>() val actualChunks2 = slot>() val cachedChunk3 = CachedChunk(chunk3, 0, 0) - coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps + coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots every { - retentionManager.getSnapshotsToDelete(snapshotTimestamps) - } returns listOf(snapshot1.timeStart) - coEvery { snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart) } returns snapshot1 - coEvery { plugin.deleteBackupSnapshot(snapshot1.timeStart) } just Runs + retentionManager.getSnapshotsToDelete(storedSnapshots) + } returns listOf(storedSnapshot1) + coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1 + coEvery { plugin.deleteBackupSnapshot(storedSnapshot1) } just Runs every { db.applyInParts(capture(actualChunks), captureLambda()) } answers { diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt index 784013cb..7c19d7e5 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt @@ -5,6 +5,8 @@ import android.content.SharedPreferences import io.mockk.every import io.mockk.mockk import org.calyxos.backup.storage.api.SnapshotRetention +import org.calyxos.backup.storage.api.StoredSnapshot +import org.calyxos.backup.storage.getRandomString import org.junit.Assert.assertEquals import org.junit.Test import java.time.LocalDateTime @@ -17,19 +19,21 @@ internal class RetentionManagerTest { private val retention = RetentionManager(context) + private val userId = getRandomString() + @Test fun testDailyRetention() { expectGetRetention(SnapshotRetention(2, 0, 0, 0)) - val timestamps = listOf( + val storedSnapshots = listOf( // 1577919600000 LocalDateTime.of(2020, 1, 1, 23, 0).toMillis(), // 1577872800000 LocalDateTime.of(2020, 1, 1, 10, 0).toMillis(), // 1583276400000 LocalDateTime.of(2020, 3, 3, 23, 0).toMillis(), - ) - val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1577872800000), toDelete) + ).map { StoredSnapshot(userId, it) } + val toDelete = retention.getSnapshotsToDelete(storedSnapshots) + assertEquals(listOf(1577872800000), toDelete.map { it.timestamp }) } @Test @@ -45,9 +49,9 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(), // 1608678000000 LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1608544800000, 1608638400000), toDelete) + assertEquals(listOf(1608544800000, 1608638400000), toDelete.map { it.timestamp }) } @Test @@ -63,9 +67,9 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(), // 1608678000000 LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1580857200000), toDelete) + assertEquals(listOf(1580857200000), toDelete.map { it.timestamp }) } @Test @@ -83,10 +87,10 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(), // 1608678000000 LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } // keeps only the latest one for each year, so three in total, even though keep is four val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1549321200000, 1608544800000), toDelete) + assertEquals(listOf(1549321200000, 1608544800000), toDelete.map { it.timestamp }) } @Test @@ -116,9 +120,11 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), // 1608638400000 LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000), toDelete) + assertEquals( + listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000), + toDelete.map { it.timestamp }) } private fun expectGetRetention(snapshotRetention: SnapshotRetention) {