diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt index 6fb3dea8..2b4e398f 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt @@ -62,6 +62,7 @@ import kotlinx.coroutines.launch import org.calyxos.backup.storage.api.SnapshotItem import org.calyxos.backup.storage.api.StorageBackup import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START +import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID import org.calyxos.backup.storage.ui.restore.SnapshotViewModel import java.util.LinkedList import kotlin.coroutines.Continuation @@ -392,6 +393,7 @@ internal class RestoreViewModel( @UiThread internal fun startFilesRestore(item: SnapshotItem) { val i = Intent(app, StorageRestoreService::class.java) + i.putExtra(EXTRA_USER_ID, item.storedSnapshot.userId) i.putExtra(EXTRA_TIMESTAMP_START, item.time) app.startForegroundService(i) mDisplayFragment.setEvent(RESTORE_FILES_STARTED) diff --git a/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt b/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt index bd55b1e4..d68d76cf 100644 --- a/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt +++ b/storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt @@ -123,6 +123,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati // example for how to do restore via foreground service // app.startForegroundService(Intent(app, DemoRestoreService::class.java).apply { +// putExtra(EXTRA_USER_ID, item.storedSnapshot.userId) // putExtra(EXTRA_TIMESTAMP_START, snapshot.timeStart) // }) @@ -130,7 +131,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati _restoreProgressVisible.value = true val restoreObserver = RestoreStats(app, _restoreLog) viewModelScope.launch { - storageBackup.restoreBackupSnapshot(snapshot, restoreObserver) + storageBackup.restoreBackupSnapshot(item.storedSnapshot, snapshot, restoreObserver) _restoreProgressVisible.value = false } } diff --git a/storage/doc/design.md b/storage/doc/design.md index 332bb571..f43c51f4 100644 --- a/storage/doc/design.md +++ b/storage/doc/design.md @@ -388,7 +388,6 @@ but are considered out-of-scope of the current design for time and budget reason * using a rolling hash to produce chunks in order to increase likelihood of obtaining same chunks even if file contents change slightly or shift * external secret-less corruption checks that would use checksums over encrypted data -* supporting different backup clients backing up to the same storage * concealing file sizes (though zip chunks helps a bit here) * implementing different storage plugins diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt index d8c11bd8..4e936664 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt @@ -3,15 +3,28 @@ package org.calyxos.backup.storage.api import org.calyxos.backup.storage.backup.BackupSnapshot public data class SnapshotItem( - public val time: Long, + public val storedSnapshot: StoredSnapshot, public val snapshot: BackupSnapshot?, -) +) { + val time: Long get() = storedSnapshot.timestamp +} public sealed class SnapshotResult { public data class Success(val snapshots: List) : SnapshotResult() public data class Error(val e: Exception) : SnapshotResult() } +public data class StoredSnapshot( + /** + * The unique ID of the current device/user combination chosen by the [StoragePlugin]. + */ + public val userId: String, + /** + * The timestamp identifying a snapshot of the [userId]. + */ + public val timestamp: Long, +) + /** * Defines which backup snapshots should be retained when pruning backups. * diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt index 969c9851..1bb556a6 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt @@ -103,10 +103,12 @@ public class StorageBackup( * Run this on a new storage location to ensure that there are no old snapshots * (potentially encrypted with an old key) laying around. * Using a storage location with existing data is not supported. + * Using the same root folder for storage on different devices or user profiles is fine though + * as the [StoragePlugin] should isolate storage per [StoredSnapshot.userId]. */ public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) { try { - plugin.getAvailableBackupSnapshots().forEach { + plugin.getCurrentBackupSnapshots().forEach { try { plugin.deleteBackupSnapshot(it) } catch (e: IOException) { @@ -183,15 +185,16 @@ public class StorageBackup( } public suspend fun restoreBackupSnapshot( - snapshot: BackupSnapshot, - restoreObserver: RestoreObserver? = null + storedSnapshot: StoredSnapshot, + snapshot: BackupSnapshot? = null, + restoreObserver: RestoreObserver? = null, ): Boolean = withContext(dispatcher) { if (restoreRunning.getAndSet(true)) { Log.w(TAG, "Restore already running, not starting a new one") return@withContext false } try { - restore.restoreBackupSnapshot(snapshot, restoreObserver) + restore.restoreBackupSnapshot(storedSnapshot, snapshot, restoreObserver) true } catch (e: Exception) { Log.e(TAG, "Error during restore", e) @@ -201,17 +204,4 @@ public class StorageBackup( } } - public suspend fun restoreBackupSnapshot( - timestamp: Long, - restoreObserver: RestoreObserver? = null - ): Boolean = withContext(dispatcher) { - try { - restore.restoreBackupSnapshot(timestamp, restoreObserver) - true - } catch (e: Exception) { - Log.e(TAG, "Error during restore", e) - false - } - } - } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt index 465d9c39..aa1be4d6 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt @@ -35,22 +35,36 @@ public interface StoragePlugin { /* Restore */ /** - * Returns the timestamps representing a backup snapshot that are available on storage. + * Returns *all* [StoredSnapshot]s that are available on storage + * independent of user ID and whether they can be decrypted + * with the key returned by [getMasterKey]. */ @Throws(IOException::class) - public suspend fun getAvailableBackupSnapshots(): List + public suspend fun getBackupSnapshotsForRestore(): List @Throws(IOException::class) - public suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream + public suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream @Throws(IOException::class) - public suspend fun getChunkInputStream(chunkId: String): InputStream + public suspend fun getChunkInputStream(snapshot: StoredSnapshot, chunkId: String): InputStream /* Pruning */ + /** + * Returns [StoredSnapshot]s for the currently active user ID. + */ @Throws(IOException::class) - public suspend fun deleteBackupSnapshot(timestamp: Long) + public suspend fun getCurrentBackupSnapshots(): List + /** + * Deletes the given [StoredSnapshot]. + */ + @Throws(IOException::class) + public suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) + + /** + * Deletes the given chunks of the *current* user ID only. + */ @Throws(IOException::class) public suspend fun deleteChunks(chunkIds: List) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt index cc9caf94..5e7afc06 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt @@ -7,6 +7,7 @@ import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.plugin.SnapshotRetriever import java.io.IOException +import java.security.GeneralSecurityException import java.util.concurrent.TimeUnit.MILLISECONDS import kotlin.time.ExperimentalTime import kotlin.time.toDuration @@ -37,8 +38,13 @@ internal class ChunksCacheRepopulater( availableChunkIds: HashSet ) { val start = System.currentTimeMillis() - val snapshots = storagePlugin.getAvailableBackupSnapshots().map { timestamp -> - snapshotRetriever.getSnapshot(streamKey, timestamp) + val snapshots = storagePlugin.getCurrentBackupSnapshots().mapNotNull { storedSnapshot -> + try { + snapshotRetriever.getSnapshot(streamKey, storedSnapshot) + } catch (e: GeneralSecurityException) { + Log.w(TAG, "Error fetching snapshot $storedSnapshot", e) + null + } } val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS) Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration") diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt index 050b3be9..38fdcc14 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt @@ -2,6 +2,7 @@ package org.calyxos.backup.storage.plugin import com.google.protobuf.InvalidProtocolBufferException import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.restore.readVersion @@ -19,9 +20,10 @@ internal class SnapshotRetriever( GeneralSecurityException::class, InvalidProtocolBufferException::class, ) - suspend fun getSnapshot(streamKey: ByteArray, timestamp: Long): BackupSnapshot { - return storagePlugin.getBackupSnapshotInputStream(timestamp).use { inputStream -> + suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot { + return storagePlugin.getBackupSnapshotInputStream(storedSnapshot).use { inputStream -> val version = inputStream.readVersion() + val timestamp = storedSnapshot.timestamp val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte()) streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream -> BackupSnapshot.parseFrom(decryptedStream) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt new file mode 100644 index 00000000..0480a1eb --- /dev/null +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt @@ -0,0 +1,32 @@ +package org.calyxos.backup.storage.plugin.saf + +import androidx.documentfile.provider.DocumentFile +import org.calyxos.backup.storage.api.StoredSnapshot + +/** + * Accessing files and attributes via SAF is costly. + * This class caches them to speed up SAF related operations. + */ +internal class SafCache { + + /** + * The folder for the current user ID (here "${ANDROID_ID}.sv"). + */ + var currentFolder: DocumentFile? = null + + /** + * Folders containing chunks for backups of the current user ID. + */ + val backupChunkFolders = HashMap(CHUNK_FOLDER_COUNT) + + /** + * Folders containing chunks for restore of a chosen [StoredSnapshot]. + */ + val restoreChunkFolders = HashMap(CHUNK_FOLDER_COUNT) + + /** + * Files for each [StoredSnapshot]. + */ + val snapshotFiles = HashMap() + +} diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt index 93337c6f..9d4b3168 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt @@ -1,9 +1,13 @@ package org.calyxos.backup.storage.plugin.saf +import android.annotation.SuppressLint import android.content.Context +import android.provider.Settings +import android.provider.Settings.Secure.ANDROID_ID import android.util.Log import androidx.documentfile.provider.DocumentFile import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createDirectoryOrThrow import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createFileOrThrow @@ -15,11 +19,12 @@ import java.io.IOException import java.io.InputStream import java.io.OutputStream +private val folderRegex = Regex("^[a-f0-9]{16}\\.sv$") private val chunkFolderRegex = Regex("[a-f0-9]{2}") private val chunkRegex = Regex("[a-f0-9]{64}") private val snapshotRegex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286 -private const val CHUNK_FOLDER_COUNT = 256 private const val MIME_TYPE: String = "application/octet-stream" +internal const val CHUNK_FOLDER_COUNT = 256 private const val TAG = "SafStoragePlugin" @@ -28,12 +33,30 @@ public abstract class SafStoragePlugin( private val context: Context, ) : StoragePlugin { + private val cache = SafCache() protected abstract val root: DocumentFile? - private val contentResolver = context.contentResolver + private val folder: DocumentFile? + get() { + val root = this.root ?: return null + if (cache.currentFolder != null) return cache.currentFolder - private val chunkFolders = HashMap(CHUNK_FOLDER_COUNT) - private val snapshotFiles = HashMap() + @SuppressLint("HardwareIds") + // this is unique to each combination of app-signing key, user, and device + // so we don't leak anything by not hashing this and can use it as is + val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID) + // the folder name is our user ID + val folderName = "$androidId.sv" + cache.currentFolder = try { + root.findFile(folderName) ?: root.createDirectoryOrThrow(folderName) + } catch (e: IOException) { + Log.e(TAG, "Error creating storage folder $folderName") + null + } + return cache.currentFolder + } + + private val contentResolver = context.contentResolver private fun timestampToSnapshot(timestamp: Long): String { return "$timestamp.SeedSnap" @@ -41,27 +64,33 @@ public abstract class SafStoragePlugin( @Throws(IOException::class) override suspend fun getAvailableChunkIds(): List { - val root = root ?: return emptyList() + val folder = folder ?: return emptyList() val chunkIds = ArrayList() - populateChunkFolders(root) { file, name -> + populateChunkFolders(folder, cache.backupChunkFolders) { file, name -> if (chunkFolderRegex.matches(name)) { chunkIds.addAll(getChunksFromFolder(file)) } } - Log.e(TAG, "Got ${chunkIds.size} available chunks") + Log.i(TAG, "Got ${chunkIds.size} available chunks") return chunkIds } + /** + * Goes through all files in the given [folder] and performs the optional [fileOp] on them. + * Afterwards, it creates missing chunk folders, as needed. + * Chunk folders will get cached in the given [chunkFolders] for faster access. + */ @Throws(IOException::class) private suspend fun populateChunkFolders( - root: DocumentFile, + folder: DocumentFile, + chunkFolders: HashMap, fileOp: ((DocumentFile, String) -> Unit)? = null ) { val expectedChunkFolders = (0x00..0xff).map { Integer.toHexString(it).padStart(2, '0') }.toHashSet() val duration = measure { - for (file in root.listFilesBlocking(context)) { + for (file in folder.listFilesBlocking(context)) { val name = file.name ?: continue if (chunkFolderRegex.matches(name)) { chunkFolders[name] = file @@ -70,8 +99,8 @@ public abstract class SafStoragePlugin( fileOp?.invoke(file, name) } } - Log.e(TAG, "Retrieving chunk folders took $duration") - createMissingChunkFolders(root, expectedChunkFolders) + Log.i(TAG, "Retrieving chunk folders took $duration") + createMissingChunkFolders(folder, chunkFolders, expectedChunkFolders) } @Throws(IOException::class) @@ -89,7 +118,11 @@ public abstract class SafStoragePlugin( } @Throws(IOException::class) - private fun createMissingChunkFolders(root: DocumentFile, expectedChunkFolders: Set) { + private fun createMissingChunkFolders( + root: DocumentFile, + chunkFolders: HashMap, + expectedChunkFolders: Set + ) { val s = expectedChunkFolders.size val duration = measure { for ((i, chunkFolderName) in expectedChunkFolders.withIndex()) { @@ -101,13 +134,14 @@ public abstract class SafStoragePlugin( throw IOException("Only have ${chunkFolders.size} chunk folders.") } } - if (s > 0) Log.e(TAG, "Creating $s missing chunk folders took $duration") + if (s > 0) Log.i(TAG, "Creating $s missing chunk folders took $duration") } @Throws(IOException::class) override fun getChunkOutputStream(chunkId: String): OutputStream { val chunkFolderName = chunkId.substring(0, 2) - val chunkFolder = chunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId") + val chunkFolder = + cache.backupChunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId") // TODO should we check if it exists first? val chunkFile = chunkFolder.createFileOrThrow(chunkId, MIME_TYPE) return chunkFile.getOutputStream(context.contentResolver) @@ -115,48 +149,58 @@ public abstract class SafStoragePlugin( @Throws(IOException::class) override fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream { - val root = root ?: throw IOException() + val folder = folder ?: throw IOException() val name = timestampToSnapshot(timestamp) // TODO should we check if it exists first? - val snapshotFile = root.createFileOrThrow(name, MIME_TYPE) + val snapshotFile = folder.createFileOrThrow(name, MIME_TYPE) return snapshotFile.getOutputStream(contentResolver) } /************************* Restore *******************************/ @Throws(IOException::class) - override suspend fun getAvailableBackupSnapshots(): List { - val root = root ?: return emptyList() - val snapshots = ArrayList() + override suspend fun getBackupSnapshotsForRestore(): List { + val snapshots = ArrayList() - populateChunkFolders(root) { file, name -> - val match = snapshotRegex.matchEntire(name) - if (match != null) { - val timestamp = match.groupValues[1].toLong() - snapshots.add(timestamp) - snapshotFiles[timestamp] = file + root?.listFilesBlocking(context)?.forEach { folder -> + val folderName = folder.name ?: "" + if (!folderRegex.matches(folderName)) return@forEach + + Log.i(TAG, "Checking $folderName for snapshots...") + for (file in folder.listFilesBlocking(context)) { + val name = file.name ?: continue + val match = snapshotRegex.matchEntire(name) + if (match != null) { + val timestamp = match.groupValues[1].toLong() + val storedSnapshot = StoredSnapshot(folderName, timestamp) + snapshots.add(storedSnapshot) + cache.snapshotFiles[storedSnapshot] = file + } } } - Log.e(TAG, "Got ${snapshots.size} snapshots while populating chunk folders") + Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders") return snapshots } @Throws(IOException::class) - override suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream { - val snapshotFile = snapshotFiles.getOrElse(timestamp) { - root?.findFileBlocking(context, timestampToSnapshot(timestamp)) + override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream { + val timestamp = storedSnapshot.timestamp + val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) { + getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp)) } ?: throw IOException("Could not get file for snapshot $timestamp") return snapshotFile.getInputStream(contentResolver) } @Throws(IOException::class) - override suspend fun getChunkInputStream(chunkId: String): InputStream { - if (chunkFolders.size < CHUNK_FOLDER_COUNT) { - val root = root ?: throw IOException("Could not get root") - populateChunkFolders(root) + override suspend fun getChunkInputStream( + snapshot: StoredSnapshot, + chunkId: String + ): InputStream { + if (cache.restoreChunkFolders.size < CHUNK_FOLDER_COUNT) { + populateChunkFolders(getFolder(snapshot), cache.restoreChunkFolders) } val chunkFolderName = chunkId.substring(0, 2) - val chunkFolder = chunkFolders[chunkFolderName] + val chunkFolder = cache.restoreChunkFolders[chunkFolderName] ?: throw IOException("No folder for chunk $chunkId") val chunkFile = chunkFolder.findFileBlocking(context, chunkId) ?: throw IOException("No chunk $chunkId") @@ -164,23 +208,56 @@ public abstract class SafStoragePlugin( } @Throws(IOException::class) - override suspend fun deleteBackupSnapshot(timestamp: Long) { - Log.d(TAG, "Deleting snapshot $timestamp") - val snapshotFile = snapshotFiles.getOrElse(timestamp) { - root?.findFileBlocking(context, timestampToSnapshot(timestamp)) - } ?: throw IOException("Could not get file for snapshot $timestamp") - if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp") + private suspend fun getFolder(storedSnapshot: StoredSnapshot): DocumentFile { + // not cached, because used in several places only once and + // [getBackupSnapshotInputStream] uses snapshot files cache and + // [getChunkInputStream] uses restore chunk folders cache + return root?.findFileBlocking(context, storedSnapshot.userId) + ?: throw IOException("Could not find snapshot $storedSnapshot") } + /************************* Pruning *******************************/ + + @Throws(IOException::class) + override suspend fun getCurrentBackupSnapshots(): List { + val folder = folder ?: return emptyList() + val folderName = folder.name ?: error("Folder suddenly has no more name") + val snapshots = ArrayList() + + populateChunkFolders(folder, cache.backupChunkFolders) { file, name -> + val match = snapshotRegex.matchEntire(name) + if (match != null) { + val timestamp = match.groupValues[1].toLong() + val storedSnapshot = StoredSnapshot(folderName, timestamp) + snapshots.add(storedSnapshot) + cache.snapshotFiles[storedSnapshot] = file + } + } + Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders") + return snapshots + } + + @Throws(IOException::class) + override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) { + val timestamp = storedSnapshot.timestamp + Log.d(TAG, "Deleting snapshot $timestamp") + val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) { + getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp)) + } ?: throw IOException("Could not get file for snapshot $timestamp") + if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp") + cache.snapshotFiles.remove(storedSnapshot) + } + + @Throws(IOException::class) override suspend fun deleteChunks(chunkIds: List) { - if (chunkFolders.size < CHUNK_FOLDER_COUNT) { - val root = root ?: throw IOException("Could not get root") - populateChunkFolders(root) + if (cache.backupChunkFolders.size < CHUNK_FOLDER_COUNT) { + val folder = folder ?: throw IOException("Could not get current folder in root") + populateChunkFolders(folder, cache.backupChunkFolders) } for (chunkId in chunkIds) { Log.d(TAG, "Deleting chunk $chunkId") val chunkFolderName = chunkId.substring(0, 2) - val chunkFolder = chunkFolders[chunkFolderName] + val chunkFolder = cache.backupChunkFolders[chunkFolderName] ?: throw IOException("No folder for chunk $chunkId") val chunkFile = chunkFolder.findFileBlocking(context, chunkId) if (chunkFile == null) { diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt index 23082746..9426cf5e 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt @@ -3,6 +3,7 @@ package org.calyxos.backup.storage.prune import android.util.Log import org.calyxos.backup.storage.api.BackupObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.measure @@ -33,15 +34,15 @@ internal class Pruner( @Throws(IOException::class) suspend fun prune(backupObserver: BackupObserver?) { val duration = measure { - val timestamps = storagePlugin.getAvailableBackupSnapshots() - val toDelete = retentionManager.getSnapshotsToDelete(timestamps) - backupObserver?.onPruneStart(toDelete) - for (timestamp in toDelete) { + val storedSnapshots = storagePlugin.getCurrentBackupSnapshots() + val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots) + backupObserver?.onPruneStart(toDelete.map { it.timestamp }) + for (snapshot in toDelete) { try { - pruneSnapshot(timestamp, backupObserver) + pruneSnapshot(snapshot, backupObserver) } catch (e: Exception) { - Log.e(TAG, "Error pruning $timestamp", e) - backupObserver?.onPruneError(timestamp, e) + Log.e(TAG, "Error pruning $snapshot", e) + backupObserver?.onPruneError(snapshot.timestamp, e) } } } @@ -50,12 +51,15 @@ internal class Pruner( } @Throws(IOException::class, GeneralSecurityException::class) - private suspend fun pruneSnapshot(timestamp: Long, backupObserver: BackupObserver?) { - val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp) + private suspend fun pruneSnapshot( + storedSnapshot: StoredSnapshot, + backupObserver: BackupObserver? + ) { + val snapshot = snapshotRetriever.getSnapshot(streamKey, storedSnapshot) val chunks = HashSet() snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) } snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) } - storagePlugin.deleteBackupSnapshot(timestamp) + storagePlugin.deleteBackupSnapshot(storedSnapshot) db.applyInParts(chunks) { chunksCache.decrementRefCount(it) } @@ -68,7 +72,7 @@ internal class Pruner( size += it.size it.id } - backupObserver?.onPruneSnapshot(timestamp, chunkIdsToDelete.size, size) + backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size) storagePlugin.deleteChunks(chunkIdsToDelete) chunksCache.deleteChunks(cachedChunksToDelete) } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt index b1efeb02..8f2097ef 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt @@ -2,6 +2,7 @@ package org.calyxos.backup.storage.prune import android.content.Context import org.calyxos.backup.storage.api.SnapshotRetention +import org.calyxos.backup.storage.api.StoredSnapshot import java.io.IOException import java.time.LocalDate import java.time.temporal.ChronoField @@ -48,37 +49,37 @@ internal class RetentionManager(private val context: Context) { } /** - * Takes a list of snapshot timestamps and returns a list of those + * Takes a list of [StoredSnapshot]s and returns a list of those * that can be deleted according to the current snapshot retention policy. */ @Throws(IOException::class) - fun getSnapshotsToDelete(snapshotTimestamps: List): List { + fun getSnapshotsToDelete(storedSnapshots: List): List { val retention = getSnapshotRetention() - val dates = snapshotTimestamps.sortedDescending().map { - Pair(it, LocalDate.ofEpochDay(it / 1000 / 60 / 60 / 24)) + val datePairs = storedSnapshots.sortedByDescending { it.timestamp }.map { s -> + Pair(s, LocalDate.ofEpochDay(s.timestamp / 1000 / 60 / 60 / 24)) } - val toKeep = HashSet() - toKeep += getToKeep(dates, retention.daily) - toKeep += getToKeep(dates, retention.weekly) { temporal: Temporal -> + val toKeep = HashSet() + toKeep += getToKeep(datePairs, retention.daily) + toKeep += getToKeep(datePairs, retention.weekly) { temporal: Temporal -> temporal.with(ChronoField.DAY_OF_WEEK, 1) } - toKeep += getToKeep(dates, retention.monthly, firstDayOfMonth()) - toKeep += getToKeep(dates, retention.yearly, firstDayOfYear()) - return snapshotTimestamps - toKeep + toKeep += getToKeep(datePairs, retention.monthly, firstDayOfMonth()) + toKeep += getToKeep(datePairs, retention.yearly, firstDayOfYear()) + return storedSnapshots - toKeep } private fun getToKeep( - pairs: List>, + pairs: List>, keep: Int, temporalAdjuster: TemporalAdjuster? = null, - ): List { - val toKeep = ArrayList() + ): List { + val toKeep = ArrayList() if (keep == 0) return toKeep var last: LocalDate? = null - for ((timestamp, date) in pairs) { + for ((snapshot, date) in pairs) { val period = if (temporalAdjuster == null) date else date.with(temporalAdjuster) if (period != last) { - toKeep.add(timestamp) + toKeep.add(snapshot) if (toKeep.size >= keep) break last = period } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt index f7111c2e..6b22c5a8 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt @@ -2,6 +2,7 @@ package org.calyxos.backup.storage.restore import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import java.io.IOException import java.io.InputStream @@ -19,10 +20,11 @@ internal abstract class AbstractChunkRestore( @Throws(IOException::class, GeneralSecurityException::class) protected suspend fun getAndDecryptChunk( version: Int, + storedSnapshot: StoredSnapshot, chunkId: String, streamReader: suspend (InputStream) -> Unit, ) { - storagePlugin.getChunkInputStream(chunkId).use { inputStream -> + storagePlugin.getChunkInputStream(storedSnapshot, chunkId).use { inputStream -> inputStream.readVersion(version) val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte()) streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream -> diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt index 83b7f2cc..372f91d7 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt @@ -4,6 +4,7 @@ import android.content.Context import android.util.Log import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import java.io.File import java.io.FileInputStream @@ -26,6 +27,7 @@ internal class MultiChunkRestore( suspend fun restore( version: Int, + storedSnapshot: StoredSnapshot, chunkMap: Map, files: Collection, observer: RestoreObserver?, @@ -34,7 +36,7 @@ internal class MultiChunkRestore( files.forEach { file -> try { restoreFile(file, observer, "L") { outputStream -> - writeChunks(version, file, chunkMap, outputStream) + writeChunks(version, storedSnapshot, file, chunkMap, outputStream) } restoredFiles++ } catch (e: Exception) { @@ -49,6 +51,7 @@ internal class MultiChunkRestore( @Throws(IOException::class, GeneralSecurityException::class) private suspend fun writeChunks( version: Int, + storedSnapshot: StoredSnapshot, file: RestorableFile, chunkMap: Map, outputStream: OutputStream, @@ -61,8 +64,11 @@ internal class MultiChunkRestore( bytes += decryptedStream.copyTo(outputStream) } val isCached = isCached(chunkId) - if (isCached || otherFiles.size > 1) getAndCacheChunk(version, chunkId, chunkWriter) - else getAndDecryptChunk(version, chunkId, chunkWriter) + if (isCached || otherFiles.size > 1) { + getAndCacheChunk(version, storedSnapshot, chunkId, chunkWriter) + } else { + getAndDecryptChunk(version, storedSnapshot, chunkId, chunkWriter) + } otherFiles.remove(file) if (isCached && otherFiles.isEmpty()) removeCachedChunk(chunkId) @@ -77,13 +83,14 @@ internal class MultiChunkRestore( @Throws(IOException::class, GeneralSecurityException::class) private suspend fun getAndCacheChunk( version: Int, + storedSnapshot: StoredSnapshot, chunkId: String, streamReader: suspend (InputStream) -> Unit, ) { val file = getChunkCacheFile(chunkId) if (!file.isFile) { FileOutputStream(file).use { outputStream -> - getAndDecryptChunk(version, chunkId) { decryptedStream -> + getAndDecryptChunk(version, storedSnapshot, chunkId) { decryptedStream -> decryptedStream.copyTo(outputStream) } } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt index 70fa18ed..819058d3 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt @@ -8,6 +8,7 @@ import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.SnapshotItem import org.calyxos.backup.storage.api.SnapshotResult import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto @@ -46,8 +47,10 @@ internal class Restore( val numSnapshots: Int val time = measure { val list = try { - storagePlugin.getAvailableBackupSnapshots().sortedDescending().map { - SnapshotItem(it, null) + storagePlugin.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot -> + storedSnapshot.timestamp + }.map { storedSnapshot -> + SnapshotItem(storedSnapshot, null) }.toMutableList() } catch (e: Exception) { Log.e("TAG", "Error retrieving snapshots", e) @@ -61,7 +64,9 @@ internal class Restore( while (iterator.hasNext()) { val oldItem = iterator.next() val item = try { - oldItem.copy(snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.time)) + oldItem.copy( + snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.storedSnapshot) + ) } catch (e: Exception) { Log.e("TAG", "Error retrieving snapshot ${oldItem.time}", e) continue @@ -73,15 +78,15 @@ internal class Restore( Log.e(TAG, "Decrypting and parsing $numSnapshots snapshots took $time") } - @Throws(IOException::class, GeneralSecurityException::class) - suspend fun restoreBackupSnapshot(timestamp: Long, observer: RestoreObserver?) { - val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp) - restoreBackupSnapshot(snapshot, observer) - } - @OptIn(ExperimentalTime::class) - @Throws(IOException::class) - suspend fun restoreBackupSnapshot(snapshot: BackupSnapshot, observer: RestoreObserver?) { + @Throws(IOException::class, GeneralSecurityException::class) + suspend fun restoreBackupSnapshot( + storedSnapshot: StoredSnapshot, + optionalSnapshot: BackupSnapshot? = null, + observer: RestoreObserver? = null, + ) { + val snapshot = optionalSnapshot ?: snapshotRetriever.getSnapshot(streamKey, storedSnapshot) + val filesTotal = snapshot.mediaFilesList.size + snapshot.documentFilesList.size val totalSize = snapshot.mediaFilesList.sumOf { it.size } + snapshot.documentFilesList.sumOf { it.size } @@ -91,19 +96,30 @@ internal class Restore( val version = snapshot.version var restoredFiles = 0 val smallFilesDuration = measure { - restoredFiles += zipChunkRestore.restore(version, split.zipChunks, observer) + restoredFiles += zipChunkRestore.restore( + version, + storedSnapshot, + split.zipChunks, + observer, + ) } Log.e(TAG, "Restoring ${split.zipChunks.size} zip chunks took $smallFilesDuration.") val singleChunkDuration = measure { - restoredFiles += singleChunkRestore.restore(version, split.singleChunks, observer) + restoredFiles += singleChunkRestore.restore( + version, + storedSnapshot, + split.singleChunks, + observer, + ) } Log.e(TAG, "Restoring ${split.singleChunks.size} single chunks took $singleChunkDuration.") val multiChunkDuration = measure { restoredFiles += multiChunkRestore.restore( version, + storedSnapshot, split.multiChunkMap, split.multiChunkFiles, - observer + observer, ) } Log.e(TAG, "Restoring ${split.multiChunkFiles.size} multi chunks took $multiChunkDuration.") diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt index 568864ae..1917e420 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt @@ -8,20 +8,24 @@ import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StorageBackup -import org.calyxos.backup.storage.backup.BackupSnapshot +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START +import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID import org.calyxos.backup.storage.ui.NOTIFICATION_ID_RESTORE import org.calyxos.backup.storage.ui.Notifications /** * Start to trigger restore as a foreground service. Ensure that you provide the snapshot - * to be restored with [Intent.putExtra] as a [Long] in [EXTRA_TIMESTAMP_START]. - * See [BackupSnapshot.getTimeStart]. + * to be restored with [Intent.putExtra]: + * * the user ID of the snapshot as a [String] in [EXTRA_USER_ID] + * * the snapshot's timestamp as a [Long] in [EXTRA_TIMESTAMP_START]. + * See [BackupSnapshot.getTimeStart]. */ public abstract class RestoreService : Service() { public companion object { private const val TAG = "RestoreService" + public const val EXTRA_USER_ID: String = "userId" public const val EXTRA_TIMESTAMP_START: String = "timestamp" } @@ -31,13 +35,15 @@ public abstract class RestoreService : Service() { override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Log.d(TAG, "onStartCommand $intent $flags $startId") - val timestamp = intent?.getLongExtra(EXTRA_TIMESTAMP_START, -1) - if (timestamp == null || timestamp < 0) error("No timestamp in intent: $intent") + val userId = intent?.getStringExtra(EXTRA_USER_ID) ?: error("No user ID in intent: $intent") + val timestamp = intent.getLongExtra(EXTRA_TIMESTAMP_START, -1) + if (timestamp < 0) error("No timestamp in intent: $intent") + val storedSnapshot = StoredSnapshot(userId, timestamp) startForeground(NOTIFICATION_ID_RESTORE, n.getRestoreNotification()) GlobalScope.launch { // TODO offer a way to try again if failed, or do an automatic retry here - storageBackup.restoreBackupSnapshot(timestamp, restoreObserver) + storageBackup.restoreBackupSnapshot(storedSnapshot, null, restoreObserver) stopSelf(startId) } return START_STICKY_COMPATIBILITY diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt index 0d04ccf6..f6e23806 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt @@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore import android.util.Log import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto private const val TAG = "SingleChunkRestore" @@ -17,6 +18,7 @@ internal class SingleChunkRestore( suspend fun restore( version: Int, + storedSnapshot: StoredSnapshot, chunks: Collection, observer: RestoreObserver? ): Int { @@ -25,7 +27,7 @@ internal class SingleChunkRestore( check(chunk.files.size == 1) val file = chunk.files[0] try { - getAndDecryptChunk(version, chunk.chunkId) { decryptedStream -> + getAndDecryptChunk(version, storedSnapshot, chunk.chunkId) { decryptedStream -> restoreFile(file, observer, "M") { outputStream -> decryptedStream.copyTo(outputStream) } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt index e27982bd..b937bc85 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt @@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore import android.util.Log import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.crypto.StreamCrypto import java.io.IOException import java.io.InputStream @@ -24,13 +25,14 @@ internal class ZipChunkRestore( */ suspend fun restore( version: Int, + storedSnapshot: StoredSnapshot, zipChunks: Collection, observer: RestoreObserver? ): Int { var restoredFiles = 0 zipChunks.forEach { zipChunk -> try { - getAndDecryptChunk(version, zipChunk.chunkId) { decryptedStream -> + getAndDecryptChunk(version, storedSnapshot, zipChunk.chunkId) { decryptedStream -> restoredFiles += restoreZipChunk(zipChunk, decryptedStream, observer) } } catch (e: Exception) { diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt index 9597f5cd..78ef34a6 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt @@ -19,6 +19,7 @@ import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.SnapshotResult import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX import org.calyxos.backup.storage.backup.Backup.Companion.SMALL_FILE_SIZE_MAX @@ -173,14 +174,16 @@ internal class BackupRestoreTest { // RESTORE + val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured) + val smallFileMOutputStream = ByteArrayOutputStream() val smallFileDOutputStream = ByteArrayOutputStream() val fileMOutputStream = ByteArrayOutputStream() val fileDOutputStream = ByteArrayOutputStream() - coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured) + coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot) coEvery { - plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured) + plugin.getBackupSnapshotInputStream(storedSnapshot) } returns ByteArrayInputStream(snapshotOutputStream.toByteArray()) // retrieve snapshots @@ -197,14 +200,14 @@ internal class BackupRestoreTest { // pipe chunks back in coEvery { - plugin.getChunkInputStream(cachedFiles[0].chunks[0]) + plugin.getChunkInputStream(storedSnapshot, cachedFiles[0].chunks[0]) } returns ByteArrayInputStream(zipChunkOutputStream.toByteArray()) // cachedFiles[0].chunks[1] is in previous zipChunk coEvery { - plugin.getChunkInputStream(cachedFiles[2].chunks[0]) + plugin.getChunkInputStream(storedSnapshot, cachedFiles[2].chunks[0]) } returns ByteArrayInputStream(mOutputStream.toByteArray()) coEvery { - plugin.getChunkInputStream(cachedFiles[3].chunks[0]) + plugin.getChunkInputStream(storedSnapshot, cachedFiles[3].chunks[0]) } returns ByteArrayInputStream(dOutputStream.toByteArray()) // provide file output streams for restore @@ -217,7 +220,7 @@ internal class BackupRestoreTest { val fileDRestorable = getRestorableFileD(fileD, snapshot) expectRestoreFile(fileDRestorable, fileDOutputStream) - restore.restoreBackupSnapshot(snapshot, null) + restore.restoreBackupSnapshot(storedSnapshot, snapshot, null) // restored files match backed up files exactly assertArrayEquals(smallFileMBytes, smallFileMOutputStream.toByteArray()) @@ -337,9 +340,11 @@ internal class BackupRestoreTest { // RESTORE - coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured) + val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured) + + coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot) coEvery { - plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured) + plugin.getBackupSnapshotInputStream(storedSnapshot) } returns ByteArrayInputStream(snapshotOutputStream.toByteArray()) // retrieve snapshots @@ -354,19 +359,27 @@ internal class BackupRestoreTest { // pipe chunks back in coEvery { plugin.getChunkInputStream( - "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3") + storedSnapshot, + "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" + ) } returns ByteArrayInputStream(id040f32.toByteArray()) coEvery { plugin.getChunkInputStream( - "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29") + storedSnapshot, + "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" + ) } returns ByteArrayInputStream(id901fbc.toByteArray()) coEvery { plugin.getChunkInputStream( - "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d") + storedSnapshot, + "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" + ) } returns ByteArrayInputStream(id5adea3.toByteArray()) coEvery { plugin.getChunkInputStream( - "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67") + storedSnapshot, + "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" + ) } returns ByteArrayInputStream(id40d00c.toByteArray()) // provide file output streams for restore @@ -375,7 +388,7 @@ internal class BackupRestoreTest { val file2Restorable = getRestorableFileD(file2, snapshot) expectRestoreFile(file2Restorable, file2OutputStream) - restore.restoreBackupSnapshot(snapshot, null) + restore.restoreBackupSnapshot(storedSnapshot, snapshot, null) // restored files match backed up files exactly assertArrayEquals(file1Bytes, file1OutputStream.toByteArray()) @@ -384,13 +397,21 @@ internal class BackupRestoreTest { // chunks were only read from storage once coVerify(exactly = 1) { plugin.getChunkInputStream( - "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3") + storedSnapshot, + "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" + ) plugin.getChunkInputStream( - "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29") + storedSnapshot, + "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" + ) plugin.getChunkInputStream( - "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d") + storedSnapshot, + "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" + ) plugin.getChunkInputStream( - "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67") + storedSnapshot, + "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" + ) } } diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt index 64fbac94..b1fb556a 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt @@ -9,6 +9,7 @@ import io.mockk.mockk import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.db.CachedChunk import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.db.Db @@ -56,7 +57,9 @@ internal class ChunksCacheRepopulaterTest { .addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4)) .addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk5)) .build() - val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart) + val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart) + val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart) + val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2) val cachedChunks = listOf( CachedChunk(chunk1, 2, 0), CachedChunk(chunk2, 2, 0), @@ -64,12 +67,12 @@ internal class ChunksCacheRepopulaterTest { ) // chunk3 is not referenced and should get deleted val cachedChunksSlot = slot>() - coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps + coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots coEvery { - snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart) + snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1 coEvery { - snapshotRetriever.getSnapshot(streamKey, snapshot2.timeStart) + snapshotRetriever.getSnapshot(streamKey, storedSnapshot2) } returns snapshot2 every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt index 56cb71fb..3c359cf6 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt @@ -8,6 +8,7 @@ import io.mockk.mockk import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.StoragePlugin +import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.backup.BackupDocumentFile import org.calyxos.backup.storage.backup.BackupMediaFile import org.calyxos.backup.storage.backup.BackupSnapshot @@ -66,18 +67,20 @@ internal class PrunerTest { .addMediaFiles(BackupMediaFile.newBuilder().addChunkIds(chunk2)) .addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4)) .build() - val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart) + val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart) + val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart) + val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2) val expectedChunks = listOf(chunk1, chunk2, chunk3) val actualChunks = slot>() val actualChunks2 = slot>() val cachedChunk3 = CachedChunk(chunk3, 0, 0) - coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps + coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots every { - retentionManager.getSnapshotsToDelete(snapshotTimestamps) - } returns listOf(snapshot1.timeStart) - coEvery { snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart) } returns snapshot1 - coEvery { plugin.deleteBackupSnapshot(snapshot1.timeStart) } just Runs + retentionManager.getSnapshotsToDelete(storedSnapshots) + } returns listOf(storedSnapshot1) + coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1 + coEvery { plugin.deleteBackupSnapshot(storedSnapshot1) } just Runs every { db.applyInParts(capture(actualChunks), captureLambda()) } answers { diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt index 784013cb..7c19d7e5 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt @@ -5,6 +5,8 @@ import android.content.SharedPreferences import io.mockk.every import io.mockk.mockk import org.calyxos.backup.storage.api.SnapshotRetention +import org.calyxos.backup.storage.api.StoredSnapshot +import org.calyxos.backup.storage.getRandomString import org.junit.Assert.assertEquals import org.junit.Test import java.time.LocalDateTime @@ -17,19 +19,21 @@ internal class RetentionManagerTest { private val retention = RetentionManager(context) + private val userId = getRandomString() + @Test fun testDailyRetention() { expectGetRetention(SnapshotRetention(2, 0, 0, 0)) - val timestamps = listOf( + val storedSnapshots = listOf( // 1577919600000 LocalDateTime.of(2020, 1, 1, 23, 0).toMillis(), // 1577872800000 LocalDateTime.of(2020, 1, 1, 10, 0).toMillis(), // 1583276400000 LocalDateTime.of(2020, 3, 3, 23, 0).toMillis(), - ) - val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1577872800000), toDelete) + ).map { StoredSnapshot(userId, it) } + val toDelete = retention.getSnapshotsToDelete(storedSnapshots) + assertEquals(listOf(1577872800000), toDelete.map { it.timestamp }) } @Test @@ -45,9 +49,9 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(), // 1608678000000 LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1608544800000, 1608638400000), toDelete) + assertEquals(listOf(1608544800000, 1608638400000), toDelete.map { it.timestamp }) } @Test @@ -63,9 +67,9 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(), // 1608678000000 LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1580857200000), toDelete) + assertEquals(listOf(1580857200000), toDelete.map { it.timestamp }) } @Test @@ -83,10 +87,10 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(), // 1608678000000 LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } // keeps only the latest one for each year, so three in total, even though keep is four val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1549321200000, 1608544800000), toDelete) + assertEquals(listOf(1549321200000, 1608544800000), toDelete.map { it.timestamp }) } @Test @@ -116,9 +120,11 @@ internal class RetentionManagerTest { LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(), // 1608638400000 LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(), - ) + ).map { StoredSnapshot(userId, it) } val toDelete = retention.getSnapshotsToDelete(timestamps) - assertEquals(listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000), toDelete) + assertEquals( + listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000), + toDelete.map { it.timestamp }) } private fun expectGetRetention(snapshotRetention: SnapshotRetention) {