mobi/seedvault
1
0
Fork 0
Code Pull requests Activity

Allow using the same storage location on different devices or user profiles

Browse source 
Previously, we would put our files directly in the root of the storage location and delete any existing backups there. When used by different devices or user profiles, these would keep deleting each other's backups.
This commit is contained in:
Torsten Grote 2021-06-18 17:55:27 -03:00 committed by Chirayu Desai
parent a762d1b64e
commit 347d2a316f
22 changed files with 375 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/src/main/java/com/stevesoltys/seedvault/restore/RestoreViewModel.kt
View file

@ -62,6 +62,7 @@ import kotlinx.coroutines.launch
import org.calyxos.backup.storage.api.SnapshotItem
import org.calyxos.backup.storage.api.StorageBackup
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID
import org.calyxos.backup.storage.ui.restore.SnapshotViewModel
import java.util.LinkedList
import kotlin.coroutines.Continuation
@ -392,6 +393,7 @@ internal class RestoreViewModel(
@UiThread
internal fun startFilesRestore(item: SnapshotItem) {
val i = Intent(app, StorageRestoreService::class.java)
i.putExtra(EXTRA_USER_ID, item.storedSnapshot.userId)
i.putExtra(EXTRA_TIMESTAMP_START, item.time)
app.startForegroundService(i)
mDisplayFragment.setEvent(RESTORE_FILES_STARTED)

3
storage/demo/src/main/java/de/grobox/storagebackuptester/MainViewModel.kt
View file

@ -123,6 +123,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati
// example for how to do restore via foreground service
// app.startForegroundService(Intent(app, DemoRestoreService::class.java).apply {
// putExtra(EXTRA_USER_ID, item.storedSnapshot.userId)
// putExtra(EXTRA_TIMESTAMP_START, snapshot.timeStart)
// })
@ -130,7 +131,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati
_restoreProgressVisible.value = true
val restoreObserver = RestoreStats(app, _restoreLog)
viewModelScope.launch {
storageBackup.restoreBackupSnapshot(snapshot, restoreObserver)
storageBackup.restoreBackupSnapshot(item.storedSnapshot, snapshot, restoreObserver)
_restoreProgressVisible.value = false
}
}

1
storage/doc/design.md
View file

@ -388,7 +388,6 @@ but are considered out-of-scope of the current design for time and budget reason
* using a rolling hash to produce chunks in order to increase likelihood of obtaining same chunks
even if file contents change slightly or shift
* external secret-less corruption checks that would use checksums over encrypted data
* supporting different backup clients backing up to the same storage
* concealing file sizes (though zip chunks helps a bit here)
* implementing different storage plugins

17
storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt
View file

@ -3,15 +3,28 @@ package org.calyxos.backup.storage.api
import org.calyxos.backup.storage.backup.BackupSnapshot
public data class SnapshotItem(
public val time: Long,
public val storedSnapshot: StoredSnapshot,
public val snapshot: BackupSnapshot?,
)
) {
val time: Long get() = storedSnapshot.timestamp
}
public sealed class SnapshotResult {
public data class Success(val snapshots: List<SnapshotItem>) : SnapshotResult()
public data class Error(val e: Exception) : SnapshotResult()
}
public data class StoredSnapshot(
/**
* The unique ID of the current device/user combination chosen by the [StoragePlugin].
*/
public val userId: String,
/**
* The timestamp identifying a snapshot of the [userId].
*/
public val timestamp: Long,
)
/**
* Defines which backup snapshots should be retained when pruning backups.
*

24
storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt
View file

@ -103,10 +103,12 @@ public class StorageBackup(
* Run this on a new storage location to ensure that there are no old snapshots
* (potentially encrypted with an old key) laying around.
* Using a storage location with existing data is not supported.
* Using the same root folder for storage on different devices or user profiles is fine though
* as the [StoragePlugin] should isolate storage per [StoredSnapshot.userId].
*/
public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) {
try {
plugin.getAvailableBackupSnapshots().forEach {
plugin.getCurrentBackupSnapshots().forEach {
try {
plugin.deleteBackupSnapshot(it)
} catch (e: IOException) {
@ -183,15 +185,16 @@ public class StorageBackup(
}
public suspend fun restoreBackupSnapshot(
snapshot: BackupSnapshot,
restoreObserver: RestoreObserver? = null
storedSnapshot: StoredSnapshot,
snapshot: BackupSnapshot? = null,
restoreObserver: RestoreObserver? = null,
): Boolean = withContext(dispatcher) {
if (restoreRunning.getAndSet(true)) {
Log.w(TAG, "Restore already running, not starting a new one")
return@withContext false
}
try {
restore.restoreBackupSnapshot(snapshot, restoreObserver)
restore.restoreBackupSnapshot(storedSnapshot, snapshot, restoreObserver)
true
} catch (e: Exception) {
Log.e(TAG, "Error during restore", e)
@ -201,17 +204,4 @@ public class StorageBackup(
}
}
public suspend fun restoreBackupSnapshot(
timestamp: Long,
restoreObserver: RestoreObserver? = null
): Boolean = withContext(dispatcher) {
try {
restore.restoreBackupSnapshot(timestamp, restoreObserver)
true
} catch (e: Exception) {
Log.e(TAG, "Error during restore", e)
false
}
}
}

24
storage/lib/src/main/java/org/calyxos/backup/storage/api/StoragePlugin.kt
View file

@ -35,22 +35,36 @@ public interface StoragePlugin {
/* Restore */
/**
* Returns the timestamps representing a backup snapshot that are available on storage.
* Returns *all* [StoredSnapshot]s that are available on storage
* independent of user ID and whether they can be decrypted
* with the key returned by [getMasterKey].
*/
@Throws(IOException::class)
public suspend fun getAvailableBackupSnapshots(): List<Long>
public suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot>
@Throws(IOException::class)
public suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream
public suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream
@Throws(IOException::class)
public suspend fun getChunkInputStream(chunkId: String): InputStream
public suspend fun getChunkInputStream(snapshot: StoredSnapshot, chunkId: String): InputStream
/* Pruning */
/**
* Returns [StoredSnapshot]s for the currently active user ID.
*/
@Throws(IOException::class)
public suspend fun deleteBackupSnapshot(timestamp: Long)
public suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot>
/**
* Deletes the given [StoredSnapshot].
*/
@Throws(IOException::class)
public suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot)
/**
* Deletes the given chunks of the *current* user ID only.
*/
@Throws(IOException::class)
public suspend fun deleteChunks(chunkIds: List<String>)

10
storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt
View file

@ -7,6 +7,7 @@ import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import java.io.IOException
import java.security.GeneralSecurityException
import java.util.concurrent.TimeUnit.MILLISECONDS
import kotlin.time.ExperimentalTime
import kotlin.time.toDuration
@ -37,8 +38,13 @@ internal class ChunksCacheRepopulater(
availableChunkIds: HashSet<String>
) {
val start = System.currentTimeMillis()
val snapshots = storagePlugin.getAvailableBackupSnapshots().map { timestamp ->
snapshotRetriever.getSnapshot(streamKey, timestamp)
val snapshots = storagePlugin.getCurrentBackupSnapshots().mapNotNull { storedSnapshot ->
try {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
} catch (e: GeneralSecurityException) {
Log.w(TAG, "Error fetching snapshot $storedSnapshot", e)
null
}
}
val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS)
Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration")

6
storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt
View file

@ -2,6 +2,7 @@ package org.calyxos.backup.storage.plugin
import com.google.protobuf.InvalidProtocolBufferException
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.restore.readVersion
@ -19,9 +20,10 @@ internal class SnapshotRetriever(
GeneralSecurityException::class,
InvalidProtocolBufferException::class,
)
suspend fun getSnapshot(streamKey: ByteArray, timestamp: Long): BackupSnapshot {
return storagePlugin.getBackupSnapshotInputStream(timestamp).use { inputStream ->
suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot {
return storagePlugin.getBackupSnapshotInputStream(storedSnapshot).use { inputStream ->
val version = inputStream.readVersion()
val timestamp = storedSnapshot.timestamp
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
BackupSnapshot.parseFrom(decryptedStream)

32
storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafCache.kt Normal file
View file

@ -0,0 +1,32 @@
package org.calyxos.backup.storage.plugin.saf
import androidx.documentfile.provider.DocumentFile
import org.calyxos.backup.storage.api.StoredSnapshot
/**
* Accessing files and attributes via SAF is costly.
* This class caches them to speed up SAF related operations.
*/
internal class SafCache {
/**
* The folder for the current user ID (here "${ANDROID_ID}.sv").
*/
var currentFolder: DocumentFile? = null
/**
* Folders containing chunks for backups of the current user ID.
*/
val backupChunkFolders = HashMap<String, DocumentFile>(CHUNK_FOLDER_COUNT)
/**
* Folders containing chunks for restore of a chosen [StoredSnapshot].
*/
val restoreChunkFolders = HashMap<String, DocumentFile>(CHUNK_FOLDER_COUNT)
/**
* Files for each [StoredSnapshot].
*/
val snapshotFiles = HashMap<StoredSnapshot, DocumentFile>()
}

165
storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt
View file

@ -1,9 +1,13 @@
package org.calyxos.backup.storage.plugin.saf
import android.annotation.SuppressLint
import android.content.Context
import android.provider.Settings
import android.provider.Settings.Secure.ANDROID_ID
import android.util.Log
import androidx.documentfile.provider.DocumentFile
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createDirectoryOrThrow
import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createFileOrThrow
@ -15,11 +19,12 @@ import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
private val folderRegex = Regex("^[a-f0-9]{16}\\.sv$")
private val chunkFolderRegex = Regex("[a-f0-9]{2}")
private val chunkRegex = Regex("[a-f0-9]{64}")
private val snapshotRegex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
private const val CHUNK_FOLDER_COUNT = 256
private const val MIME_TYPE: String = "application/octet-stream"
internal const val CHUNK_FOLDER_COUNT = 256
private const val TAG = "SafStoragePlugin"
@ -28,12 +33,30 @@ public abstract class SafStoragePlugin(
private val context: Context,
) : StoragePlugin {
private val cache = SafCache()
protected abstract val root: DocumentFile?
private val contentResolver = context.contentResolver
private val folder: DocumentFile?
get() {
val root = this.root ?: return null
if (cache.currentFolder != null) return cache.currentFolder
private val chunkFolders = HashMap<String, DocumentFile>(CHUNK_FOLDER_COUNT)
private val snapshotFiles = HashMap<Long, DocumentFile>()
@SuppressLint("HardwareIds")
// this is unique to each combination of app-signing key, user, and device
// so we don't leak anything by not hashing this and can use it as is
val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID)
// the folder name is our user ID
val folderName = "$androidId.sv"
cache.currentFolder = try {
root.findFile(folderName) ?: root.createDirectoryOrThrow(folderName)
} catch (e: IOException) {
Log.e(TAG, "Error creating storage folder $folderName")
null
}
return cache.currentFolder
}
private val contentResolver = context.contentResolver
private fun timestampToSnapshot(timestamp: Long): String {
return "$timestamp.SeedSnap"
@ -41,27 +64,33 @@ public abstract class SafStoragePlugin(
@Throws(IOException::class)
override suspend fun getAvailableChunkIds(): List<String> {
val root = root ?: return emptyList()
val folder = folder ?: return emptyList()
val chunkIds = ArrayList<String>()
populateChunkFolders(root) { file, name ->
populateChunkFolders(folder, cache.backupChunkFolders) { file, name ->
if (chunkFolderRegex.matches(name)) {
chunkIds.addAll(getChunksFromFolder(file))
}
}
Log.e(TAG, "Got ${chunkIds.size} available chunks")
Log.i(TAG, "Got ${chunkIds.size} available chunks")
return chunkIds
}
/**
* Goes through all files in the given [folder] and performs the optional [fileOp] on them.
* Afterwards, it creates missing chunk folders, as needed.
* Chunk folders will get cached in the given [chunkFolders] for faster access.
*/
@Throws(IOException::class)
private suspend fun populateChunkFolders(
root: DocumentFile,
folder: DocumentFile,
chunkFolders: HashMap<String, DocumentFile>,
fileOp: ((DocumentFile, String) -> Unit)? = null
) {
val expectedChunkFolders = (0x00..0xff).map {
Integer.toHexString(it).padStart(2, '0')
}.toHashSet()
val duration = measure {
for (file in root.listFilesBlocking(context)) {
for (file in folder.listFilesBlocking(context)) {
val name = file.name ?: continue
if (chunkFolderRegex.matches(name)) {
chunkFolders[name] = file
@ -70,8 +99,8 @@ public abstract class SafStoragePlugin(
fileOp?.invoke(file, name)
}
}
Log.e(TAG, "Retrieving chunk folders took $duration")
createMissingChunkFolders(root, expectedChunkFolders)
Log.i(TAG, "Retrieving chunk folders took $duration")
createMissingChunkFolders(folder, chunkFolders, expectedChunkFolders)
}
@Throws(IOException::class)
@ -89,7 +118,11 @@ public abstract class SafStoragePlugin(
}
@Throws(IOException::class)
private fun createMissingChunkFolders(root: DocumentFile, expectedChunkFolders: Set<String>) {
private fun createMissingChunkFolders(
root: DocumentFile,
chunkFolders: HashMap<String, DocumentFile>,
expectedChunkFolders: Set<String>
) {
val s = expectedChunkFolders.size
val duration = measure {
for ((i, chunkFolderName) in expectedChunkFolders.withIndex()) {
@ -101,13 +134,14 @@ public abstract class SafStoragePlugin(
throw IOException("Only have ${chunkFolders.size} chunk folders.")
}
}
if (s > 0) Log.e(TAG, "Creating $s missing chunk folders took $duration")
if (s > 0) Log.i(TAG, "Creating $s missing chunk folders took $duration")
}
@Throws(IOException::class)
override fun getChunkOutputStream(chunkId: String): OutputStream {
val chunkFolderName = chunkId.substring(0, 2)
val chunkFolder = chunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId")
val chunkFolder =
cache.backupChunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId")
// TODO should we check if it exists first?
val chunkFile = chunkFolder.createFileOrThrow(chunkId, MIME_TYPE)
return chunkFile.getOutputStream(context.contentResolver)
@ -115,48 +149,58 @@ public abstract class SafStoragePlugin(
@Throws(IOException::class)
override fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
val root = root ?: throw IOException()
val folder = folder ?: throw IOException()
val name = timestampToSnapshot(timestamp)
// TODO should we check if it exists first?
val snapshotFile = root.createFileOrThrow(name, MIME_TYPE)
val snapshotFile = folder.createFileOrThrow(name, MIME_TYPE)
return snapshotFile.getOutputStream(contentResolver)
}
/************************* Restore *******************************/
@Throws(IOException::class)
override suspend fun getAvailableBackupSnapshots(): List<Long> {
val root = root ?: return emptyList()
val snapshots = ArrayList<Long>()
override suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
populateChunkFolders(root) { file, name ->
val match = snapshotRegex.matchEntire(name)
if (match != null) {
val timestamp = match.groupValues[1].toLong()
snapshots.add(timestamp)
snapshotFiles[timestamp] = file
root?.listFilesBlocking(context)?.forEach { folder ->
val folderName = folder.name ?: ""
if (!folderRegex.matches(folderName)) return@forEach
Log.i(TAG, "Checking $folderName for snapshots...")
for (file in folder.listFilesBlocking(context)) {
val name = file.name ?: continue
val match = snapshotRegex.matchEntire(name)
if (match != null) {
val timestamp = match.groupValues[1].toLong()
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
cache.snapshotFiles[storedSnapshot] = file
}
}
}
Log.e(TAG, "Got ${snapshots.size} snapshots while populating chunk folders")
Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders")
return snapshots
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream {
val snapshotFile = snapshotFiles.getOrElse(timestamp) {
root?.findFileBlocking(context, timestampToSnapshot(timestamp))
override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream {
val timestamp = storedSnapshot.timestamp
val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) {
getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp))
} ?: throw IOException("Could not get file for snapshot $timestamp")
return snapshotFile.getInputStream(contentResolver)
}
@Throws(IOException::class)
override suspend fun getChunkInputStream(chunkId: String): InputStream {
if (chunkFolders.size < CHUNK_FOLDER_COUNT) {
val root = root ?: throw IOException("Could not get root")
populateChunkFolders(root)
override suspend fun getChunkInputStream(
snapshot: StoredSnapshot,
chunkId: String
): InputStream {
if (cache.restoreChunkFolders.size < CHUNK_FOLDER_COUNT) {
populateChunkFolders(getFolder(snapshot), cache.restoreChunkFolders)
}
val chunkFolderName = chunkId.substring(0, 2)
val chunkFolder = chunkFolders[chunkFolderName]
val chunkFolder = cache.restoreChunkFolders[chunkFolderName]
?: throw IOException("No folder for chunk $chunkId")
val chunkFile = chunkFolder.findFileBlocking(context, chunkId)
?: throw IOException("No chunk $chunkId")
@ -164,23 +208,56 @@ public abstract class SafStoragePlugin(
}
@Throws(IOException::class)
override suspend fun deleteBackupSnapshot(timestamp: Long) {
Log.d(TAG, "Deleting snapshot $timestamp")
val snapshotFile = snapshotFiles.getOrElse(timestamp) {
root?.findFileBlocking(context, timestampToSnapshot(timestamp))
} ?: throw IOException("Could not get file for snapshot $timestamp")
if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp")
private suspend fun getFolder(storedSnapshot: StoredSnapshot): DocumentFile {
// not cached, because used in several places only once and
// [getBackupSnapshotInputStream] uses snapshot files cache and
// [getChunkInputStream] uses restore chunk folders cache
return root?.findFileBlocking(context, storedSnapshot.userId)
?: throw IOException("Could not find snapshot $storedSnapshot")
}
/************************* Pruning *******************************/
@Throws(IOException::class)
override suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot> {
val folder = folder ?: return emptyList()
val folderName = folder.name ?: error("Folder suddenly has no more name")
val snapshots = ArrayList<StoredSnapshot>()
populateChunkFolders(folder, cache.backupChunkFolders) { file, name ->
val match = snapshotRegex.matchEntire(name)
if (match != null) {
val timestamp = match.groupValues[1].toLong()
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
cache.snapshotFiles[storedSnapshot] = file
}
}
Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders")
return snapshots
}
@Throws(IOException::class)
override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) {
val timestamp = storedSnapshot.timestamp
Log.d(TAG, "Deleting snapshot $timestamp")
val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) {
getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp))
} ?: throw IOException("Could not get file for snapshot $timestamp")
if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp")
cache.snapshotFiles.remove(storedSnapshot)
}
@Throws(IOException::class)
override suspend fun deleteChunks(chunkIds: List<String>) {
if (chunkFolders.size < CHUNK_FOLDER_COUNT) {
val root = root ?: throw IOException("Could not get root")
populateChunkFolders(root)
if (cache.backupChunkFolders.size < CHUNK_FOLDER_COUNT) {
val folder = folder ?: throw IOException("Could not get current folder in root")
populateChunkFolders(folder, cache.backupChunkFolders)
}
for (chunkId in chunkIds) {
Log.d(TAG, "Deleting chunk $chunkId")
val chunkFolderName = chunkId.substring(0, 2)
val chunkFolder = chunkFolders[chunkFolderName]
val chunkFolder = cache.backupChunkFolders[chunkFolderName]
?: throw IOException("No folder for chunk $chunkId")
val chunkFile = chunkFolder.findFileBlocking(context, chunkId)
if (chunkFile == null) {

26
storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt
View file

@ -3,6 +3,7 @@ package org.calyxos.backup.storage.prune
import android.util.Log
import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure
@ -33,15 +34,15 @@ internal class Pruner(
@Throws(IOException::class)
suspend fun prune(backupObserver: BackupObserver?) {
val duration = measure {
val timestamps = storagePlugin.getAvailableBackupSnapshots()
val toDelete = retentionManager.getSnapshotsToDelete(timestamps)
backupObserver?.onPruneStart(toDelete)
for (timestamp in toDelete) {
val storedSnapshots = storagePlugin.getCurrentBackupSnapshots()
val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots)
backupObserver?.onPruneStart(toDelete.map { it.timestamp })
for (snapshot in toDelete) {
try {
pruneSnapshot(timestamp, backupObserver)
pruneSnapshot(snapshot, backupObserver)
} catch (e: Exception) {
Log.e(TAG, "Error pruning $timestamp", e)
backupObserver?.onPruneError(timestamp, e)
Log.e(TAG, "Error pruning $snapshot", e)
backupObserver?.onPruneError(snapshot.timestamp, e)
}
}
}
@ -50,12 +51,15 @@ internal class Pruner(
}
@Throws(IOException::class, GeneralSecurityException::class)
private suspend fun pruneSnapshot(timestamp: Long, backupObserver: BackupObserver?) {
val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp)
private suspend fun pruneSnapshot(
storedSnapshot: StoredSnapshot,
backupObserver: BackupObserver?
) {
val snapshot = snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
val chunks = HashSet<String>()
snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) }
snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) }
storagePlugin.deleteBackupSnapshot(timestamp)
storagePlugin.deleteBackupSnapshot(storedSnapshot)
db.applyInParts(chunks) {
chunksCache.decrementRefCount(it)
}
@ -68,7 +72,7 @@ internal class Pruner(
size += it.size
it.id
}
backupObserver?.onPruneSnapshot(timestamp, chunkIdsToDelete.size, size)
backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size)
storagePlugin.deleteChunks(chunkIdsToDelete)
chunksCache.deleteChunks(cachedChunksToDelete)
}

31
storage/lib/src/main/java/org/calyxos/backup/storage/prune/RetentionManager.kt
View file

@ -2,6 +2,7 @@ package org.calyxos.backup.storage.prune
import android.content.Context
import org.calyxos.backup.storage.api.SnapshotRetention
import org.calyxos.backup.storage.api.StoredSnapshot
import java.io.IOException
import java.time.LocalDate
import java.time.temporal.ChronoField
@ -48,37 +49,37 @@ internal class RetentionManager(private val context: Context) {
}
/**
* Takes a list of snapshot timestamps and returns a list of those
* Takes a list of [StoredSnapshot]s and returns a list of those
* that can be deleted according to the current snapshot retention policy.
*/
@Throws(IOException::class)
fun getSnapshotsToDelete(snapshotTimestamps: List<Long>): List<Long> {
fun getSnapshotsToDelete(storedSnapshots: List<StoredSnapshot>): List<StoredSnapshot> {
val retention = getSnapshotRetention()
val dates = snapshotTimestamps.sortedDescending().map {
Pair(it, LocalDate.ofEpochDay(it / 1000 / 60 / 60 / 24))
val datePairs = storedSnapshots.sortedByDescending { it.timestamp }.map { s ->
Pair(s, LocalDate.ofEpochDay(s.timestamp / 1000 / 60 / 60 / 24))
}
val toKeep = HashSet<Long>()
toKeep += getToKeep(dates, retention.daily)
toKeep += getToKeep(dates, retention.weekly) { temporal: Temporal ->
val toKeep = HashSet<StoredSnapshot>()
toKeep += getToKeep(datePairs, retention.daily)
toKeep += getToKeep(datePairs, retention.weekly) { temporal: Temporal ->
temporal.with(ChronoField.DAY_OF_WEEK, 1)
}
toKeep += getToKeep(dates, retention.monthly, firstDayOfMonth())
toKeep += getToKeep(dates, retention.yearly, firstDayOfYear())
return snapshotTimestamps - toKeep
toKeep += getToKeep(datePairs, retention.monthly, firstDayOfMonth())
toKeep += getToKeep(datePairs, retention.yearly, firstDayOfYear())
return storedSnapshots - toKeep
}
private fun getToKeep(
pairs: List<Pair<Long, LocalDate>>,
pairs: List<Pair<StoredSnapshot, LocalDate>>,
keep: Int,
temporalAdjuster: TemporalAdjuster? = null,
): List<Long> {
val toKeep = ArrayList<Long>()
): List<StoredSnapshot> {
val toKeep = ArrayList<StoredSnapshot>()
if (keep == 0) return toKeep
var last: LocalDate? = null
for ((timestamp, date) in pairs) {
for ((snapshot, date) in pairs) {
val period = if (temporalAdjuster == null) date else date.with(temporalAdjuster)
if (period != last) {
toKeep.add(timestamp)
toKeep.add(snapshot)
if (toKeep.size >= keep) break
last = period
}

4
storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt
View file

@ -2,6 +2,7 @@ package org.calyxos.backup.storage.restore
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import java.io.IOException
import java.io.InputStream
@ -19,10 +20,11 @@ internal abstract class AbstractChunkRestore(
@Throws(IOException::class, GeneralSecurityException::class)
protected suspend fun getAndDecryptChunk(
version: Int,
storedSnapshot: StoredSnapshot,
chunkId: String,
streamReader: suspend (InputStream) -> Unit,
) {
storagePlugin.getChunkInputStream(chunkId).use { inputStream ->
storagePlugin.getChunkInputStream(storedSnapshot, chunkId).use { inputStream ->
inputStream.readVersion(version)
val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->

15
storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt
View file

@ -4,6 +4,7 @@ import android.content.Context
import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import java.io.File
import java.io.FileInputStream
@ -26,6 +27,7 @@ internal class MultiChunkRestore(
suspend fun restore(
version: Int,
storedSnapshot: StoredSnapshot,
chunkMap: Map<String, RestorableChunk>,
files: Collection<RestorableFile>,
observer: RestoreObserver?,
@ -34,7 +36,7 @@ internal class MultiChunkRestore(
files.forEach { file ->
try {
restoreFile(file, observer, "L") { outputStream ->
writeChunks(version, file, chunkMap, outputStream)
writeChunks(version, storedSnapshot, file, chunkMap, outputStream)
}
restoredFiles++
} catch (e: Exception) {
@ -49,6 +51,7 @@ internal class MultiChunkRestore(
@Throws(IOException::class, GeneralSecurityException::class)
private suspend fun writeChunks(
version: Int,
storedSnapshot: StoredSnapshot,
file: RestorableFile,
chunkMap: Map<String, RestorableChunk>,
outputStream: OutputStream,
@ -61,8 +64,11 @@ internal class MultiChunkRestore(
bytes += decryptedStream.copyTo(outputStream)
}
val isCached = isCached(chunkId)
if (isCached || otherFiles.size > 1) getAndCacheChunk(version, chunkId, chunkWriter)
else getAndDecryptChunk(version, chunkId, chunkWriter)
if (isCached || otherFiles.size > 1) {
getAndCacheChunk(version, storedSnapshot, chunkId, chunkWriter)
} else {
getAndDecryptChunk(version, storedSnapshot, chunkId, chunkWriter)
}
otherFiles.remove(file)
if (isCached && otherFiles.isEmpty()) removeCachedChunk(chunkId)
@ -77,13 +83,14 @@ internal class MultiChunkRestore(
@Throws(IOException::class, GeneralSecurityException::class)
private suspend fun getAndCacheChunk(
version: Int,
storedSnapshot: StoredSnapshot,
chunkId: String,
streamReader: suspend (InputStream) -> Unit,
) {
val file = getChunkCacheFile(chunkId)
if (!file.isFile) {
FileOutputStream(file).use { outputStream ->
getAndDecryptChunk(version, chunkId) { decryptedStream ->
getAndDecryptChunk(version, storedSnapshot, chunkId) { decryptedStream ->
decryptedStream.copyTo(outputStream)
}
}

44
storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt
View file

@ -8,6 +8,7 @@ import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.SnapshotItem
import org.calyxos.backup.storage.api.SnapshotResult
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
@ -46,8 +47,10 @@ internal class Restore(
val numSnapshots: Int
val time = measure {
val list = try {
storagePlugin.getAvailableBackupSnapshots().sortedDescending().map {
SnapshotItem(it, null)
storagePlugin.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot ->
storedSnapshot.timestamp
}.map { storedSnapshot ->
SnapshotItem(storedSnapshot, null)
}.toMutableList()
} catch (e: Exception) {
Log.e("TAG", "Error retrieving snapshots", e)
@ -61,7 +64,9 @@ internal class Restore(
while (iterator.hasNext()) {
val oldItem = iterator.next()
val item = try {
oldItem.copy(snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.time))
oldItem.copy(
snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.storedSnapshot)
)
} catch (e: Exception) {
Log.e("TAG", "Error retrieving snapshot ${oldItem.time}", e)
continue
@ -73,15 +78,15 @@ internal class Restore(
Log.e(TAG, "Decrypting and parsing $numSnapshots snapshots took $time")
}
@Throws(IOException::class, GeneralSecurityException::class)
suspend fun restoreBackupSnapshot(timestamp: Long, observer: RestoreObserver?) {
val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp)
restoreBackupSnapshot(snapshot, observer)
}
@OptIn(ExperimentalTime::class)
@Throws(IOException::class)
suspend fun restoreBackupSnapshot(snapshot: BackupSnapshot, observer: RestoreObserver?) {
@Throws(IOException::class, GeneralSecurityException::class)
suspend fun restoreBackupSnapshot(
storedSnapshot: StoredSnapshot,
optionalSnapshot: BackupSnapshot? = null,
observer: RestoreObserver? = null,
) {
val snapshot = optionalSnapshot ?: snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
val filesTotal = snapshot.mediaFilesList.size + snapshot.documentFilesList.size
val totalSize =
snapshot.mediaFilesList.sumOf { it.size } + snapshot.documentFilesList.sumOf { it.size }
@ -91,19 +96,30 @@ internal class Restore(
val version = snapshot.version
var restoredFiles = 0
val smallFilesDuration = measure {
restoredFiles += zipChunkRestore.restore(version, split.zipChunks, observer)
restoredFiles += zipChunkRestore.restore(
version,
storedSnapshot,
split.zipChunks,
observer,
)
}
Log.e(TAG, "Restoring ${split.zipChunks.size} zip chunks took $smallFilesDuration.")
val singleChunkDuration = measure {
restoredFiles += singleChunkRestore.restore(version, split.singleChunks, observer)
restoredFiles += singleChunkRestore.restore(
version,
storedSnapshot,
split.singleChunks,
observer,
)
}
Log.e(TAG, "Restoring ${split.singleChunks.size} single chunks took $singleChunkDuration.")
val multiChunkDuration = measure {
restoredFiles += multiChunkRestore.restore(
version,
storedSnapshot,
split.multiChunkMap,
split.multiChunkFiles,
observer
observer,
)
}
Log.e(TAG, "Restoring ${split.multiChunkFiles.size} multi chunks took $multiChunkDuration.")

18
storage/lib/src/main/java/org/calyxos/backup/storage/restore/RestoreService.kt
View file

@ -8,20 +8,24 @@ import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StorageBackup
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID
import org.calyxos.backup.storage.ui.NOTIFICATION_ID_RESTORE
import org.calyxos.backup.storage.ui.Notifications
/**
* Start to trigger restore as a foreground service. Ensure that you provide the snapshot
* to be restored with [Intent.putExtra] as a [Long] in [EXTRA_TIMESTAMP_START].
* See [BackupSnapshot.getTimeStart].
* to be restored with [Intent.putExtra]:
* * the user ID of the snapshot as a [String] in [EXTRA_USER_ID]
* * the snapshot's timestamp as a [Long] in [EXTRA_TIMESTAMP_START].
* See [BackupSnapshot.getTimeStart].
*/
public abstract class RestoreService : Service() {
public companion object {
private const val TAG = "RestoreService"
public const val EXTRA_USER_ID: String = "userId"
public const val EXTRA_TIMESTAMP_START: String = "timestamp"
}
@ -31,13 +35,15 @@ public abstract class RestoreService : Service() {
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
Log.d(TAG, "onStartCommand $intent $flags $startId")
val timestamp = intent?.getLongExtra(EXTRA_TIMESTAMP_START, -1)
if (timestamp == null || timestamp < 0) error("No timestamp in intent: $intent")
val userId = intent?.getStringExtra(EXTRA_USER_ID) ?: error("No user ID in intent: $intent")
val timestamp = intent.getLongExtra(EXTRA_TIMESTAMP_START, -1)
if (timestamp < 0) error("No timestamp in intent: $intent")
val storedSnapshot = StoredSnapshot(userId, timestamp)
startForeground(NOTIFICATION_ID_RESTORE, n.getRestoreNotification())
GlobalScope.launch {
// TODO offer a way to try again if failed, or do an automatic retry here
storageBackup.restoreBackupSnapshot(timestamp, restoreObserver)
storageBackup.restoreBackupSnapshot(storedSnapshot, null, restoreObserver)
stopSelf(startId)
}
return START_STICKY_COMPATIBILITY

4
storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt
View file

@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore
import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
private const val TAG = "SingleChunkRestore"
@ -17,6 +18,7 @@ internal class SingleChunkRestore(
suspend fun restore(
version: Int,
storedSnapshot: StoredSnapshot,
chunks: Collection<RestorableChunk>,
observer: RestoreObserver?
): Int {
@ -25,7 +27,7 @@ internal class SingleChunkRestore(
check(chunk.files.size == 1)
val file = chunk.files[0]
try {
getAndDecryptChunk(version, chunk.chunkId) { decryptedStream ->
getAndDecryptChunk(version, storedSnapshot, chunk.chunkId) { decryptedStream ->
restoreFile(file, observer, "M") { outputStream ->
decryptedStream.copyTo(outputStream)
}

4
storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt
View file

@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore
import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import java.io.IOException
import java.io.InputStream
@ -24,13 +25,14 @@ internal class ZipChunkRestore(
*/
suspend fun restore(
version: Int,
storedSnapshot: StoredSnapshot,
zipChunks: Collection<RestorableChunk>,
observer: RestoreObserver?
): Int {
var restoredFiles = 0
zipChunks.forEach { zipChunk ->
try {
getAndDecryptChunk(version, zipChunk.chunkId) { decryptedStream ->
getAndDecryptChunk(version, storedSnapshot, zipChunk.chunkId) { decryptedStream ->
restoredFiles += restoreZipChunk(zipChunk, decryptedStream, observer)
}
} catch (e: Exception) {

55
storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt
View file

@ -19,6 +19,7 @@ import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.SnapshotResult
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX
import org.calyxos.backup.storage.backup.Backup.Companion.SMALL_FILE_SIZE_MAX
@ -173,14 +174,16 @@ internal class BackupRestoreTest {
// RESTORE
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured)
val smallFileMOutputStream = ByteArrayOutputStream()
val smallFileDOutputStream = ByteArrayOutputStream()
val fileMOutputStream = ByteArrayOutputStream()
val fileDOutputStream = ByteArrayOutputStream()
coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured)
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery {
plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured)
plugin.getBackupSnapshotInputStream(storedSnapshot)
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
// retrieve snapshots
@ -197,14 +200,14 @@ internal class BackupRestoreTest {
// pipe chunks back in
coEvery {
plugin.getChunkInputStream(cachedFiles[0].chunks[0])
plugin.getChunkInputStream(storedSnapshot, cachedFiles[0].chunks[0])
} returns ByteArrayInputStream(zipChunkOutputStream.toByteArray())
// cachedFiles[0].chunks[1] is in previous zipChunk
coEvery {
plugin.getChunkInputStream(cachedFiles[2].chunks[0])
plugin.getChunkInputStream(storedSnapshot, cachedFiles[2].chunks[0])
} returns ByteArrayInputStream(mOutputStream.toByteArray())
coEvery {
plugin.getChunkInputStream(cachedFiles[3].chunks[0])
plugin.getChunkInputStream(storedSnapshot, cachedFiles[3].chunks[0])
} returns ByteArrayInputStream(dOutputStream.toByteArray())
// provide file output streams for restore
@ -217,7 +220,7 @@ internal class BackupRestoreTest {
val fileDRestorable = getRestorableFileD(fileD, snapshot)
expectRestoreFile(fileDRestorable, fileDOutputStream)
restore.restoreBackupSnapshot(snapshot, null)
restore.restoreBackupSnapshot(storedSnapshot, snapshot, null)
// restored files match backed up files exactly
assertArrayEquals(smallFileMBytes, smallFileMOutputStream.toByteArray())
@ -337,9 +340,11 @@ internal class BackupRestoreTest {
// RESTORE
coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured)
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured)
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery {
plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured)
plugin.getBackupSnapshotInputStream(storedSnapshot)
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
// retrieve snapshots
@ -354,19 +359,27 @@ internal class BackupRestoreTest {
// pipe chunks back in
coEvery {
plugin.getChunkInputStream(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3")
storedSnapshot,
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
)
} returns ByteArrayInputStream(id040f32.toByteArray())
coEvery {
plugin.getChunkInputStream(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29")
storedSnapshot,
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
)
} returns ByteArrayInputStream(id901fbc.toByteArray())
coEvery {
plugin.getChunkInputStream(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d")
storedSnapshot,
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
)
} returns ByteArrayInputStream(id5adea3.toByteArray())
coEvery {
plugin.getChunkInputStream(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67")
storedSnapshot,
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
)
} returns ByteArrayInputStream(id40d00c.toByteArray())
// provide file output streams for restore
@ -375,7 +388,7 @@ internal class BackupRestoreTest {
val file2Restorable = getRestorableFileD(file2, snapshot)
expectRestoreFile(file2Restorable, file2OutputStream)
restore.restoreBackupSnapshot(snapshot, null)
restore.restoreBackupSnapshot(storedSnapshot, snapshot, null)
// restored files match backed up files exactly
assertArrayEquals(file1Bytes, file1OutputStream.toByteArray())
@ -384,13 +397,21 @@ internal class BackupRestoreTest {
// chunks were only read from storage once
coVerify(exactly = 1) {
plugin.getChunkInputStream(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3")
storedSnapshot,
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
)
plugin.getChunkInputStream(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29")
storedSnapshot,
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
)
plugin.getChunkInputStream(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d")
storedSnapshot,
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
)
plugin.getChunkInputStream(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67")
storedSnapshot,
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
)
}
}

11
storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt
View file

@ -9,6 +9,7 @@ import io.mockk.mockk
import io.mockk.slot
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.db.CachedChunk
import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db
@ -56,7 +57,9 @@ internal class ChunksCacheRepopulaterTest {
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4))
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk5))
.build()
val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart)
val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart)
val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart)
val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2)
val cachedChunks = listOf(
CachedChunk(chunk1, 2, 0),
CachedChunk(chunk2, 2, 0),
@ -64,12 +67,12 @@ internal class ChunksCacheRepopulaterTest {
) // chunk3 is not referenced and should get deleted
val cachedChunksSlot = slot<Collection<CachedChunk>>()
coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots
coEvery {
snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart)
snapshotRetriever.getSnapshot(streamKey, storedSnapshot1)
} returns snapshot1
coEvery {
snapshotRetriever.getSnapshot(streamKey, snapshot2.timeStart)
snapshotRetriever.getSnapshot(streamKey, storedSnapshot2)
} returns snapshot2
every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs

15
storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt
View file

@ -8,6 +8,7 @@ import io.mockk.mockk
import io.mockk.slot
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupDocumentFile
import org.calyxos.backup.storage.backup.BackupMediaFile
import org.calyxos.backup.storage.backup.BackupSnapshot
@ -66,18 +67,20 @@ internal class PrunerTest {
.addMediaFiles(BackupMediaFile.newBuilder().addChunkIds(chunk2))
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4))
.build()
val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart)
val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart)
val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart)
val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2)
val expectedChunks = listOf(chunk1, chunk2, chunk3)
val actualChunks = slot<Collection<String>>()
val actualChunks2 = slot<Collection<String>>()
val cachedChunk3 = CachedChunk(chunk3, 0, 0)
coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots
every {
retentionManager.getSnapshotsToDelete(snapshotTimestamps)
} returns listOf(snapshot1.timeStart)
coEvery { snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart) } returns snapshot1
coEvery { plugin.deleteBackupSnapshot(snapshot1.timeStart) } just Runs
retentionManager.getSnapshotsToDelete(storedSnapshots)
} returns listOf(storedSnapshot1)
coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1
coEvery { plugin.deleteBackupSnapshot(storedSnapshot1) } just Runs
every {
db.applyInParts(capture(actualChunks), captureLambda())
} answers {

30
storage/lib/src/test/java/org/calyxos/backup/storage/prune/RetentionManagerTest.kt
View file

@ -5,6 +5,8 @@ import android.content.SharedPreferences
import io.mockk.every
import io.mockk.mockk
import org.calyxos.backup.storage.api.SnapshotRetention
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.getRandomString
import org.junit.Assert.assertEquals
import org.junit.Test
import java.time.LocalDateTime
@ -17,19 +19,21 @@ internal class RetentionManagerTest {
private val retention = RetentionManager(context)
private val userId = getRandomString()
@Test
fun testDailyRetention() {
expectGetRetention(SnapshotRetention(2, 0, 0, 0))
val timestamps = listOf(
val storedSnapshots = listOf(
// 1577919600000
LocalDateTime.of(2020, 1, 1, 23, 0).toMillis(),
// 1577872800000
LocalDateTime.of(2020, 1, 1, 10, 0).toMillis(),
// 1583276400000
LocalDateTime.of(2020, 3, 3, 23, 0).toMillis(),
)
val toDelete = retention.getSnapshotsToDelete(timestamps)
assertEquals(listOf(1577872800000), toDelete)
).map { StoredSnapshot(userId, it) }
val toDelete = retention.getSnapshotsToDelete(storedSnapshots)
assertEquals(listOf(1577872800000), toDelete.map { it.timestamp })
}
@Test
@ -45,9 +49,9 @@ internal class RetentionManagerTest {
LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(),
// 1608678000000
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
)
).map { StoredSnapshot(userId, it) }
val toDelete = retention.getSnapshotsToDelete(timestamps)
assertEquals(listOf(1608544800000, 1608638400000), toDelete)
assertEquals(listOf(1608544800000, 1608638400000), toDelete.map { it.timestamp })
}
@Test
@ -63,9 +67,9 @@ internal class RetentionManagerTest {
LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(),
// 1608678000000
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
)
).map { StoredSnapshot(userId, it) }
val toDelete = retention.getSnapshotsToDelete(timestamps)
assertEquals(listOf(1580857200000), toDelete)
assertEquals(listOf(1580857200000), toDelete.map { it.timestamp })
}
@Test
@ -83,10 +87,10 @@ internal class RetentionManagerTest {
LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(),
// 1608678000000
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
)
).map { StoredSnapshot(userId, it) }
// keeps only the latest one for each year, so three in total, even though keep is four
val toDelete = retention.getSnapshotsToDelete(timestamps)
assertEquals(listOf(1549321200000, 1608544800000), toDelete)
assertEquals(listOf(1549321200000, 1608544800000), toDelete.map { it.timestamp })
}
@Test
@ -116,9 +120,11 @@ internal class RetentionManagerTest {
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
// 1608638400000
LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(),
)
).map { StoredSnapshot(userId, it) }
val toDelete = retention.getSnapshotsToDelete(timestamps)
assertEquals(listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000), toDelete)
assertEquals(
listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000),
toDelete.map { it.timestamp })
}
private fun expectGetRetention(snapshotRetention: SnapshotRetention) {