Allow using the same storage location on different devices or user profiles
Previously, we would put our files directly in the root of the storage location and delete any existing backups there. When used by different devices or user profiles, these would keep deleting each other's backups.
This commit is contained in:
parent
a762d1b64e
commit
347d2a316f
22 changed files with 375 additions and 166 deletions
|
@ -62,6 +62,7 @@ import kotlinx.coroutines.launch
|
||||||
import org.calyxos.backup.storage.api.SnapshotItem
|
import org.calyxos.backup.storage.api.SnapshotItem
|
||||||
import org.calyxos.backup.storage.api.StorageBackup
|
import org.calyxos.backup.storage.api.StorageBackup
|
||||||
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START
|
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START
|
||||||
|
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID
|
||||||
import org.calyxos.backup.storage.ui.restore.SnapshotViewModel
|
import org.calyxos.backup.storage.ui.restore.SnapshotViewModel
|
||||||
import java.util.LinkedList
|
import java.util.LinkedList
|
||||||
import kotlin.coroutines.Continuation
|
import kotlin.coroutines.Continuation
|
||||||
|
@ -392,6 +393,7 @@ internal class RestoreViewModel(
|
||||||
@UiThread
|
@UiThread
|
||||||
internal fun startFilesRestore(item: SnapshotItem) {
|
internal fun startFilesRestore(item: SnapshotItem) {
|
||||||
val i = Intent(app, StorageRestoreService::class.java)
|
val i = Intent(app, StorageRestoreService::class.java)
|
||||||
|
i.putExtra(EXTRA_USER_ID, item.storedSnapshot.userId)
|
||||||
i.putExtra(EXTRA_TIMESTAMP_START, item.time)
|
i.putExtra(EXTRA_TIMESTAMP_START, item.time)
|
||||||
app.startForegroundService(i)
|
app.startForegroundService(i)
|
||||||
mDisplayFragment.setEvent(RESTORE_FILES_STARTED)
|
mDisplayFragment.setEvent(RESTORE_FILES_STARTED)
|
||||||
|
|
|
@ -123,6 +123,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati
|
||||||
|
|
||||||
// example for how to do restore via foreground service
|
// example for how to do restore via foreground service
|
||||||
// app.startForegroundService(Intent(app, DemoRestoreService::class.java).apply {
|
// app.startForegroundService(Intent(app, DemoRestoreService::class.java).apply {
|
||||||
|
// putExtra(EXTRA_USER_ID, item.storedSnapshot.userId)
|
||||||
// putExtra(EXTRA_TIMESTAMP_START, snapshot.timeStart)
|
// putExtra(EXTRA_TIMESTAMP_START, snapshot.timeStart)
|
||||||
// })
|
// })
|
||||||
|
|
||||||
|
@ -130,7 +131,7 @@ class MainViewModel(application: Application) : BackupContentViewModel(applicati
|
||||||
_restoreProgressVisible.value = true
|
_restoreProgressVisible.value = true
|
||||||
val restoreObserver = RestoreStats(app, _restoreLog)
|
val restoreObserver = RestoreStats(app, _restoreLog)
|
||||||
viewModelScope.launch {
|
viewModelScope.launch {
|
||||||
storageBackup.restoreBackupSnapshot(snapshot, restoreObserver)
|
storageBackup.restoreBackupSnapshot(item.storedSnapshot, snapshot, restoreObserver)
|
||||||
_restoreProgressVisible.value = false
|
_restoreProgressVisible.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -388,7 +388,6 @@ but are considered out-of-scope of the current design for time and budget reason
|
||||||
* using a rolling hash to produce chunks in order to increase likelihood of obtaining same chunks
|
* using a rolling hash to produce chunks in order to increase likelihood of obtaining same chunks
|
||||||
even if file contents change slightly or shift
|
even if file contents change slightly or shift
|
||||||
* external secret-less corruption checks that would use checksums over encrypted data
|
* external secret-less corruption checks that would use checksums over encrypted data
|
||||||
* supporting different backup clients backing up to the same storage
|
|
||||||
* concealing file sizes (though zip chunks helps a bit here)
|
* concealing file sizes (though zip chunks helps a bit here)
|
||||||
* implementing different storage plugins
|
* implementing different storage plugins
|
||||||
|
|
||||||
|
|
|
@ -3,15 +3,28 @@ package org.calyxos.backup.storage.api
|
||||||
import org.calyxos.backup.storage.backup.BackupSnapshot
|
import org.calyxos.backup.storage.backup.BackupSnapshot
|
||||||
|
|
||||||
public data class SnapshotItem(
|
public data class SnapshotItem(
|
||||||
public val time: Long,
|
public val storedSnapshot: StoredSnapshot,
|
||||||
public val snapshot: BackupSnapshot?,
|
public val snapshot: BackupSnapshot?,
|
||||||
)
|
) {
|
||||||
|
val time: Long get() = storedSnapshot.timestamp
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class SnapshotResult {
|
public sealed class SnapshotResult {
|
||||||
public data class Success(val snapshots: List<SnapshotItem>) : SnapshotResult()
|
public data class Success(val snapshots: List<SnapshotItem>) : SnapshotResult()
|
||||||
public data class Error(val e: Exception) : SnapshotResult()
|
public data class Error(val e: Exception) : SnapshotResult()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public data class StoredSnapshot(
|
||||||
|
/**
|
||||||
|
* The unique ID of the current device/user combination chosen by the [StoragePlugin].
|
||||||
|
*/
|
||||||
|
public val userId: String,
|
||||||
|
/**
|
||||||
|
* The timestamp identifying a snapshot of the [userId].
|
||||||
|
*/
|
||||||
|
public val timestamp: Long,
|
||||||
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Defines which backup snapshots should be retained when pruning backups.
|
* Defines which backup snapshots should be retained when pruning backups.
|
||||||
*
|
*
|
||||||
|
|
|
@ -103,10 +103,12 @@ public class StorageBackup(
|
||||||
* Run this on a new storage location to ensure that there are no old snapshots
|
* Run this on a new storage location to ensure that there are no old snapshots
|
||||||
* (potentially encrypted with an old key) laying around.
|
* (potentially encrypted with an old key) laying around.
|
||||||
* Using a storage location with existing data is not supported.
|
* Using a storage location with existing data is not supported.
|
||||||
|
* Using the same root folder for storage on different devices or user profiles is fine though
|
||||||
|
* as the [StoragePlugin] should isolate storage per [StoredSnapshot.userId].
|
||||||
*/
|
*/
|
||||||
public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) {
|
public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) {
|
||||||
try {
|
try {
|
||||||
plugin.getAvailableBackupSnapshots().forEach {
|
plugin.getCurrentBackupSnapshots().forEach {
|
||||||
try {
|
try {
|
||||||
plugin.deleteBackupSnapshot(it)
|
plugin.deleteBackupSnapshot(it)
|
||||||
} catch (e: IOException) {
|
} catch (e: IOException) {
|
||||||
|
@ -183,15 +185,16 @@ public class StorageBackup(
|
||||||
}
|
}
|
||||||
|
|
||||||
public suspend fun restoreBackupSnapshot(
|
public suspend fun restoreBackupSnapshot(
|
||||||
snapshot: BackupSnapshot,
|
storedSnapshot: StoredSnapshot,
|
||||||
restoreObserver: RestoreObserver? = null
|
snapshot: BackupSnapshot? = null,
|
||||||
|
restoreObserver: RestoreObserver? = null,
|
||||||
): Boolean = withContext(dispatcher) {
|
): Boolean = withContext(dispatcher) {
|
||||||
if (restoreRunning.getAndSet(true)) {
|
if (restoreRunning.getAndSet(true)) {
|
||||||
Log.w(TAG, "Restore already running, not starting a new one")
|
Log.w(TAG, "Restore already running, not starting a new one")
|
||||||
return@withContext false
|
return@withContext false
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
restore.restoreBackupSnapshot(snapshot, restoreObserver)
|
restore.restoreBackupSnapshot(storedSnapshot, snapshot, restoreObserver)
|
||||||
true
|
true
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
Log.e(TAG, "Error during restore", e)
|
Log.e(TAG, "Error during restore", e)
|
||||||
|
@ -201,17 +204,4 @@ public class StorageBackup(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public suspend fun restoreBackupSnapshot(
|
|
||||||
timestamp: Long,
|
|
||||||
restoreObserver: RestoreObserver? = null
|
|
||||||
): Boolean = withContext(dispatcher) {
|
|
||||||
try {
|
|
||||||
restore.restoreBackupSnapshot(timestamp, restoreObserver)
|
|
||||||
true
|
|
||||||
} catch (e: Exception) {
|
|
||||||
Log.e(TAG, "Error during restore", e)
|
|
||||||
false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,22 +35,36 @@ public interface StoragePlugin {
|
||||||
/* Restore */
|
/* Restore */
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the timestamps representing a backup snapshot that are available on storage.
|
* Returns *all* [StoredSnapshot]s that are available on storage
|
||||||
|
* independent of user ID and whether they can be decrypted
|
||||||
|
* with the key returned by [getMasterKey].
|
||||||
*/
|
*/
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
public suspend fun getAvailableBackupSnapshots(): List<Long>
|
public suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot>
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
public suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream
|
public suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
public suspend fun getChunkInputStream(chunkId: String): InputStream
|
public suspend fun getChunkInputStream(snapshot: StoredSnapshot, chunkId: String): InputStream
|
||||||
|
|
||||||
/* Pruning */
|
/* Pruning */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns [StoredSnapshot]s for the currently active user ID.
|
||||||
|
*/
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
public suspend fun deleteBackupSnapshot(timestamp: Long)
|
public suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes the given [StoredSnapshot].
|
||||||
|
*/
|
||||||
|
@Throws(IOException::class)
|
||||||
|
public suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes the given chunks of the *current* user ID only.
|
||||||
|
*/
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
public suspend fun deleteChunks(chunkIds: List<String>)
|
public suspend fun deleteChunks(chunkIds: List<String>)
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@ import org.calyxos.backup.storage.db.Db
|
||||||
import org.calyxos.backup.storage.measure
|
import org.calyxos.backup.storage.measure
|
||||||
import org.calyxos.backup.storage.plugin.SnapshotRetriever
|
import org.calyxos.backup.storage.plugin.SnapshotRetriever
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
|
import java.security.GeneralSecurityException
|
||||||
import java.util.concurrent.TimeUnit.MILLISECONDS
|
import java.util.concurrent.TimeUnit.MILLISECONDS
|
||||||
import kotlin.time.ExperimentalTime
|
import kotlin.time.ExperimentalTime
|
||||||
import kotlin.time.toDuration
|
import kotlin.time.toDuration
|
||||||
|
@ -37,8 +38,13 @@ internal class ChunksCacheRepopulater(
|
||||||
availableChunkIds: HashSet<String>
|
availableChunkIds: HashSet<String>
|
||||||
) {
|
) {
|
||||||
val start = System.currentTimeMillis()
|
val start = System.currentTimeMillis()
|
||||||
val snapshots = storagePlugin.getAvailableBackupSnapshots().map { timestamp ->
|
val snapshots = storagePlugin.getCurrentBackupSnapshots().mapNotNull { storedSnapshot ->
|
||||||
snapshotRetriever.getSnapshot(streamKey, timestamp)
|
try {
|
||||||
|
snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
|
||||||
|
} catch (e: GeneralSecurityException) {
|
||||||
|
Log.w(TAG, "Error fetching snapshot $storedSnapshot", e)
|
||||||
|
null
|
||||||
|
}
|
||||||
}
|
}
|
||||||
val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS)
|
val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS)
|
||||||
Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration")
|
Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration")
|
||||||
|
|
|
@ -2,6 +2,7 @@ package org.calyxos.backup.storage.plugin
|
||||||
|
|
||||||
import com.google.protobuf.InvalidProtocolBufferException
|
import com.google.protobuf.InvalidProtocolBufferException
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.backup.BackupSnapshot
|
import org.calyxos.backup.storage.backup.BackupSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
import org.calyxos.backup.storage.restore.readVersion
|
import org.calyxos.backup.storage.restore.readVersion
|
||||||
|
@ -19,9 +20,10 @@ internal class SnapshotRetriever(
|
||||||
GeneralSecurityException::class,
|
GeneralSecurityException::class,
|
||||||
InvalidProtocolBufferException::class,
|
InvalidProtocolBufferException::class,
|
||||||
)
|
)
|
||||||
suspend fun getSnapshot(streamKey: ByteArray, timestamp: Long): BackupSnapshot {
|
suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot {
|
||||||
return storagePlugin.getBackupSnapshotInputStream(timestamp).use { inputStream ->
|
return storagePlugin.getBackupSnapshotInputStream(storedSnapshot).use { inputStream ->
|
||||||
val version = inputStream.readVersion()
|
val version = inputStream.readVersion()
|
||||||
|
val timestamp = storedSnapshot.timestamp
|
||||||
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
|
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
|
||||||
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
|
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
|
||||||
BackupSnapshot.parseFrom(decryptedStream)
|
BackupSnapshot.parseFrom(decryptedStream)
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
package org.calyxos.backup.storage.plugin.saf
|
||||||
|
|
||||||
|
import androidx.documentfile.provider.DocumentFile
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Accessing files and attributes via SAF is costly.
|
||||||
|
* This class caches them to speed up SAF related operations.
|
||||||
|
*/
|
||||||
|
internal class SafCache {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The folder for the current user ID (here "${ANDROID_ID}.sv").
|
||||||
|
*/
|
||||||
|
var currentFolder: DocumentFile? = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Folders containing chunks for backups of the current user ID.
|
||||||
|
*/
|
||||||
|
val backupChunkFolders = HashMap<String, DocumentFile>(CHUNK_FOLDER_COUNT)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Folders containing chunks for restore of a chosen [StoredSnapshot].
|
||||||
|
*/
|
||||||
|
val restoreChunkFolders = HashMap<String, DocumentFile>(CHUNK_FOLDER_COUNT)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Files for each [StoredSnapshot].
|
||||||
|
*/
|
||||||
|
val snapshotFiles = HashMap<StoredSnapshot, DocumentFile>()
|
||||||
|
|
||||||
|
}
|
|
@ -1,9 +1,13 @@
|
||||||
package org.calyxos.backup.storage.plugin.saf
|
package org.calyxos.backup.storage.plugin.saf
|
||||||
|
|
||||||
|
import android.annotation.SuppressLint
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
|
import android.provider.Settings
|
||||||
|
import android.provider.Settings.Secure.ANDROID_ID
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
import androidx.documentfile.provider.DocumentFile
|
import androidx.documentfile.provider.DocumentFile
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.measure
|
import org.calyxos.backup.storage.measure
|
||||||
import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createDirectoryOrThrow
|
import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createDirectoryOrThrow
|
||||||
import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createFileOrThrow
|
import org.calyxos.backup.storage.plugin.saf.DocumentFileExt.createFileOrThrow
|
||||||
|
@ -15,11 +19,12 @@ import java.io.IOException
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.io.OutputStream
|
import java.io.OutputStream
|
||||||
|
|
||||||
|
private val folderRegex = Regex("^[a-f0-9]{16}\\.sv$")
|
||||||
private val chunkFolderRegex = Regex("[a-f0-9]{2}")
|
private val chunkFolderRegex = Regex("[a-f0-9]{2}")
|
||||||
private val chunkRegex = Regex("[a-f0-9]{64}")
|
private val chunkRegex = Regex("[a-f0-9]{64}")
|
||||||
private val snapshotRegex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
|
private val snapshotRegex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
|
||||||
private const val CHUNK_FOLDER_COUNT = 256
|
|
||||||
private const val MIME_TYPE: String = "application/octet-stream"
|
private const val MIME_TYPE: String = "application/octet-stream"
|
||||||
|
internal const val CHUNK_FOLDER_COUNT = 256
|
||||||
|
|
||||||
private const val TAG = "SafStoragePlugin"
|
private const val TAG = "SafStoragePlugin"
|
||||||
|
|
||||||
|
@ -28,12 +33,30 @@ public abstract class SafStoragePlugin(
|
||||||
private val context: Context,
|
private val context: Context,
|
||||||
) : StoragePlugin {
|
) : StoragePlugin {
|
||||||
|
|
||||||
|
private val cache = SafCache()
|
||||||
protected abstract val root: DocumentFile?
|
protected abstract val root: DocumentFile?
|
||||||
|
|
||||||
private val contentResolver = context.contentResolver
|
private val folder: DocumentFile?
|
||||||
|
get() {
|
||||||
|
val root = this.root ?: return null
|
||||||
|
if (cache.currentFolder != null) return cache.currentFolder
|
||||||
|
|
||||||
private val chunkFolders = HashMap<String, DocumentFile>(CHUNK_FOLDER_COUNT)
|
@SuppressLint("HardwareIds")
|
||||||
private val snapshotFiles = HashMap<Long, DocumentFile>()
|
// this is unique to each combination of app-signing key, user, and device
|
||||||
|
// so we don't leak anything by not hashing this and can use it as is
|
||||||
|
val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID)
|
||||||
|
// the folder name is our user ID
|
||||||
|
val folderName = "$androidId.sv"
|
||||||
|
cache.currentFolder = try {
|
||||||
|
root.findFile(folderName) ?: root.createDirectoryOrThrow(folderName)
|
||||||
|
} catch (e: IOException) {
|
||||||
|
Log.e(TAG, "Error creating storage folder $folderName")
|
||||||
|
null
|
||||||
|
}
|
||||||
|
return cache.currentFolder
|
||||||
|
}
|
||||||
|
|
||||||
|
private val contentResolver = context.contentResolver
|
||||||
|
|
||||||
private fun timestampToSnapshot(timestamp: Long): String {
|
private fun timestampToSnapshot(timestamp: Long): String {
|
||||||
return "$timestamp.SeedSnap"
|
return "$timestamp.SeedSnap"
|
||||||
|
@ -41,27 +64,33 @@ public abstract class SafStoragePlugin(
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override suspend fun getAvailableChunkIds(): List<String> {
|
override suspend fun getAvailableChunkIds(): List<String> {
|
||||||
val root = root ?: return emptyList()
|
val folder = folder ?: return emptyList()
|
||||||
val chunkIds = ArrayList<String>()
|
val chunkIds = ArrayList<String>()
|
||||||
populateChunkFolders(root) { file, name ->
|
populateChunkFolders(folder, cache.backupChunkFolders) { file, name ->
|
||||||
if (chunkFolderRegex.matches(name)) {
|
if (chunkFolderRegex.matches(name)) {
|
||||||
chunkIds.addAll(getChunksFromFolder(file))
|
chunkIds.addAll(getChunksFromFolder(file))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Log.e(TAG, "Got ${chunkIds.size} available chunks")
|
Log.i(TAG, "Got ${chunkIds.size} available chunks")
|
||||||
return chunkIds
|
return chunkIds
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Goes through all files in the given [folder] and performs the optional [fileOp] on them.
|
||||||
|
* Afterwards, it creates missing chunk folders, as needed.
|
||||||
|
* Chunk folders will get cached in the given [chunkFolders] for faster access.
|
||||||
|
*/
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
private suspend fun populateChunkFolders(
|
private suspend fun populateChunkFolders(
|
||||||
root: DocumentFile,
|
folder: DocumentFile,
|
||||||
|
chunkFolders: HashMap<String, DocumentFile>,
|
||||||
fileOp: ((DocumentFile, String) -> Unit)? = null
|
fileOp: ((DocumentFile, String) -> Unit)? = null
|
||||||
) {
|
) {
|
||||||
val expectedChunkFolders = (0x00..0xff).map {
|
val expectedChunkFolders = (0x00..0xff).map {
|
||||||
Integer.toHexString(it).padStart(2, '0')
|
Integer.toHexString(it).padStart(2, '0')
|
||||||
}.toHashSet()
|
}.toHashSet()
|
||||||
val duration = measure {
|
val duration = measure {
|
||||||
for (file in root.listFilesBlocking(context)) {
|
for (file in folder.listFilesBlocking(context)) {
|
||||||
val name = file.name ?: continue
|
val name = file.name ?: continue
|
||||||
if (chunkFolderRegex.matches(name)) {
|
if (chunkFolderRegex.matches(name)) {
|
||||||
chunkFolders[name] = file
|
chunkFolders[name] = file
|
||||||
|
@ -70,8 +99,8 @@ public abstract class SafStoragePlugin(
|
||||||
fileOp?.invoke(file, name)
|
fileOp?.invoke(file, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Log.e(TAG, "Retrieving chunk folders took $duration")
|
Log.i(TAG, "Retrieving chunk folders took $duration")
|
||||||
createMissingChunkFolders(root, expectedChunkFolders)
|
createMissingChunkFolders(folder, chunkFolders, expectedChunkFolders)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
|
@ -89,7 +118,11 @@ public abstract class SafStoragePlugin(
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
private fun createMissingChunkFolders(root: DocumentFile, expectedChunkFolders: Set<String>) {
|
private fun createMissingChunkFolders(
|
||||||
|
root: DocumentFile,
|
||||||
|
chunkFolders: HashMap<String, DocumentFile>,
|
||||||
|
expectedChunkFolders: Set<String>
|
||||||
|
) {
|
||||||
val s = expectedChunkFolders.size
|
val s = expectedChunkFolders.size
|
||||||
val duration = measure {
|
val duration = measure {
|
||||||
for ((i, chunkFolderName) in expectedChunkFolders.withIndex()) {
|
for ((i, chunkFolderName) in expectedChunkFolders.withIndex()) {
|
||||||
|
@ -101,13 +134,14 @@ public abstract class SafStoragePlugin(
|
||||||
throw IOException("Only have ${chunkFolders.size} chunk folders.")
|
throw IOException("Only have ${chunkFolders.size} chunk folders.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (s > 0) Log.e(TAG, "Creating $s missing chunk folders took $duration")
|
if (s > 0) Log.i(TAG, "Creating $s missing chunk folders took $duration")
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override fun getChunkOutputStream(chunkId: String): OutputStream {
|
override fun getChunkOutputStream(chunkId: String): OutputStream {
|
||||||
val chunkFolderName = chunkId.substring(0, 2)
|
val chunkFolderName = chunkId.substring(0, 2)
|
||||||
val chunkFolder = chunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId")
|
val chunkFolder =
|
||||||
|
cache.backupChunkFolders[chunkFolderName] ?: error("No folder for chunk $chunkId")
|
||||||
// TODO should we check if it exists first?
|
// TODO should we check if it exists first?
|
||||||
val chunkFile = chunkFolder.createFileOrThrow(chunkId, MIME_TYPE)
|
val chunkFile = chunkFolder.createFileOrThrow(chunkId, MIME_TYPE)
|
||||||
return chunkFile.getOutputStream(context.contentResolver)
|
return chunkFile.getOutputStream(context.contentResolver)
|
||||||
|
@ -115,48 +149,58 @@ public abstract class SafStoragePlugin(
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
|
override fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
|
||||||
val root = root ?: throw IOException()
|
val folder = folder ?: throw IOException()
|
||||||
val name = timestampToSnapshot(timestamp)
|
val name = timestampToSnapshot(timestamp)
|
||||||
// TODO should we check if it exists first?
|
// TODO should we check if it exists first?
|
||||||
val snapshotFile = root.createFileOrThrow(name, MIME_TYPE)
|
val snapshotFile = folder.createFileOrThrow(name, MIME_TYPE)
|
||||||
return snapshotFile.getOutputStream(contentResolver)
|
return snapshotFile.getOutputStream(contentResolver)
|
||||||
}
|
}
|
||||||
|
|
||||||
/************************* Restore *******************************/
|
/************************* Restore *******************************/
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override suspend fun getAvailableBackupSnapshots(): List<Long> {
|
override suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot> {
|
||||||
val root = root ?: return emptyList()
|
val snapshots = ArrayList<StoredSnapshot>()
|
||||||
val snapshots = ArrayList<Long>()
|
|
||||||
|
|
||||||
populateChunkFolders(root) { file, name ->
|
root?.listFilesBlocking(context)?.forEach { folder ->
|
||||||
val match = snapshotRegex.matchEntire(name)
|
val folderName = folder.name ?: ""
|
||||||
if (match != null) {
|
if (!folderRegex.matches(folderName)) return@forEach
|
||||||
val timestamp = match.groupValues[1].toLong()
|
|
||||||
snapshots.add(timestamp)
|
Log.i(TAG, "Checking $folderName for snapshots...")
|
||||||
snapshotFiles[timestamp] = file
|
for (file in folder.listFilesBlocking(context)) {
|
||||||
|
val name = file.name ?: continue
|
||||||
|
val match = snapshotRegex.matchEntire(name)
|
||||||
|
if (match != null) {
|
||||||
|
val timestamp = match.groupValues[1].toLong()
|
||||||
|
val storedSnapshot = StoredSnapshot(folderName, timestamp)
|
||||||
|
snapshots.add(storedSnapshot)
|
||||||
|
cache.snapshotFiles[storedSnapshot] = file
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Log.e(TAG, "Got ${snapshots.size} snapshots while populating chunk folders")
|
Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders")
|
||||||
return snapshots
|
return snapshots
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override suspend fun getBackupSnapshotInputStream(timestamp: Long): InputStream {
|
override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream {
|
||||||
val snapshotFile = snapshotFiles.getOrElse(timestamp) {
|
val timestamp = storedSnapshot.timestamp
|
||||||
root?.findFileBlocking(context, timestampToSnapshot(timestamp))
|
val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) {
|
||||||
|
getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp))
|
||||||
} ?: throw IOException("Could not get file for snapshot $timestamp")
|
} ?: throw IOException("Could not get file for snapshot $timestamp")
|
||||||
return snapshotFile.getInputStream(contentResolver)
|
return snapshotFile.getInputStream(contentResolver)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override suspend fun getChunkInputStream(chunkId: String): InputStream {
|
override suspend fun getChunkInputStream(
|
||||||
if (chunkFolders.size < CHUNK_FOLDER_COUNT) {
|
snapshot: StoredSnapshot,
|
||||||
val root = root ?: throw IOException("Could not get root")
|
chunkId: String
|
||||||
populateChunkFolders(root)
|
): InputStream {
|
||||||
|
if (cache.restoreChunkFolders.size < CHUNK_FOLDER_COUNT) {
|
||||||
|
populateChunkFolders(getFolder(snapshot), cache.restoreChunkFolders)
|
||||||
}
|
}
|
||||||
val chunkFolderName = chunkId.substring(0, 2)
|
val chunkFolderName = chunkId.substring(0, 2)
|
||||||
val chunkFolder = chunkFolders[chunkFolderName]
|
val chunkFolder = cache.restoreChunkFolders[chunkFolderName]
|
||||||
?: throw IOException("No folder for chunk $chunkId")
|
?: throw IOException("No folder for chunk $chunkId")
|
||||||
val chunkFile = chunkFolder.findFileBlocking(context, chunkId)
|
val chunkFile = chunkFolder.findFileBlocking(context, chunkId)
|
||||||
?: throw IOException("No chunk $chunkId")
|
?: throw IOException("No chunk $chunkId")
|
||||||
|
@ -164,23 +208,56 @@ public abstract class SafStoragePlugin(
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
override suspend fun deleteBackupSnapshot(timestamp: Long) {
|
private suspend fun getFolder(storedSnapshot: StoredSnapshot): DocumentFile {
|
||||||
Log.d(TAG, "Deleting snapshot $timestamp")
|
// not cached, because used in several places only once and
|
||||||
val snapshotFile = snapshotFiles.getOrElse(timestamp) {
|
// [getBackupSnapshotInputStream] uses snapshot files cache and
|
||||||
root?.findFileBlocking(context, timestampToSnapshot(timestamp))
|
// [getChunkInputStream] uses restore chunk folders cache
|
||||||
} ?: throw IOException("Could not get file for snapshot $timestamp")
|
return root?.findFileBlocking(context, storedSnapshot.userId)
|
||||||
if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp")
|
?: throw IOException("Could not find snapshot $storedSnapshot")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/************************* Pruning *******************************/
|
||||||
|
|
||||||
|
@Throws(IOException::class)
|
||||||
|
override suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot> {
|
||||||
|
val folder = folder ?: return emptyList()
|
||||||
|
val folderName = folder.name ?: error("Folder suddenly has no more name")
|
||||||
|
val snapshots = ArrayList<StoredSnapshot>()
|
||||||
|
|
||||||
|
populateChunkFolders(folder, cache.backupChunkFolders) { file, name ->
|
||||||
|
val match = snapshotRegex.matchEntire(name)
|
||||||
|
if (match != null) {
|
||||||
|
val timestamp = match.groupValues[1].toLong()
|
||||||
|
val storedSnapshot = StoredSnapshot(folderName, timestamp)
|
||||||
|
snapshots.add(storedSnapshot)
|
||||||
|
cache.snapshotFiles[storedSnapshot] = file
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Log.i(TAG, "Got ${snapshots.size} snapshots while populating chunk folders")
|
||||||
|
return snapshots
|
||||||
|
}
|
||||||
|
|
||||||
|
@Throws(IOException::class)
|
||||||
|
override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) {
|
||||||
|
val timestamp = storedSnapshot.timestamp
|
||||||
|
Log.d(TAG, "Deleting snapshot $timestamp")
|
||||||
|
val snapshotFile = cache.snapshotFiles.getOrElse(storedSnapshot) {
|
||||||
|
getFolder(storedSnapshot).findFileBlocking(context, timestampToSnapshot(timestamp))
|
||||||
|
} ?: throw IOException("Could not get file for snapshot $timestamp")
|
||||||
|
if (!snapshotFile.delete()) throw IOException("Could not delete snapshot $timestamp")
|
||||||
|
cache.snapshotFiles.remove(storedSnapshot)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Throws(IOException::class)
|
||||||
override suspend fun deleteChunks(chunkIds: List<String>) {
|
override suspend fun deleteChunks(chunkIds: List<String>) {
|
||||||
if (chunkFolders.size < CHUNK_FOLDER_COUNT) {
|
if (cache.backupChunkFolders.size < CHUNK_FOLDER_COUNT) {
|
||||||
val root = root ?: throw IOException("Could not get root")
|
val folder = folder ?: throw IOException("Could not get current folder in root")
|
||||||
populateChunkFolders(root)
|
populateChunkFolders(folder, cache.backupChunkFolders)
|
||||||
}
|
}
|
||||||
for (chunkId in chunkIds) {
|
for (chunkId in chunkIds) {
|
||||||
Log.d(TAG, "Deleting chunk $chunkId")
|
Log.d(TAG, "Deleting chunk $chunkId")
|
||||||
val chunkFolderName = chunkId.substring(0, 2)
|
val chunkFolderName = chunkId.substring(0, 2)
|
||||||
val chunkFolder = chunkFolders[chunkFolderName]
|
val chunkFolder = cache.backupChunkFolders[chunkFolderName]
|
||||||
?: throw IOException("No folder for chunk $chunkId")
|
?: throw IOException("No folder for chunk $chunkId")
|
||||||
val chunkFile = chunkFolder.findFileBlocking(context, chunkId)
|
val chunkFile = chunkFolder.findFileBlocking(context, chunkId)
|
||||||
if (chunkFile == null) {
|
if (chunkFile == null) {
|
||||||
|
|
|
@ -3,6 +3,7 @@ package org.calyxos.backup.storage.prune
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
import org.calyxos.backup.storage.api.BackupObserver
|
import org.calyxos.backup.storage.api.BackupObserver
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
import org.calyxos.backup.storage.db.Db
|
import org.calyxos.backup.storage.db.Db
|
||||||
import org.calyxos.backup.storage.measure
|
import org.calyxos.backup.storage.measure
|
||||||
|
@ -33,15 +34,15 @@ internal class Pruner(
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
suspend fun prune(backupObserver: BackupObserver?) {
|
suspend fun prune(backupObserver: BackupObserver?) {
|
||||||
val duration = measure {
|
val duration = measure {
|
||||||
val timestamps = storagePlugin.getAvailableBackupSnapshots()
|
val storedSnapshots = storagePlugin.getCurrentBackupSnapshots()
|
||||||
val toDelete = retentionManager.getSnapshotsToDelete(timestamps)
|
val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots)
|
||||||
backupObserver?.onPruneStart(toDelete)
|
backupObserver?.onPruneStart(toDelete.map { it.timestamp })
|
||||||
for (timestamp in toDelete) {
|
for (snapshot in toDelete) {
|
||||||
try {
|
try {
|
||||||
pruneSnapshot(timestamp, backupObserver)
|
pruneSnapshot(snapshot, backupObserver)
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
Log.e(TAG, "Error pruning $timestamp", e)
|
Log.e(TAG, "Error pruning $snapshot", e)
|
||||||
backupObserver?.onPruneError(timestamp, e)
|
backupObserver?.onPruneError(snapshot.timestamp, e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -50,12 +51,15 @@ internal class Pruner(
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class, GeneralSecurityException::class)
|
@Throws(IOException::class, GeneralSecurityException::class)
|
||||||
private suspend fun pruneSnapshot(timestamp: Long, backupObserver: BackupObserver?) {
|
private suspend fun pruneSnapshot(
|
||||||
val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp)
|
storedSnapshot: StoredSnapshot,
|
||||||
|
backupObserver: BackupObserver?
|
||||||
|
) {
|
||||||
|
val snapshot = snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
|
||||||
val chunks = HashSet<String>()
|
val chunks = HashSet<String>()
|
||||||
snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) }
|
snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) }
|
||||||
snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) }
|
snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) }
|
||||||
storagePlugin.deleteBackupSnapshot(timestamp)
|
storagePlugin.deleteBackupSnapshot(storedSnapshot)
|
||||||
db.applyInParts(chunks) {
|
db.applyInParts(chunks) {
|
||||||
chunksCache.decrementRefCount(it)
|
chunksCache.decrementRefCount(it)
|
||||||
}
|
}
|
||||||
|
@ -68,7 +72,7 @@ internal class Pruner(
|
||||||
size += it.size
|
size += it.size
|
||||||
it.id
|
it.id
|
||||||
}
|
}
|
||||||
backupObserver?.onPruneSnapshot(timestamp, chunkIdsToDelete.size, size)
|
backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size)
|
||||||
storagePlugin.deleteChunks(chunkIdsToDelete)
|
storagePlugin.deleteChunks(chunkIdsToDelete)
|
||||||
chunksCache.deleteChunks(cachedChunksToDelete)
|
chunksCache.deleteChunks(cachedChunksToDelete)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ package org.calyxos.backup.storage.prune
|
||||||
|
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
import org.calyxos.backup.storage.api.SnapshotRetention
|
import org.calyxos.backup.storage.api.SnapshotRetention
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.time.LocalDate
|
import java.time.LocalDate
|
||||||
import java.time.temporal.ChronoField
|
import java.time.temporal.ChronoField
|
||||||
|
@ -48,37 +49,37 @@ internal class RetentionManager(private val context: Context) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Takes a list of snapshot timestamps and returns a list of those
|
* Takes a list of [StoredSnapshot]s and returns a list of those
|
||||||
* that can be deleted according to the current snapshot retention policy.
|
* that can be deleted according to the current snapshot retention policy.
|
||||||
*/
|
*/
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class)
|
||||||
fun getSnapshotsToDelete(snapshotTimestamps: List<Long>): List<Long> {
|
fun getSnapshotsToDelete(storedSnapshots: List<StoredSnapshot>): List<StoredSnapshot> {
|
||||||
val retention = getSnapshotRetention()
|
val retention = getSnapshotRetention()
|
||||||
val dates = snapshotTimestamps.sortedDescending().map {
|
val datePairs = storedSnapshots.sortedByDescending { it.timestamp }.map { s ->
|
||||||
Pair(it, LocalDate.ofEpochDay(it / 1000 / 60 / 60 / 24))
|
Pair(s, LocalDate.ofEpochDay(s.timestamp / 1000 / 60 / 60 / 24))
|
||||||
}
|
}
|
||||||
val toKeep = HashSet<Long>()
|
val toKeep = HashSet<StoredSnapshot>()
|
||||||
toKeep += getToKeep(dates, retention.daily)
|
toKeep += getToKeep(datePairs, retention.daily)
|
||||||
toKeep += getToKeep(dates, retention.weekly) { temporal: Temporal ->
|
toKeep += getToKeep(datePairs, retention.weekly) { temporal: Temporal ->
|
||||||
temporal.with(ChronoField.DAY_OF_WEEK, 1)
|
temporal.with(ChronoField.DAY_OF_WEEK, 1)
|
||||||
}
|
}
|
||||||
toKeep += getToKeep(dates, retention.monthly, firstDayOfMonth())
|
toKeep += getToKeep(datePairs, retention.monthly, firstDayOfMonth())
|
||||||
toKeep += getToKeep(dates, retention.yearly, firstDayOfYear())
|
toKeep += getToKeep(datePairs, retention.yearly, firstDayOfYear())
|
||||||
return snapshotTimestamps - toKeep
|
return storedSnapshots - toKeep
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun getToKeep(
|
private fun getToKeep(
|
||||||
pairs: List<Pair<Long, LocalDate>>,
|
pairs: List<Pair<StoredSnapshot, LocalDate>>,
|
||||||
keep: Int,
|
keep: Int,
|
||||||
temporalAdjuster: TemporalAdjuster? = null,
|
temporalAdjuster: TemporalAdjuster? = null,
|
||||||
): List<Long> {
|
): List<StoredSnapshot> {
|
||||||
val toKeep = ArrayList<Long>()
|
val toKeep = ArrayList<StoredSnapshot>()
|
||||||
if (keep == 0) return toKeep
|
if (keep == 0) return toKeep
|
||||||
var last: LocalDate? = null
|
var last: LocalDate? = null
|
||||||
for ((timestamp, date) in pairs) {
|
for ((snapshot, date) in pairs) {
|
||||||
val period = if (temporalAdjuster == null) date else date.with(temporalAdjuster)
|
val period = if (temporalAdjuster == null) date else date.with(temporalAdjuster)
|
||||||
if (period != last) {
|
if (period != last) {
|
||||||
toKeep.add(timestamp)
|
toKeep.add(snapshot)
|
||||||
if (toKeep.size >= keep) break
|
if (toKeep.size >= keep) break
|
||||||
last = period
|
last = period
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ package org.calyxos.backup.storage.restore
|
||||||
|
|
||||||
import org.calyxos.backup.storage.api.RestoreObserver
|
import org.calyxos.backup.storage.api.RestoreObserver
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
|
@ -19,10 +20,11 @@ internal abstract class AbstractChunkRestore(
|
||||||
@Throws(IOException::class, GeneralSecurityException::class)
|
@Throws(IOException::class, GeneralSecurityException::class)
|
||||||
protected suspend fun getAndDecryptChunk(
|
protected suspend fun getAndDecryptChunk(
|
||||||
version: Int,
|
version: Int,
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
chunkId: String,
|
chunkId: String,
|
||||||
streamReader: suspend (InputStream) -> Unit,
|
streamReader: suspend (InputStream) -> Unit,
|
||||||
) {
|
) {
|
||||||
storagePlugin.getChunkInputStream(chunkId).use { inputStream ->
|
storagePlugin.getChunkInputStream(storedSnapshot, chunkId).use { inputStream ->
|
||||||
inputStream.readVersion(version)
|
inputStream.readVersion(version)
|
||||||
val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte())
|
val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte())
|
||||||
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
|
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
|
||||||
|
|
|
@ -4,6 +4,7 @@ import android.content.Context
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
import org.calyxos.backup.storage.api.RestoreObserver
|
import org.calyxos.backup.storage.api.RestoreObserver
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.io.FileInputStream
|
import java.io.FileInputStream
|
||||||
|
@ -26,6 +27,7 @@ internal class MultiChunkRestore(
|
||||||
|
|
||||||
suspend fun restore(
|
suspend fun restore(
|
||||||
version: Int,
|
version: Int,
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
chunkMap: Map<String, RestorableChunk>,
|
chunkMap: Map<String, RestorableChunk>,
|
||||||
files: Collection<RestorableFile>,
|
files: Collection<RestorableFile>,
|
||||||
observer: RestoreObserver?,
|
observer: RestoreObserver?,
|
||||||
|
@ -34,7 +36,7 @@ internal class MultiChunkRestore(
|
||||||
files.forEach { file ->
|
files.forEach { file ->
|
||||||
try {
|
try {
|
||||||
restoreFile(file, observer, "L") { outputStream ->
|
restoreFile(file, observer, "L") { outputStream ->
|
||||||
writeChunks(version, file, chunkMap, outputStream)
|
writeChunks(version, storedSnapshot, file, chunkMap, outputStream)
|
||||||
}
|
}
|
||||||
restoredFiles++
|
restoredFiles++
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
|
@ -49,6 +51,7 @@ internal class MultiChunkRestore(
|
||||||
@Throws(IOException::class, GeneralSecurityException::class)
|
@Throws(IOException::class, GeneralSecurityException::class)
|
||||||
private suspend fun writeChunks(
|
private suspend fun writeChunks(
|
||||||
version: Int,
|
version: Int,
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
file: RestorableFile,
|
file: RestorableFile,
|
||||||
chunkMap: Map<String, RestorableChunk>,
|
chunkMap: Map<String, RestorableChunk>,
|
||||||
outputStream: OutputStream,
|
outputStream: OutputStream,
|
||||||
|
@ -61,8 +64,11 @@ internal class MultiChunkRestore(
|
||||||
bytes += decryptedStream.copyTo(outputStream)
|
bytes += decryptedStream.copyTo(outputStream)
|
||||||
}
|
}
|
||||||
val isCached = isCached(chunkId)
|
val isCached = isCached(chunkId)
|
||||||
if (isCached || otherFiles.size > 1) getAndCacheChunk(version, chunkId, chunkWriter)
|
if (isCached || otherFiles.size > 1) {
|
||||||
else getAndDecryptChunk(version, chunkId, chunkWriter)
|
getAndCacheChunk(version, storedSnapshot, chunkId, chunkWriter)
|
||||||
|
} else {
|
||||||
|
getAndDecryptChunk(version, storedSnapshot, chunkId, chunkWriter)
|
||||||
|
}
|
||||||
|
|
||||||
otherFiles.remove(file)
|
otherFiles.remove(file)
|
||||||
if (isCached && otherFiles.isEmpty()) removeCachedChunk(chunkId)
|
if (isCached && otherFiles.isEmpty()) removeCachedChunk(chunkId)
|
||||||
|
@ -77,13 +83,14 @@ internal class MultiChunkRestore(
|
||||||
@Throws(IOException::class, GeneralSecurityException::class)
|
@Throws(IOException::class, GeneralSecurityException::class)
|
||||||
private suspend fun getAndCacheChunk(
|
private suspend fun getAndCacheChunk(
|
||||||
version: Int,
|
version: Int,
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
chunkId: String,
|
chunkId: String,
|
||||||
streamReader: suspend (InputStream) -> Unit,
|
streamReader: suspend (InputStream) -> Unit,
|
||||||
) {
|
) {
|
||||||
val file = getChunkCacheFile(chunkId)
|
val file = getChunkCacheFile(chunkId)
|
||||||
if (!file.isFile) {
|
if (!file.isFile) {
|
||||||
FileOutputStream(file).use { outputStream ->
|
FileOutputStream(file).use { outputStream ->
|
||||||
getAndDecryptChunk(version, chunkId) { decryptedStream ->
|
getAndDecryptChunk(version, storedSnapshot, chunkId) { decryptedStream ->
|
||||||
decryptedStream.copyTo(outputStream)
|
decryptedStream.copyTo(outputStream)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ import org.calyxos.backup.storage.api.RestoreObserver
|
||||||
import org.calyxos.backup.storage.api.SnapshotItem
|
import org.calyxos.backup.storage.api.SnapshotItem
|
||||||
import org.calyxos.backup.storage.api.SnapshotResult
|
import org.calyxos.backup.storage.api.SnapshotResult
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.backup.Backup
|
import org.calyxos.backup.storage.backup.Backup
|
||||||
import org.calyxos.backup.storage.backup.BackupSnapshot
|
import org.calyxos.backup.storage.backup.BackupSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
|
@ -46,8 +47,10 @@ internal class Restore(
|
||||||
val numSnapshots: Int
|
val numSnapshots: Int
|
||||||
val time = measure {
|
val time = measure {
|
||||||
val list = try {
|
val list = try {
|
||||||
storagePlugin.getAvailableBackupSnapshots().sortedDescending().map {
|
storagePlugin.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot ->
|
||||||
SnapshotItem(it, null)
|
storedSnapshot.timestamp
|
||||||
|
}.map { storedSnapshot ->
|
||||||
|
SnapshotItem(storedSnapshot, null)
|
||||||
}.toMutableList()
|
}.toMutableList()
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
Log.e("TAG", "Error retrieving snapshots", e)
|
Log.e("TAG", "Error retrieving snapshots", e)
|
||||||
|
@ -61,7 +64,9 @@ internal class Restore(
|
||||||
while (iterator.hasNext()) {
|
while (iterator.hasNext()) {
|
||||||
val oldItem = iterator.next()
|
val oldItem = iterator.next()
|
||||||
val item = try {
|
val item = try {
|
||||||
oldItem.copy(snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.time))
|
oldItem.copy(
|
||||||
|
snapshot = snapshotRetriever.getSnapshot(streamKey, oldItem.storedSnapshot)
|
||||||
|
)
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
Log.e("TAG", "Error retrieving snapshot ${oldItem.time}", e)
|
Log.e("TAG", "Error retrieving snapshot ${oldItem.time}", e)
|
||||||
continue
|
continue
|
||||||
|
@ -73,15 +78,15 @@ internal class Restore(
|
||||||
Log.e(TAG, "Decrypting and parsing $numSnapshots snapshots took $time")
|
Log.e(TAG, "Decrypting and parsing $numSnapshots snapshots took $time")
|
||||||
}
|
}
|
||||||
|
|
||||||
@Throws(IOException::class, GeneralSecurityException::class)
|
|
||||||
suspend fun restoreBackupSnapshot(timestamp: Long, observer: RestoreObserver?) {
|
|
||||||
val snapshot = snapshotRetriever.getSnapshot(streamKey, timestamp)
|
|
||||||
restoreBackupSnapshot(snapshot, observer)
|
|
||||||
}
|
|
||||||
|
|
||||||
@OptIn(ExperimentalTime::class)
|
@OptIn(ExperimentalTime::class)
|
||||||
@Throws(IOException::class)
|
@Throws(IOException::class, GeneralSecurityException::class)
|
||||||
suspend fun restoreBackupSnapshot(snapshot: BackupSnapshot, observer: RestoreObserver?) {
|
suspend fun restoreBackupSnapshot(
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
|
optionalSnapshot: BackupSnapshot? = null,
|
||||||
|
observer: RestoreObserver? = null,
|
||||||
|
) {
|
||||||
|
val snapshot = optionalSnapshot ?: snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
|
||||||
|
|
||||||
val filesTotal = snapshot.mediaFilesList.size + snapshot.documentFilesList.size
|
val filesTotal = snapshot.mediaFilesList.size + snapshot.documentFilesList.size
|
||||||
val totalSize =
|
val totalSize =
|
||||||
snapshot.mediaFilesList.sumOf { it.size } + snapshot.documentFilesList.sumOf { it.size }
|
snapshot.mediaFilesList.sumOf { it.size } + snapshot.documentFilesList.sumOf { it.size }
|
||||||
|
@ -91,19 +96,30 @@ internal class Restore(
|
||||||
val version = snapshot.version
|
val version = snapshot.version
|
||||||
var restoredFiles = 0
|
var restoredFiles = 0
|
||||||
val smallFilesDuration = measure {
|
val smallFilesDuration = measure {
|
||||||
restoredFiles += zipChunkRestore.restore(version, split.zipChunks, observer)
|
restoredFiles += zipChunkRestore.restore(
|
||||||
|
version,
|
||||||
|
storedSnapshot,
|
||||||
|
split.zipChunks,
|
||||||
|
observer,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Log.e(TAG, "Restoring ${split.zipChunks.size} zip chunks took $smallFilesDuration.")
|
Log.e(TAG, "Restoring ${split.zipChunks.size} zip chunks took $smallFilesDuration.")
|
||||||
val singleChunkDuration = measure {
|
val singleChunkDuration = measure {
|
||||||
restoredFiles += singleChunkRestore.restore(version, split.singleChunks, observer)
|
restoredFiles += singleChunkRestore.restore(
|
||||||
|
version,
|
||||||
|
storedSnapshot,
|
||||||
|
split.singleChunks,
|
||||||
|
observer,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Log.e(TAG, "Restoring ${split.singleChunks.size} single chunks took $singleChunkDuration.")
|
Log.e(TAG, "Restoring ${split.singleChunks.size} single chunks took $singleChunkDuration.")
|
||||||
val multiChunkDuration = measure {
|
val multiChunkDuration = measure {
|
||||||
restoredFiles += multiChunkRestore.restore(
|
restoredFiles += multiChunkRestore.restore(
|
||||||
version,
|
version,
|
||||||
|
storedSnapshot,
|
||||||
split.multiChunkMap,
|
split.multiChunkMap,
|
||||||
split.multiChunkFiles,
|
split.multiChunkFiles,
|
||||||
observer
|
observer,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
Log.e(TAG, "Restoring ${split.multiChunkFiles.size} multi chunks took $multiChunkDuration.")
|
Log.e(TAG, "Restoring ${split.multiChunkFiles.size} multi chunks took $multiChunkDuration.")
|
||||||
|
|
|
@ -8,20 +8,24 @@ import kotlinx.coroutines.GlobalScope
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import org.calyxos.backup.storage.api.RestoreObserver
|
import org.calyxos.backup.storage.api.RestoreObserver
|
||||||
import org.calyxos.backup.storage.api.StorageBackup
|
import org.calyxos.backup.storage.api.StorageBackup
|
||||||
import org.calyxos.backup.storage.backup.BackupSnapshot
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START
|
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_TIMESTAMP_START
|
||||||
|
import org.calyxos.backup.storage.restore.RestoreService.Companion.EXTRA_USER_ID
|
||||||
import org.calyxos.backup.storage.ui.NOTIFICATION_ID_RESTORE
|
import org.calyxos.backup.storage.ui.NOTIFICATION_ID_RESTORE
|
||||||
import org.calyxos.backup.storage.ui.Notifications
|
import org.calyxos.backup.storage.ui.Notifications
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start to trigger restore as a foreground service. Ensure that you provide the snapshot
|
* Start to trigger restore as a foreground service. Ensure that you provide the snapshot
|
||||||
* to be restored with [Intent.putExtra] as a [Long] in [EXTRA_TIMESTAMP_START].
|
* to be restored with [Intent.putExtra]:
|
||||||
* See [BackupSnapshot.getTimeStart].
|
* * the user ID of the snapshot as a [String] in [EXTRA_USER_ID]
|
||||||
|
* * the snapshot's timestamp as a [Long] in [EXTRA_TIMESTAMP_START].
|
||||||
|
* See [BackupSnapshot.getTimeStart].
|
||||||
*/
|
*/
|
||||||
public abstract class RestoreService : Service() {
|
public abstract class RestoreService : Service() {
|
||||||
|
|
||||||
public companion object {
|
public companion object {
|
||||||
private const val TAG = "RestoreService"
|
private const val TAG = "RestoreService"
|
||||||
|
public const val EXTRA_USER_ID: String = "userId"
|
||||||
public const val EXTRA_TIMESTAMP_START: String = "timestamp"
|
public const val EXTRA_TIMESTAMP_START: String = "timestamp"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,13 +35,15 @@ public abstract class RestoreService : Service() {
|
||||||
|
|
||||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||||
Log.d(TAG, "onStartCommand $intent $flags $startId")
|
Log.d(TAG, "onStartCommand $intent $flags $startId")
|
||||||
val timestamp = intent?.getLongExtra(EXTRA_TIMESTAMP_START, -1)
|
val userId = intent?.getStringExtra(EXTRA_USER_ID) ?: error("No user ID in intent: $intent")
|
||||||
if (timestamp == null || timestamp < 0) error("No timestamp in intent: $intent")
|
val timestamp = intent.getLongExtra(EXTRA_TIMESTAMP_START, -1)
|
||||||
|
if (timestamp < 0) error("No timestamp in intent: $intent")
|
||||||
|
val storedSnapshot = StoredSnapshot(userId, timestamp)
|
||||||
|
|
||||||
startForeground(NOTIFICATION_ID_RESTORE, n.getRestoreNotification())
|
startForeground(NOTIFICATION_ID_RESTORE, n.getRestoreNotification())
|
||||||
GlobalScope.launch {
|
GlobalScope.launch {
|
||||||
// TODO offer a way to try again if failed, or do an automatic retry here
|
// TODO offer a way to try again if failed, or do an automatic retry here
|
||||||
storageBackup.restoreBackupSnapshot(timestamp, restoreObserver)
|
storageBackup.restoreBackupSnapshot(storedSnapshot, null, restoreObserver)
|
||||||
stopSelf(startId)
|
stopSelf(startId)
|
||||||
}
|
}
|
||||||
return START_STICKY_COMPATIBILITY
|
return START_STICKY_COMPATIBILITY
|
||||||
|
|
|
@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
import org.calyxos.backup.storage.api.RestoreObserver
|
import org.calyxos.backup.storage.api.RestoreObserver
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
|
|
||||||
private const val TAG = "SingleChunkRestore"
|
private const val TAG = "SingleChunkRestore"
|
||||||
|
@ -17,6 +18,7 @@ internal class SingleChunkRestore(
|
||||||
|
|
||||||
suspend fun restore(
|
suspend fun restore(
|
||||||
version: Int,
|
version: Int,
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
chunks: Collection<RestorableChunk>,
|
chunks: Collection<RestorableChunk>,
|
||||||
observer: RestoreObserver?
|
observer: RestoreObserver?
|
||||||
): Int {
|
): Int {
|
||||||
|
@ -25,7 +27,7 @@ internal class SingleChunkRestore(
|
||||||
check(chunk.files.size == 1)
|
check(chunk.files.size == 1)
|
||||||
val file = chunk.files[0]
|
val file = chunk.files[0]
|
||||||
try {
|
try {
|
||||||
getAndDecryptChunk(version, chunk.chunkId) { decryptedStream ->
|
getAndDecryptChunk(version, storedSnapshot, chunk.chunkId) { decryptedStream ->
|
||||||
restoreFile(file, observer, "M") { outputStream ->
|
restoreFile(file, observer, "M") { outputStream ->
|
||||||
decryptedStream.copyTo(outputStream)
|
decryptedStream.copyTo(outputStream)
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package org.calyxos.backup.storage.restore
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
import org.calyxos.backup.storage.api.RestoreObserver
|
import org.calyxos.backup.storage.api.RestoreObserver
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.crypto.StreamCrypto
|
import org.calyxos.backup.storage.crypto.StreamCrypto
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
|
@ -24,13 +25,14 @@ internal class ZipChunkRestore(
|
||||||
*/
|
*/
|
||||||
suspend fun restore(
|
suspend fun restore(
|
||||||
version: Int,
|
version: Int,
|
||||||
|
storedSnapshot: StoredSnapshot,
|
||||||
zipChunks: Collection<RestorableChunk>,
|
zipChunks: Collection<RestorableChunk>,
|
||||||
observer: RestoreObserver?
|
observer: RestoreObserver?
|
||||||
): Int {
|
): Int {
|
||||||
var restoredFiles = 0
|
var restoredFiles = 0
|
||||||
zipChunks.forEach { zipChunk ->
|
zipChunks.forEach { zipChunk ->
|
||||||
try {
|
try {
|
||||||
getAndDecryptChunk(version, zipChunk.chunkId) { decryptedStream ->
|
getAndDecryptChunk(version, storedSnapshot, zipChunk.chunkId) { decryptedStream ->
|
||||||
restoredFiles += restoreZipChunk(zipChunk, decryptedStream, observer)
|
restoredFiles += restoreZipChunk(zipChunk, decryptedStream, observer)
|
||||||
}
|
}
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
|
|
|
@ -19,6 +19,7 @@ import kotlinx.coroutines.flow.toList
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import org.calyxos.backup.storage.api.SnapshotResult
|
import org.calyxos.backup.storage.api.SnapshotResult
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.backup.Backup
|
import org.calyxos.backup.storage.backup.Backup
|
||||||
import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX
|
import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX
|
||||||
import org.calyxos.backup.storage.backup.Backup.Companion.SMALL_FILE_SIZE_MAX
|
import org.calyxos.backup.storage.backup.Backup.Companion.SMALL_FILE_SIZE_MAX
|
||||||
|
@ -173,14 +174,16 @@ internal class BackupRestoreTest {
|
||||||
|
|
||||||
// RESTORE
|
// RESTORE
|
||||||
|
|
||||||
|
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured)
|
||||||
|
|
||||||
val smallFileMOutputStream = ByteArrayOutputStream()
|
val smallFileMOutputStream = ByteArrayOutputStream()
|
||||||
val smallFileDOutputStream = ByteArrayOutputStream()
|
val smallFileDOutputStream = ByteArrayOutputStream()
|
||||||
val fileMOutputStream = ByteArrayOutputStream()
|
val fileMOutputStream = ByteArrayOutputStream()
|
||||||
val fileDOutputStream = ByteArrayOutputStream()
|
val fileDOutputStream = ByteArrayOutputStream()
|
||||||
|
|
||||||
coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured)
|
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured)
|
plugin.getBackupSnapshotInputStream(storedSnapshot)
|
||||||
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
|
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
|
||||||
|
|
||||||
// retrieve snapshots
|
// retrieve snapshots
|
||||||
|
@ -197,14 +200,14 @@ internal class BackupRestoreTest {
|
||||||
|
|
||||||
// pipe chunks back in
|
// pipe chunks back in
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(cachedFiles[0].chunks[0])
|
plugin.getChunkInputStream(storedSnapshot, cachedFiles[0].chunks[0])
|
||||||
} returns ByteArrayInputStream(zipChunkOutputStream.toByteArray())
|
} returns ByteArrayInputStream(zipChunkOutputStream.toByteArray())
|
||||||
// cachedFiles[0].chunks[1] is in previous zipChunk
|
// cachedFiles[0].chunks[1] is in previous zipChunk
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(cachedFiles[2].chunks[0])
|
plugin.getChunkInputStream(storedSnapshot, cachedFiles[2].chunks[0])
|
||||||
} returns ByteArrayInputStream(mOutputStream.toByteArray())
|
} returns ByteArrayInputStream(mOutputStream.toByteArray())
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(cachedFiles[3].chunks[0])
|
plugin.getChunkInputStream(storedSnapshot, cachedFiles[3].chunks[0])
|
||||||
} returns ByteArrayInputStream(dOutputStream.toByteArray())
|
} returns ByteArrayInputStream(dOutputStream.toByteArray())
|
||||||
|
|
||||||
// provide file output streams for restore
|
// provide file output streams for restore
|
||||||
|
@ -217,7 +220,7 @@ internal class BackupRestoreTest {
|
||||||
val fileDRestorable = getRestorableFileD(fileD, snapshot)
|
val fileDRestorable = getRestorableFileD(fileD, snapshot)
|
||||||
expectRestoreFile(fileDRestorable, fileDOutputStream)
|
expectRestoreFile(fileDRestorable, fileDOutputStream)
|
||||||
|
|
||||||
restore.restoreBackupSnapshot(snapshot, null)
|
restore.restoreBackupSnapshot(storedSnapshot, snapshot, null)
|
||||||
|
|
||||||
// restored files match backed up files exactly
|
// restored files match backed up files exactly
|
||||||
assertArrayEquals(smallFileMBytes, smallFileMOutputStream.toByteArray())
|
assertArrayEquals(smallFileMBytes, smallFileMOutputStream.toByteArray())
|
||||||
|
@ -337,9 +340,11 @@ internal class BackupRestoreTest {
|
||||||
|
|
||||||
// RESTORE
|
// RESTORE
|
||||||
|
|
||||||
coEvery { plugin.getAvailableBackupSnapshots() } returns listOf(snapshotTimestamp.captured)
|
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured)
|
||||||
|
|
||||||
|
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getBackupSnapshotInputStream(snapshotTimestamp.captured)
|
plugin.getBackupSnapshotInputStream(storedSnapshot)
|
||||||
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
|
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
|
||||||
|
|
||||||
// retrieve snapshots
|
// retrieve snapshots
|
||||||
|
@ -354,19 +359,27 @@ internal class BackupRestoreTest {
|
||||||
// pipe chunks back in
|
// pipe chunks back in
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3")
|
storedSnapshot,
|
||||||
|
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
|
||||||
|
)
|
||||||
} returns ByteArrayInputStream(id040f32.toByteArray())
|
} returns ByteArrayInputStream(id040f32.toByteArray())
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29")
|
storedSnapshot,
|
||||||
|
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
|
||||||
|
)
|
||||||
} returns ByteArrayInputStream(id901fbc.toByteArray())
|
} returns ByteArrayInputStream(id901fbc.toByteArray())
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d")
|
storedSnapshot,
|
||||||
|
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
|
||||||
|
)
|
||||||
} returns ByteArrayInputStream(id5adea3.toByteArray())
|
} returns ByteArrayInputStream(id5adea3.toByteArray())
|
||||||
coEvery {
|
coEvery {
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67")
|
storedSnapshot,
|
||||||
|
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
|
||||||
|
)
|
||||||
} returns ByteArrayInputStream(id40d00c.toByteArray())
|
} returns ByteArrayInputStream(id40d00c.toByteArray())
|
||||||
|
|
||||||
// provide file output streams for restore
|
// provide file output streams for restore
|
||||||
|
@ -375,7 +388,7 @@ internal class BackupRestoreTest {
|
||||||
val file2Restorable = getRestorableFileD(file2, snapshot)
|
val file2Restorable = getRestorableFileD(file2, snapshot)
|
||||||
expectRestoreFile(file2Restorable, file2OutputStream)
|
expectRestoreFile(file2Restorable, file2OutputStream)
|
||||||
|
|
||||||
restore.restoreBackupSnapshot(snapshot, null)
|
restore.restoreBackupSnapshot(storedSnapshot, snapshot, null)
|
||||||
|
|
||||||
// restored files match backed up files exactly
|
// restored files match backed up files exactly
|
||||||
assertArrayEquals(file1Bytes, file1OutputStream.toByteArray())
|
assertArrayEquals(file1Bytes, file1OutputStream.toByteArray())
|
||||||
|
@ -384,13 +397,21 @@ internal class BackupRestoreTest {
|
||||||
// chunks were only read from storage once
|
// chunks were only read from storage once
|
||||||
coVerify(exactly = 1) {
|
coVerify(exactly = 1) {
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3")
|
storedSnapshot,
|
||||||
|
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
|
||||||
|
)
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29")
|
storedSnapshot,
|
||||||
|
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
|
||||||
|
)
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d")
|
storedSnapshot,
|
||||||
|
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
|
||||||
|
)
|
||||||
plugin.getChunkInputStream(
|
plugin.getChunkInputStream(
|
||||||
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67")
|
storedSnapshot,
|
||||||
|
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ import io.mockk.mockk
|
||||||
import io.mockk.slot
|
import io.mockk.slot
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.db.CachedChunk
|
import org.calyxos.backup.storage.db.CachedChunk
|
||||||
import org.calyxos.backup.storage.db.ChunksCache
|
import org.calyxos.backup.storage.db.ChunksCache
|
||||||
import org.calyxos.backup.storage.db.Db
|
import org.calyxos.backup.storage.db.Db
|
||||||
|
@ -56,7 +57,9 @@ internal class ChunksCacheRepopulaterTest {
|
||||||
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4))
|
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4))
|
||||||
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk5))
|
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk5))
|
||||||
.build()
|
.build()
|
||||||
val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart)
|
val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart)
|
||||||
|
val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart)
|
||||||
|
val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2)
|
||||||
val cachedChunks = listOf(
|
val cachedChunks = listOf(
|
||||||
CachedChunk(chunk1, 2, 0),
|
CachedChunk(chunk1, 2, 0),
|
||||||
CachedChunk(chunk2, 2, 0),
|
CachedChunk(chunk2, 2, 0),
|
||||||
|
@ -64,12 +67,12 @@ internal class ChunksCacheRepopulaterTest {
|
||||||
) // chunk3 is not referenced and should get deleted
|
) // chunk3 is not referenced and should get deleted
|
||||||
val cachedChunksSlot = slot<Collection<CachedChunk>>()
|
val cachedChunksSlot = slot<Collection<CachedChunk>>()
|
||||||
|
|
||||||
coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps
|
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots
|
||||||
coEvery {
|
coEvery {
|
||||||
snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart)
|
snapshotRetriever.getSnapshot(streamKey, storedSnapshot1)
|
||||||
} returns snapshot1
|
} returns snapshot1
|
||||||
coEvery {
|
coEvery {
|
||||||
snapshotRetriever.getSnapshot(streamKey, snapshot2.timeStart)
|
snapshotRetriever.getSnapshot(streamKey, storedSnapshot2)
|
||||||
} returns snapshot2
|
} returns snapshot2
|
||||||
every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs
|
every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs
|
||||||
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs
|
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs
|
||||||
|
|
|
@ -8,6 +8,7 @@ import io.mockk.mockk
|
||||||
import io.mockk.slot
|
import io.mockk.slot
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import org.calyxos.backup.storage.api.StoragePlugin
|
import org.calyxos.backup.storage.api.StoragePlugin
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
import org.calyxos.backup.storage.backup.BackupDocumentFile
|
import org.calyxos.backup.storage.backup.BackupDocumentFile
|
||||||
import org.calyxos.backup.storage.backup.BackupMediaFile
|
import org.calyxos.backup.storage.backup.BackupMediaFile
|
||||||
import org.calyxos.backup.storage.backup.BackupSnapshot
|
import org.calyxos.backup.storage.backup.BackupSnapshot
|
||||||
|
@ -66,18 +67,20 @@ internal class PrunerTest {
|
||||||
.addMediaFiles(BackupMediaFile.newBuilder().addChunkIds(chunk2))
|
.addMediaFiles(BackupMediaFile.newBuilder().addChunkIds(chunk2))
|
||||||
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4))
|
.addDocumentFiles(BackupDocumentFile.newBuilder().addChunkIds(chunk4))
|
||||||
.build()
|
.build()
|
||||||
val snapshotTimestamps = listOf(snapshot1.timeStart, snapshot2.timeStart)
|
val storedSnapshot1 = StoredSnapshot("foo", snapshot1.timeStart)
|
||||||
|
val storedSnapshot2 = StoredSnapshot("bar", snapshot2.timeStart)
|
||||||
|
val storedSnapshots = listOf(storedSnapshot1, storedSnapshot2)
|
||||||
val expectedChunks = listOf(chunk1, chunk2, chunk3)
|
val expectedChunks = listOf(chunk1, chunk2, chunk3)
|
||||||
val actualChunks = slot<Collection<String>>()
|
val actualChunks = slot<Collection<String>>()
|
||||||
val actualChunks2 = slot<Collection<String>>()
|
val actualChunks2 = slot<Collection<String>>()
|
||||||
val cachedChunk3 = CachedChunk(chunk3, 0, 0)
|
val cachedChunk3 = CachedChunk(chunk3, 0, 0)
|
||||||
|
|
||||||
coEvery { plugin.getAvailableBackupSnapshots() } returns snapshotTimestamps
|
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots
|
||||||
every {
|
every {
|
||||||
retentionManager.getSnapshotsToDelete(snapshotTimestamps)
|
retentionManager.getSnapshotsToDelete(storedSnapshots)
|
||||||
} returns listOf(snapshot1.timeStart)
|
} returns listOf(storedSnapshot1)
|
||||||
coEvery { snapshotRetriever.getSnapshot(streamKey, snapshot1.timeStart) } returns snapshot1
|
coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1
|
||||||
coEvery { plugin.deleteBackupSnapshot(snapshot1.timeStart) } just Runs
|
coEvery { plugin.deleteBackupSnapshot(storedSnapshot1) } just Runs
|
||||||
every {
|
every {
|
||||||
db.applyInParts(capture(actualChunks), captureLambda())
|
db.applyInParts(capture(actualChunks), captureLambda())
|
||||||
} answers {
|
} answers {
|
||||||
|
|
|
@ -5,6 +5,8 @@ import android.content.SharedPreferences
|
||||||
import io.mockk.every
|
import io.mockk.every
|
||||||
import io.mockk.mockk
|
import io.mockk.mockk
|
||||||
import org.calyxos.backup.storage.api.SnapshotRetention
|
import org.calyxos.backup.storage.api.SnapshotRetention
|
||||||
|
import org.calyxos.backup.storage.api.StoredSnapshot
|
||||||
|
import org.calyxos.backup.storage.getRandomString
|
||||||
import org.junit.Assert.assertEquals
|
import org.junit.Assert.assertEquals
|
||||||
import org.junit.Test
|
import org.junit.Test
|
||||||
import java.time.LocalDateTime
|
import java.time.LocalDateTime
|
||||||
|
@ -17,19 +19,21 @@ internal class RetentionManagerTest {
|
||||||
|
|
||||||
private val retention = RetentionManager(context)
|
private val retention = RetentionManager(context)
|
||||||
|
|
||||||
|
private val userId = getRandomString()
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun testDailyRetention() {
|
fun testDailyRetention() {
|
||||||
expectGetRetention(SnapshotRetention(2, 0, 0, 0))
|
expectGetRetention(SnapshotRetention(2, 0, 0, 0))
|
||||||
val timestamps = listOf(
|
val storedSnapshots = listOf(
|
||||||
// 1577919600000
|
// 1577919600000
|
||||||
LocalDateTime.of(2020, 1, 1, 23, 0).toMillis(),
|
LocalDateTime.of(2020, 1, 1, 23, 0).toMillis(),
|
||||||
// 1577872800000
|
// 1577872800000
|
||||||
LocalDateTime.of(2020, 1, 1, 10, 0).toMillis(),
|
LocalDateTime.of(2020, 1, 1, 10, 0).toMillis(),
|
||||||
// 1583276400000
|
// 1583276400000
|
||||||
LocalDateTime.of(2020, 3, 3, 23, 0).toMillis(),
|
LocalDateTime.of(2020, 3, 3, 23, 0).toMillis(),
|
||||||
)
|
).map { StoredSnapshot(userId, it) }
|
||||||
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
val toDelete = retention.getSnapshotsToDelete(storedSnapshots)
|
||||||
assertEquals(listOf(1577872800000), toDelete)
|
assertEquals(listOf(1577872800000), toDelete.map { it.timestamp })
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -45,9 +49,9 @@ internal class RetentionManagerTest {
|
||||||
LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(),
|
||||||
// 1608678000000
|
// 1608678000000
|
||||||
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
||||||
)
|
).map { StoredSnapshot(userId, it) }
|
||||||
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
||||||
assertEquals(listOf(1608544800000, 1608638400000), toDelete)
|
assertEquals(listOf(1608544800000, 1608638400000), toDelete.map { it.timestamp })
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -63,9 +67,9 @@ internal class RetentionManagerTest {
|
||||||
LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(),
|
||||||
// 1608678000000
|
// 1608678000000
|
||||||
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
||||||
)
|
).map { StoredSnapshot(userId, it) }
|
||||||
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
||||||
assertEquals(listOf(1580857200000), toDelete)
|
assertEquals(listOf(1580857200000), toDelete.map { it.timestamp })
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -83,10 +87,10 @@ internal class RetentionManagerTest {
|
||||||
LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 21, 10, 0).toMillis(),
|
||||||
// 1608678000000
|
// 1608678000000
|
||||||
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
||||||
)
|
).map { StoredSnapshot(userId, it) }
|
||||||
// keeps only the latest one for each year, so three in total, even though keep is four
|
// keeps only the latest one for each year, so three in total, even though keep is four
|
||||||
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
||||||
assertEquals(listOf(1549321200000, 1608544800000), toDelete)
|
assertEquals(listOf(1549321200000, 1608544800000), toDelete.map { it.timestamp })
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -116,9 +120,11 @@ internal class RetentionManagerTest {
|
||||||
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 22, 23, 0).toMillis(),
|
||||||
// 1608638400000
|
// 1608638400000
|
||||||
LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(),
|
LocalDateTime.of(2020, 12, 22, 12, 0).toMillis(),
|
||||||
)
|
).map { StoredSnapshot(userId, it) }
|
||||||
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
val toDelete = retention.getSnapshotsToDelete(timestamps)
|
||||||
assertEquals(listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000), toDelete)
|
assertEquals(
|
||||||
|
listOf(1515106800000, 1549321200000, 1551441600000, 1608638400000),
|
||||||
|
toDelete.map { it.timestamp })
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun expectGetRetention(snapshotRetention: SnapshotRetention) {
|
private fun expectGetRetention(snapshotRetention: SnapshotRetention) {
|
||||||
|
|
Loading…
Reference in a new issue