Use new Backend directly in storage lib

This commit is contained in:
Torsten Grote 2024-08-27 17:28:03 -03:00
parent 0c1dfb316d
commit 58d58415c5
No known key found for this signature in database
GPG key ID: 3E5F77D92CF891FF
24 changed files with 446 additions and 406 deletions

View file

@ -11,5 +11,5 @@ import org.calyxos.backup.storage.api.StorageBackup
import org.koin.dsl.module import org.koin.dsl.module
val storageModule = module { val storageModule = module {
single { StorageBackup(get(), { get<StoragePluginManager>().filesPlugin }, get<KeyManager>()) } single { StorageBackup(get(), { get<StoragePluginManager>().backend }, get<KeyManager>()) }
} }

View file

@ -10,7 +10,7 @@ import android.os.StrictMode
import android.os.StrictMode.VmPolicy import android.os.StrictMode.VmPolicy
import android.util.Log import android.util.Log
import de.grobox.storagebackuptester.crypto.KeyManager import de.grobox.storagebackuptester.crypto.KeyManager
import de.grobox.storagebackuptester.plugin.TestSafStoragePlugin import de.grobox.storagebackuptester.plugin.TestSafBackend
import de.grobox.storagebackuptester.settings.SettingsManager import de.grobox.storagebackuptester.settings.SettingsManager
import org.calyxos.backup.storage.api.StorageBackup import org.calyxos.backup.storage.api.StorageBackup
import org.calyxos.backup.storage.ui.restore.FileSelectionManager import org.calyxos.backup.storage.ui.restore.FileSelectionManager
@ -19,7 +19,7 @@ class App : Application() {
val settingsManager: SettingsManager by lazy { SettingsManager(applicationContext) } val settingsManager: SettingsManager by lazy { SettingsManager(applicationContext) }
val storageBackup: StorageBackup by lazy { val storageBackup: StorageBackup by lazy {
val plugin = TestSafStoragePlugin(this) { settingsManager.getBackupLocation() } val plugin = TestSafBackend(this) { settingsManager.getBackupLocation() }
StorageBackup(this, { plugin }, KeyManager) StorageBackup(this, { plugin }, KeyManager)
} }
val fileSelectionManager: FileSelectionManager get() = FileSelectionManager() val fileSelectionManager: FileSelectionManager get() = FileSelectionManager()

View file

@ -0,0 +1,70 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package de.grobox.storagebackuptester.plugin
import android.content.Context
import android.net.Uri
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileHandle
import org.calyxos.seedvault.core.backends.FileInfo
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.backends.saf.SafBackend
import org.calyxos.seedvault.core.backends.saf.SafConfig
import java.io.InputStream
import java.io.OutputStream
import kotlin.reflect.KClass
class TestSafBackend(
private val appContext: Context,
private val getLocationUri: () -> Uri?,
) : Backend {
private val safConfig
get() = SafConfig(
config = getLocationUri() ?: error("no uri"),
name = "foo",
isUsb = false,
requiresNetwork = false,
rootId = "bar",
)
private val delegate: SafBackend get() = SafBackend(appContext, safConfig)
private val nullStream = object : OutputStream() {
override fun write(b: Int) {
// oops
}
}
override suspend fun test(): Boolean = delegate.test()
override suspend fun getFreeSpace(): Long? = delegate.getFreeSpace()
override suspend fun save(handle: FileHandle): OutputStream {
if (getLocationUri() == null) return nullStream
return delegate.save(handle)
}
override suspend fun load(handle: FileHandle): InputStream {
return delegate.load(handle)
}
override suspend fun list(
topLevelFolder: TopLevelFolder?,
vararg fileTypes: KClass<out FileHandle>,
callback: (FileInfo) -> Unit,
) = delegate.list(topLevelFolder, *fileTypes, callback = callback)
override suspend fun remove(handle: FileHandle) = delegate.remove(handle)
override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) {
delegate.rename(from, to)
}
override suspend fun removeAll() = delegate.removeAll()
override val providerPackageName: String? get() = delegate.providerPackageName
}

View file

@ -1,47 +0,0 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package de.grobox.storagebackuptester.plugin
import android.content.Context
import android.net.Uri
import org.calyxos.backup.storage.plugin.saf.SafStoragePlugin
import org.calyxos.seedvault.core.backends.saf.SafBackend
import org.calyxos.seedvault.core.backends.saf.SafConfig
import java.io.IOException
import java.io.OutputStream
class TestSafStoragePlugin(
private val appContext: Context,
private val getLocationUri: () -> Uri?,
) : SafStoragePlugin(appContext) {
private val safConfig
get() = SafConfig(
config = getLocationUri() ?: error("no uri"),
name = "foo",
isUsb = false,
requiresNetwork = false,
rootId = "bar",
)
override val delegate: SafBackend get() = SafBackend(appContext, safConfig)
private val nullStream = object : OutputStream() {
override fun write(b: Int) {
// oops
}
}
@Throws(IOException::class)
override suspend fun getChunkOutputStream(chunkId: String): OutputStream {
if (getLocationUri() == null) return nullStream
return super.getChunkOutputStream(chunkId)
}
override suspend fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
return super.getBackupSnapshotOutputStream(timestamp)
}
}

View file

@ -0,0 +1,67 @@
/*
* SPDX-FileCopyrightText: 2024 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage
import com.google.protobuf.InvalidProtocolBufferException
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.restore.readVersion
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import java.io.IOException
import java.security.GeneralSecurityException
internal class SnapshotRetriever(
private val storagePlugin: () -> Backend,
private val streamCrypto: StreamCrypto = StreamCrypto,
) {
@Throws(
IOException::class,
GeneralSecurityException::class,
InvalidProtocolBufferException::class,
)
suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot {
return storagePlugin().load(storedSnapshot.snapshotHandle).use { inputStream ->
val version = inputStream.readVersion()
val timestamp = storedSnapshot.timestamp
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
BackupSnapshot.parseFrom(decryptedStream)
}
}
}
}
@Throws(IOException::class)
internal suspend fun Backend.getCurrentBackupSnapshots(androidId: String): List<StoredSnapshot> {
val topLevelFolder = TopLevelFolder("$androidId.sv")
val snapshots = ArrayList<StoredSnapshot>()
list(topLevelFolder, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
internal suspend fun Backend.getBackupSnapshotsForRestore(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
list(null, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}

View file

@ -6,6 +6,8 @@
package org.calyxos.backup.storage.api package org.calyxos.backup.storage.api
import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
public data class SnapshotItem( public data class SnapshotItem(
public val storedSnapshot: StoredSnapshot, public val storedSnapshot: StoredSnapshot,
@ -21,7 +23,7 @@ public sealed class SnapshotResult {
public data class StoredSnapshot( public data class StoredSnapshot(
/** /**
* The unique ID of the current device/user combination chosen by the [StoragePlugin]. * The unique ID of the current device/user combination chosen by the [Backend].
* It may include an '.sv' extension. * It may include an '.sv' extension.
*/ */
public val userId: String, public val userId: String,
@ -31,6 +33,11 @@ public data class StoredSnapshot(
public val timestamp: Long, public val timestamp: Long,
) { ) {
public val androidId: String = userId.substringBefore(".sv") public val androidId: String = userId.substringBefore(".sv")
public val snapshotHandle: FileBackupFileType.Snapshot
get() = FileBackupFileType.Snapshot(
androidId = androidId,
time = timestamp,
)
} }
/** /**

View file

@ -5,10 +5,13 @@
package org.calyxos.backup.storage.api package org.calyxos.backup.storage.api
import android.annotation.SuppressLint
import android.content.Context import android.content.Context
import android.net.Uri import android.net.Uri
import android.provider.DocumentsContract.isTreeUri import android.provider.DocumentsContract.isTreeUri
import android.provider.MediaStore import android.provider.MediaStore
import android.provider.Settings
import android.provider.Settings.Secure.ANDROID_ID
import android.util.Log import android.util.Log
import androidx.annotation.WorkerThread import androidx.annotation.WorkerThread
import androidx.room.Room import androidx.room.Room
@ -16,13 +19,14 @@ import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.backup.ChunksCacheRepopulater import org.calyxos.backup.storage.backup.ChunksCacheRepopulater
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.backup.storage.getDocumentPath import org.calyxos.backup.storage.getDocumentPath
import org.calyxos.backup.storage.getMediaType import org.calyxos.backup.storage.getMediaType
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.prune.Pruner import org.calyxos.backup.storage.prune.Pruner
import org.calyxos.backup.storage.prune.RetentionManager import org.calyxos.backup.storage.prune.RetentionManager
import org.calyxos.backup.storage.restore.FileRestore import org.calyxos.backup.storage.restore.FileRestore
@ -31,6 +35,8 @@ import org.calyxos.backup.storage.scanner.DocumentScanner
import org.calyxos.backup.storage.scanner.FileScanner import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.MediaScanner import org.calyxos.backup.storage.scanner.MediaScanner
import org.calyxos.backup.storage.toStoredUri import org.calyxos.backup.storage.toStoredUri
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.crypto.KeyManager import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException import java.io.IOException
import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicBoolean
@ -39,7 +45,7 @@ private const val TAG = "StorageBackup"
public class StorageBackup( public class StorageBackup(
private val context: Context, private val context: Context,
private val pluginGetter: () -> StoragePlugin, private val pluginGetter: () -> Backend,
private val keyManager: KeyManager, private val keyManager: KeyManager,
private val dispatcher: CoroutineDispatcher = Dispatchers.IO, private val dispatcher: CoroutineDispatcher = Dispatchers.IO,
) { ) {
@ -50,13 +56,29 @@ public class StorageBackup(
} }
private val uriStore by lazy { db.getUriStore() } private val uriStore by lazy { db.getUriStore() }
@SuppressLint("HardwareIds")
private val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID)
private val mediaScanner by lazy { MediaScanner(context) } private val mediaScanner by lazy { MediaScanner(context) }
private val snapshotRetriever = SnapshotRetriever(pluginGetter) private val snapshotRetriever = SnapshotRetriever(pluginGetter)
private val chunksCacheRepopulater = ChunksCacheRepopulater(db, pluginGetter, snapshotRetriever) private val chunksCacheRepopulater = ChunksCacheRepopulater(
db = db,
storagePlugin = pluginGetter,
androidId = androidId,
snapshotRetriever = snapshotRetriever,
)
private val backup by lazy { private val backup by lazy {
val documentScanner = DocumentScanner(context) val documentScanner = DocumentScanner(context)
val fileScanner = FileScanner(uriStore, mediaScanner, documentScanner) val fileScanner = FileScanner(uriStore, mediaScanner, documentScanner)
Backup(context, db, fileScanner, pluginGetter, keyManager, chunksCacheRepopulater) Backup(
context = context,
db = db,
fileScanner = fileScanner,
backendGetter = pluginGetter,
androidId = androidId,
keyManager = keyManager,
cacheRepopulater = chunksCacheRepopulater
)
} }
private val restore by lazy { private val restore by lazy {
val fileRestore = FileRestore(context, mediaScanner) val fileRestore = FileRestore(context, mediaScanner)
@ -64,7 +86,7 @@ public class StorageBackup(
} }
private val retention = RetentionManager(context) private val retention = RetentionManager(context)
private val pruner by lazy { private val pruner by lazy {
Pruner(db, retention, pluginGetter, keyManager, snapshotRetriever) Pruner(db, retention, pluginGetter, androidId, keyManager, snapshotRetriever)
} }
private val backupRunning = AtomicBoolean(false) private val backupRunning = AtomicBoolean(false)
@ -113,7 +135,6 @@ public class StorageBackup(
* (see [deleteAllSnapshots]) as well as clears local cache (see [clearCache]). * (see [deleteAllSnapshots]) as well as clears local cache (see [clearCache]).
*/ */
public suspend fun init() { public suspend fun init() {
pluginGetter().init()
deleteAllSnapshots() deleteAllSnapshots()
clearCache() clearCache()
} }
@ -123,13 +144,14 @@ public class StorageBackup(
* (potentially encrypted with an old key) laying around. * (potentially encrypted with an old key) laying around.
* Using a storage location with existing data is not supported. * Using a storage location with existing data is not supported.
* Using the same root folder for storage on different devices or user profiles is fine though * Using the same root folder for storage on different devices or user profiles is fine though
* as the [StoragePlugin] should isolate storage per [StoredSnapshot.userId]. * as the [Backend] should isolate storage per [StoredSnapshot.userId].
*/ */
public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) { public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) {
try { try {
pluginGetter().getCurrentBackupSnapshots().forEach { pluginGetter().getCurrentBackupSnapshots(androidId).forEach {
val handle = FileBackupFileType.Snapshot(androidId, it.timestamp)
try { try {
pluginGetter().deleteBackupSnapshot(it) pluginGetter().remove(handle)
} catch (e: IOException) { } catch (e: IOException) {
Log.e(TAG, "Error deleting snapshot $it", e) Log.e(TAG, "Error deleting snapshot $it", e)
} }

View file

@ -12,13 +12,15 @@ import android.os.Build
import android.text.format.Formatter import android.text.format.Formatter
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.BackupObserver import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.crypto.ChunkCrypto import org.calyxos.backup.storage.crypto.ChunkCrypto
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.scanner.FileScanner import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.FileScannerResult import org.calyxos.backup.storage.scanner.FileScannerResult
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.crypto.KeyManager import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException import java.io.IOException
import java.security.GeneralSecurityException import java.security.GeneralSecurityException
@ -42,7 +44,8 @@ internal class Backup(
private val context: Context, private val context: Context,
private val db: Db, private val db: Db,
private val fileScanner: FileScanner, private val fileScanner: FileScanner,
private val storagePluginGetter: () -> StoragePlugin, private val backendGetter: () -> Backend,
private val androidId: String,
keyManager: KeyManager, keyManager: KeyManager,
private val cacheRepopulater: ChunksCacheRepopulater, private val cacheRepopulater: ChunksCacheRepopulater,
chunkSizeMax: Int = CHUNK_SIZE_MAX, chunkSizeMax: Int = CHUNK_SIZE_MAX,
@ -57,7 +60,7 @@ internal class Backup(
} }
private val contentResolver = context.contentResolver private val contentResolver = context.contentResolver
private val storagePlugin get() = storagePluginGetter() private val backend get() = backendGetter()
private val filesCache = db.getFilesCache() private val filesCache = db.getFilesCache()
private val chunksCache = db.getChunksCache() private val chunksCache = db.getChunksCache()
@ -71,7 +74,7 @@ internal class Backup(
} catch (e: GeneralSecurityException) { } catch (e: GeneralSecurityException) {
throw AssertionError(e) throw AssertionError(e)
} }
private val chunkWriter = ChunkWriter(streamCrypto, streamKey, chunksCache, storagePlugin) private val chunkWriter = ChunkWriter(streamCrypto, streamKey, chunksCache, backend, androidId)
private val hasMediaAccessPerm = private val hasMediaAccessPerm =
context.checkSelfPermission(ACCESS_MEDIA_LOCATION) == PERMISSION_GRANTED context.checkSelfPermission(ACCESS_MEDIA_LOCATION) == PERMISSION_GRANTED
private val fileBackup = FileBackup( private val fileBackup = FileBackup(
@ -95,7 +98,12 @@ internal class Backup(
try { try {
// get available chunks, so we do not need to rely solely on local cache // get available chunks, so we do not need to rely solely on local cache
// for checking if a chunk already exists on storage // for checking if a chunk already exists on storage
val availableChunkIds = storagePlugin.getAvailableChunkIds().toHashSet() val chunkIds = ArrayList<String>()
val topLevelFolder = TopLevelFolder.fromAndroidId(androidId)
backend.list(topLevelFolder, FileBackupFileType.Blob::class) { fileInfo ->
chunkIds.add(fileInfo.fileHandle.name)
}
val availableChunkIds = chunkIds.toHashSet()
if (!chunksCache.areAllAvailableChunksCached(db, availableChunkIds)) { if (!chunksCache.areAllAvailableChunksCached(db, availableChunkIds)) {
cacheRepopulater.repopulate(streamKey, availableChunkIds) cacheRepopulater.repopulate(streamKey, availableChunkIds)
} }
@ -154,7 +162,8 @@ internal class Backup(
.setTimeStart(startTime) .setTimeStart(startTime)
.setTimeEnd(endTime) .setTimeEnd(endTime)
.build() .build()
storagePlugin.getBackupSnapshotOutputStream(startTime).use { outputStream -> val fileHandle = FileBackupFileType.Snapshot(androidId, startTime)
backend.save(fileHandle).use { outputStream ->
outputStream.write(VERSION.toInt()) outputStream.write(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForSnapshot(startTime) val ad = streamCrypto.getAssociatedDataForSnapshot(startTime)
streamCrypto.newEncryptingStream(streamKey, outputStream, ad) streamCrypto.newEncryptingStream(streamKey, outputStream, ad)

View file

@ -6,10 +6,11 @@
package org.calyxos.backup.storage.backup package org.calyxos.backup.storage.backup
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.backup.Backup.Companion.VERSION import org.calyxos.backup.storage.backup.Backup.Companion.VERSION
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.io.IOException import java.io.IOException
import java.io.InputStream import java.io.InputStream
@ -30,7 +31,8 @@ internal class ChunkWriter(
private val streamCrypto: StreamCrypto, private val streamCrypto: StreamCrypto,
private val streamKey: ByteArray, private val streamKey: ByteArray,
private val chunksCache: ChunksCache, private val chunksCache: ChunksCache,
private val storagePlugin: StoragePlugin, private val backend: Backend,
private val androidId: String,
private val bufferSize: Int = DEFAULT_BUFFER_SIZE, private val bufferSize: Int = DEFAULT_BUFFER_SIZE,
) { ) {
@ -68,7 +70,8 @@ internal class ChunkWriter(
@Throws(IOException::class, GeneralSecurityException::class) @Throws(IOException::class, GeneralSecurityException::class)
private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit) { private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit) {
storagePlugin.getChunkOutputStream(chunkId).use { chunkStream -> val handle = FileBackupFileType.Blob(androidId, chunkId)
backend.save(handle).use { chunkStream ->
chunkStream.write(VERSION.toInt()) chunkStream.write(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForChunk(chunkId) val ad = streamCrypto.getAssociatedDataForChunk(chunkId)
streamCrypto.newEncryptingStream(streamKey, chunkStream, ad).use { encryptingStream -> streamCrypto.newEncryptingStream(streamKey, chunkStream, ad).use { encryptingStream ->

View file

@ -6,22 +6,24 @@
package org.calyxos.backup.storage.backup package org.calyxos.backup.storage.backup
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.db.CachedChunk import org.calyxos.backup.storage.db.CachedChunk
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import java.io.IOException import java.io.IOException
import java.security.GeneralSecurityException import java.security.GeneralSecurityException
import kotlin.time.DurationUnit.MILLISECONDS import kotlin.time.DurationUnit.MILLISECONDS
import kotlin.time.ExperimentalTime
import kotlin.time.toDuration import kotlin.time.toDuration
private const val TAG = "ChunksCacheRepopulater" private const val TAG = "ChunksCacheRepopulater"
internal class ChunksCacheRepopulater( internal class ChunksCacheRepopulater(
private val db: Db, private val db: Db,
private val storagePlugin: () -> StoragePlugin, private val storagePlugin: () -> Backend,
private val androidId: String,
private val snapshotRetriever: SnapshotRetriever, private val snapshotRetriever: SnapshotRetriever,
) { ) {
@ -36,20 +38,20 @@ internal class ChunksCacheRepopulater(
} }
@Throws(IOException::class) @Throws(IOException::class)
@OptIn(ExperimentalTime::class)
private suspend fun repopulateInternal( private suspend fun repopulateInternal(
streamKey: ByteArray, streamKey: ByteArray,
availableChunkIds: HashSet<String>, availableChunkIds: HashSet<String>,
) { ) {
val start = System.currentTimeMillis() val start = System.currentTimeMillis()
val snapshots = storagePlugin().getCurrentBackupSnapshots().mapNotNull { storedSnapshot -> val snapshots =
try { storagePlugin().getCurrentBackupSnapshots(androidId).mapNotNull { storedSnapshot ->
snapshotRetriever.getSnapshot(streamKey, storedSnapshot) try {
} catch (e: GeneralSecurityException) { snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
Log.w(TAG, "Error fetching snapshot $storedSnapshot", e) } catch (e: GeneralSecurityException) {
null Log.w(TAG, "Error fetching snapshot $storedSnapshot", e)
null
}
} }
}
val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS) val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS)
Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration") Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration")
@ -60,9 +62,12 @@ internal class ChunksCacheRepopulater(
Log.i(TAG, "Repopulating chunks cache took $repopulateDuration") Log.i(TAG, "Repopulating chunks cache took $repopulateDuration")
// delete chunks that are not references by any snapshot anymore // delete chunks that are not references by any snapshot anymore
val chunksToDelete = availableChunkIds.subtract(cachedChunks.map { it.id }) val chunksToDelete = availableChunkIds.subtract(cachedChunks.map { it.id }.toSet())
val deletionDuration = measure { val deletionDuration = measure {
storagePlugin().deleteChunks(chunksToDelete.toList()) chunksToDelete.forEach { chunkId ->
val handle = FileBackupFileType.Blob(androidId, chunkId)
storagePlugin().remove(handle)
}
} }
Log.i(TAG, "Deleting ${chunksToDelete.size} chunks took $deletionDuration") Log.i(TAG, "Deleting ${chunksToDelete.size} chunks took $deletionDuration")
} }

View file

@ -1,18 +0,0 @@
/*
* SPDX-FileCopyrightText: 2024 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage.plugin
public object PluginConstants {
public const val SNAPSHOT_EXT: String = ".SeedSnap"
public val folderRegex: Regex = Regex("^[a-f0-9]{16}\\.sv$")
public val chunkFolderRegex: Regex = Regex("[a-f0-9]{2}")
public val chunkRegex: Regex = Regex("[a-f0-9]{64}")
public val snapshotRegex: Regex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
public const val MIME_TYPE: String = "application/octet-stream"
public const val CHUNK_FOLDER_COUNT: Int = 256
}

View file

@ -1,38 +0,0 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage.plugin
import com.google.protobuf.InvalidProtocolBufferException
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.restore.readVersion
import java.io.IOException
import java.security.GeneralSecurityException
internal class SnapshotRetriever(
private val storagePlugin: () -> StoragePlugin,
private val streamCrypto: StreamCrypto = StreamCrypto,
) {
@Throws(
IOException::class,
GeneralSecurityException::class,
InvalidProtocolBufferException::class,
)
suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot {
return storagePlugin().getBackupSnapshotInputStream(storedSnapshot).use { inputStream ->
val version = inputStream.readVersion()
val timestamp = storedSnapshot.timestamp
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
BackupSnapshot.parseFrom(decryptedStream)
}
}
}
}

View file

@ -1,130 +0,0 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage.plugin.saf
import android.annotation.SuppressLint
import android.content.Context
import android.provider.Settings
import android.provider.Settings.Secure.ANDROID_ID
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.backends.saf.SafBackend
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
/**
* @param appContext application context provided by the storage module
*/
public abstract class SafStoragePlugin(
private val appContext: Context,
) : StoragePlugin {
protected abstract val delegate: SafBackend
private val androidId: String by lazy {
@SuppressLint("HardwareIds")
// This is unique to each combination of app-signing key, user, and device
// so we don't leak anything by not hashing this and can use it as is.
// Note: Use [appContext] here to not get the wrong ID for a different user.
val androidId = Settings.Secure.getString(appContext.contentResolver, ANDROID_ID)
androidId
}
private val topLevelFolder: TopLevelFolder by lazy {
// the folder name is our user ID
val folderName = "$androidId.sv"
TopLevelFolder(folderName)
}
override suspend fun init() {
// no-op as we are getting [root] created from super class
}
@Throws(IOException::class)
override suspend fun getAvailableChunkIds(): List<String> {
val chunkIds = ArrayList<String>()
delegate.list(topLevelFolder, FileBackupFileType.Blob::class) { fileInfo ->
chunkIds.add(fileInfo.fileHandle.name)
}
return chunkIds
}
@Throws(IOException::class)
override suspend fun getChunkOutputStream(chunkId: String): OutputStream {
val fileHandle = FileBackupFileType.Blob(androidId, chunkId)
return delegate.save(fileHandle)
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
val fileHandle = FileBackupFileType.Snapshot(androidId, timestamp)
return delegate.save(fileHandle)
}
/************************* Restore *******************************/
@Throws(IOException::class)
override suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
delegate.list(null, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream {
val androidId = storedSnapshot.androidId
val handle = FileBackupFileType.Snapshot(androidId, storedSnapshot.timestamp)
return delegate.load(handle)
}
@Throws(IOException::class)
override suspend fun getChunkInputStream(
snapshot: StoredSnapshot,
chunkId: String,
): InputStream {
val handle = FileBackupFileType.Blob(snapshot.androidId, chunkId)
return delegate.load(handle)
}
/************************* Pruning *******************************/
@Throws(IOException::class)
override suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
delegate.list(topLevelFolder, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) {
val androidId = storedSnapshot.androidId
val handle = FileBackupFileType.Snapshot(androidId, storedSnapshot.timestamp)
delegate.remove(handle)
}
@Throws(IOException::class)
override suspend fun deleteChunks(chunkIds: List<String>) {
chunkIds.forEach { chunkId ->
val androidId = topLevelFolder.name.substringBefore(".sv")
val handle = FileBackupFileType.Blob(androidId, chunkId)
delegate.remove(handle)
}
}
}

View file

@ -7,29 +7,31 @@ package org.calyxos.backup.storage.prune
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.BackupObserver import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.crypto.KeyManager import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException import java.io.IOException
import java.security.GeneralSecurityException import java.security.GeneralSecurityException
import kotlin.time.ExperimentalTime
private val TAG = Pruner::class.java.simpleName private val TAG = Pruner::class.java.simpleName
internal class Pruner( internal class Pruner(
private val db: Db, private val db: Db,
private val retentionManager: RetentionManager, private val retentionManager: RetentionManager,
private val storagePluginGetter: () -> StoragePlugin, private val storagePluginGetter: () -> Backend,
private val androidId: String,
keyManager: KeyManager, keyManager: KeyManager,
private val snapshotRetriever: SnapshotRetriever, private val snapshotRetriever: SnapshotRetriever,
streamCrypto: StreamCrypto = StreamCrypto, streamCrypto: StreamCrypto = StreamCrypto,
) { ) {
private val storagePlugin get() = storagePluginGetter() private val backend get() = storagePluginGetter()
private val chunksCache = db.getChunksCache() private val chunksCache = db.getChunksCache()
private val streamKey = try { private val streamKey = try {
streamCrypto.deriveStreamKey(keyManager.getMainKey()) streamCrypto.deriveStreamKey(keyManager.getMainKey())
@ -37,11 +39,10 @@ internal class Pruner(
throw AssertionError(e) throw AssertionError(e)
} }
@OptIn(ExperimentalTime::class)
@Throws(IOException::class) @Throws(IOException::class)
suspend fun prune(backupObserver: BackupObserver?) { suspend fun prune(backupObserver: BackupObserver?) {
val duration = measure { val duration = measure {
val storedSnapshots = storagePlugin.getCurrentBackupSnapshots() val storedSnapshots = backend.getCurrentBackupSnapshots(androidId)
val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots) val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots)
backupObserver?.onPruneStart(toDelete.map { it.timestamp }) backupObserver?.onPruneStart(toDelete.map { it.timestamp })
for (snapshot in toDelete) { for (snapshot in toDelete) {
@ -66,7 +67,7 @@ internal class Pruner(
val chunks = HashSet<String>() val chunks = HashSet<String>()
snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) } snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) }
snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) } snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) }
storagePlugin.deleteBackupSnapshot(storedSnapshot) backend.remove(storedSnapshot.snapshotHandle)
db.applyInParts(chunks) { db.applyInParts(chunks) {
chunksCache.decrementRefCount(it) chunksCache.decrementRefCount(it)
} }
@ -80,7 +81,9 @@ internal class Pruner(
it.id it.id
} }
backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size) backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size)
storagePlugin.deleteChunks(chunkIdsToDelete) chunkIdsToDelete.forEach { chunkId ->
backend.remove(FileBackupFileType.Blob(androidId, chunkId))
}
chunksCache.deleteChunks(cachedChunksToDelete) chunksCache.deleteChunks(cachedChunksToDelete)
} }

View file

@ -6,22 +6,23 @@
package org.calyxos.backup.storage.restore package org.calyxos.backup.storage.restore
import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import java.io.IOException import java.io.IOException
import java.io.InputStream import java.io.InputStream
import java.io.OutputStream import java.io.OutputStream
import java.security.GeneralSecurityException import java.security.GeneralSecurityException
internal abstract class AbstractChunkRestore( internal abstract class AbstractChunkRestore(
private val storagePluginGetter: () -> StoragePlugin, private val backendGetter: () -> Backend,
private val fileRestore: FileRestore, private val fileRestore: FileRestore,
private val streamCrypto: StreamCrypto, private val streamCrypto: StreamCrypto,
private val streamKey: ByteArray, private val streamKey: ByteArray,
) { ) {
private val storagePlugin get() = storagePluginGetter() private val backend get() = backendGetter()
@Throws(IOException::class, GeneralSecurityException::class) @Throws(IOException::class, GeneralSecurityException::class)
protected suspend fun getAndDecryptChunk( protected suspend fun getAndDecryptChunk(
@ -30,7 +31,7 @@ internal abstract class AbstractChunkRestore(
chunkId: String, chunkId: String,
streamReader: suspend (InputStream) -> Unit, streamReader: suspend (InputStream) -> Unit,
) { ) {
storagePlugin.getChunkInputStream(storedSnapshot, chunkId).use { inputStream -> backend.load(Blob(storedSnapshot.androidId, chunkId)).use { inputStream ->
inputStream.readVersion(version) inputStream.readVersion(version)
val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte()) val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream -> streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->

View file

@ -8,9 +8,9 @@ package org.calyxos.backup.storage.restore
import android.content.Context import android.content.Context
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
import java.io.File import java.io.File
import java.io.FileInputStream import java.io.FileInputStream
import java.io.FileOutputStream import java.io.FileOutputStream
@ -24,11 +24,11 @@ private const val TAG = "MultiChunkRestore"
@Suppress("BlockingMethodInNonBlockingContext") @Suppress("BlockingMethodInNonBlockingContext")
internal class MultiChunkRestore( internal class MultiChunkRestore(
private val context: Context, private val context: Context,
storagePlugin: () -> StoragePlugin, backendGetter: () -> Backend,
fileRestore: FileRestore, fileRestore: FileRestore,
streamCrypto: StreamCrypto, streamCrypto: StreamCrypto,
streamKey: ByteArray, streamKey: ByteArray,
) : AbstractChunkRestore(storagePlugin, fileRestore, streamCrypto, streamKey) { ) : AbstractChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey) {
suspend fun restore( suspend fun restore(
version: Int, version: Int,

View file

@ -12,13 +12,14 @@ import kotlinx.coroutines.flow.flow
import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.SnapshotItem import org.calyxos.backup.storage.api.SnapshotItem
import org.calyxos.backup.storage.api.SnapshotResult import org.calyxos.backup.storage.api.SnapshotResult
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.BackupSnapshot import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getBackupSnapshotsForRestore
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.crypto.KeyManager import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException import java.io.IOException
import java.io.InputStream import java.io.InputStream
@ -28,16 +29,16 @@ private const val TAG = "Restore"
internal class Restore( internal class Restore(
context: Context, context: Context,
private val storagePluginGetter: () -> StoragePlugin, private val backendGetter: () -> Backend,
private val keyManager: KeyManager, private val keyManager: KeyManager,
private val snapshotRetriever: SnapshotRetriever, private val snapshotRetriever: SnapshotRetriever,
fileRestore: FileRestore, fileRestore: FileRestore,
streamCrypto: StreamCrypto = StreamCrypto, streamCrypto: StreamCrypto = StreamCrypto,
) { ) {
private val storagePlugin get() = storagePluginGetter() private val backend get() = backendGetter()
private val streamKey by lazy { private val streamKey by lazy {
// This class might get instantiated before the StoragePlugin had time to provide the key // This class might get instantiated before the Backend had time to provide the key
// so we need to get it lazily here to prevent crashes. We can still crash later, // so we need to get it lazily here to prevent crashes. We can still crash later,
// if the plugin is not providing a key as it should when performing calls into this class. // if the plugin is not providing a key as it should when performing calls into this class.
try { try {
@ -49,13 +50,13 @@ internal class Restore(
// lazily instantiate these, so they don't try to get the streamKey too early // lazily instantiate these, so they don't try to get the streamKey too early
private val zipChunkRestore by lazy { private val zipChunkRestore by lazy {
ZipChunkRestore(storagePluginGetter, fileRestore, streamCrypto, streamKey) ZipChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey)
} }
private val singleChunkRestore by lazy { private val singleChunkRestore by lazy {
SingleChunkRestore(storagePluginGetter, fileRestore, streamCrypto, streamKey) SingleChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey)
} }
private val multiChunkRestore by lazy { private val multiChunkRestore by lazy {
MultiChunkRestore(context, storagePluginGetter, fileRestore, streamCrypto, streamKey) MultiChunkRestore(context, backendGetter, fileRestore, streamCrypto, streamKey)
} }
fun getBackupSnapshots(): Flow<SnapshotResult> = flow { fun getBackupSnapshots(): Flow<SnapshotResult> = flow {
@ -63,7 +64,7 @@ internal class Restore(
val time = measure { val time = measure {
val list = try { val list = try {
// get all available backups, they may not be usable // get all available backups, they may not be usable
storagePlugin.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot -> backend.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot ->
storedSnapshot.timestamp storedSnapshot.timestamp
}.map { storedSnapshot -> }.map { storedSnapshot ->
// as long as snapshot is null, it can't be used for restore // as long as snapshot is null, it can't be used for restore

View file

@ -7,18 +7,18 @@ package org.calyxos.backup.storage.restore
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
private const val TAG = "SingleChunkRestore" private const val TAG = "SingleChunkRestore"
internal class SingleChunkRestore( internal class SingleChunkRestore(
storagePlugin: () -> StoragePlugin, backendGetter: () -> Backend,
fileRestore: FileRestore, fileRestore: FileRestore,
streamCrypto: StreamCrypto, streamCrypto: StreamCrypto,
streamKey: ByteArray, streamKey: ByteArray,
) : AbstractChunkRestore(storagePlugin, fileRestore, streamCrypto, streamKey) { ) : AbstractChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey) {
suspend fun restore( suspend fun restore(
version: Int, version: Int,

View file

@ -7,9 +7,9 @@ package org.calyxos.backup.storage.restore
import android.util.Log import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
import java.io.IOException import java.io.IOException
import java.io.InputStream import java.io.InputStream
import java.io.OutputStream import java.io.OutputStream
@ -18,11 +18,11 @@ import java.util.zip.ZipInputStream
private const val TAG = "ZipChunkRestore" private const val TAG = "ZipChunkRestore"
internal class ZipChunkRestore( internal class ZipChunkRestore(
storagePlugin: () -> StoragePlugin, backendGetter: () -> Backend,
fileRestore: FileRestore, fileRestore: FileRestore,
streamCrypto: StreamCrypto, streamCrypto: StreamCrypto,
streamKey: ByteArray, streamKey: ByteArray,
) : AbstractChunkRestore(storagePlugin, fileRestore, streamCrypto, streamKey) { ) : AbstractChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey) {
/** /**
* Assumes that files in [zipChunks] are sorted by zipIndex with no duplicate indices. * Assumes that files in [zipChunks] are sorted by zipIndex with no duplicate indices.

View file

@ -22,7 +22,6 @@ import io.mockk.slot
import kotlinx.coroutines.flow.toList import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.SnapshotResult import org.calyxos.backup.storage.api.SnapshotResult
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.Backup import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX
@ -40,12 +39,14 @@ import org.calyxos.backup.storage.db.CachedFile
import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.db.FilesCache import org.calyxos.backup.storage.db.FilesCache
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.restore.FileRestore import org.calyxos.backup.storage.restore.FileRestore
import org.calyxos.backup.storage.restore.RestorableFile import org.calyxos.backup.storage.restore.RestorableFile
import org.calyxos.backup.storage.restore.Restore import org.calyxos.backup.storage.restore.Restore
import org.calyxos.backup.storage.scanner.FileScanner import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.FileScannerResult import org.calyxos.backup.storage.scanner.FileScannerResult
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.calyxos.seedvault.core.backends.FileBackupFileType.Snapshot
import org.calyxos.seedvault.core.crypto.KeyManager import org.calyxos.seedvault.core.crypto.KeyManager
import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
@ -71,22 +72,23 @@ internal class BackupRestoreTest {
private val contentResolver: ContentResolver = mockk() private val contentResolver: ContentResolver = mockk()
private val fileScanner: FileScanner = mockk() private val fileScanner: FileScanner = mockk()
private val pluginGetter: () -> StoragePlugin = mockk() private val backendGetter: () -> Backend = mockk()
private val androidId: String = getRandomString()
private val keyManager: KeyManager = mockk() private val keyManager: KeyManager = mockk()
private val plugin: StoragePlugin = mockk() private val backend: Backend = mockk()
private val fileRestore: FileRestore = mockk() private val fileRestore: FileRestore = mockk()
private val snapshotRetriever = SnapshotRetriever(pluginGetter) private val snapshotRetriever = SnapshotRetriever(backendGetter)
private val cacheRepopulater: ChunksCacheRepopulater = mockk() private val cacheRepopulater: ChunksCacheRepopulater = mockk()
init { init {
mockLog() mockLog()
mockkStatic("org.calyxos.backup.storage.SnapshotRetrieverKt")
mockkStatic(Formatter::class) mockkStatic(Formatter::class)
every { Formatter.formatShortFileSize(any(), any()) } returns "" every { Formatter.formatShortFileSize(any(), any()) } returns ""
mockkStatic("org.calyxos.backup.storage.UriUtilsKt") mockkStatic("org.calyxos.backup.storage.UriUtilsKt")
every { pluginGetter() } returns plugin every { backendGetter() } returns backend
every { db.getFilesCache() } returns filesCache every { db.getFilesCache() } returns filesCache
every { db.getChunksCache() } returns chunksCache every { db.getChunksCache() } returns chunksCache
every { keyManager.getMainKey() } returns SecretKeySpec( every { keyManager.getMainKey() } returns SecretKeySpec(
@ -97,11 +99,13 @@ internal class BackupRestoreTest {
every { context.contentResolver } returns contentResolver every { context.contentResolver } returns contentResolver
} }
private val restore = Restore(context, pluginGetter, keyManager, snapshotRetriever, fileRestore) private val restore =
Restore(context, backendGetter, keyManager, snapshotRetriever, fileRestore)
@Test @Test
fun testZipAndSingleRandom(): Unit = runBlocking { fun testZipAndSingleRandom(): Unit = runBlocking {
val backup = Backup(context, db, fileScanner, pluginGetter, keyManager, cacheRepopulater) val backup =
Backup(context, db, fileScanner, backendGetter, androidId, keyManager, cacheRepopulater)
val smallFileMBytes = Random.nextBytes(Random.nextInt(SMALL_FILE_SIZE_MAX)) val smallFileMBytes = Random.nextBytes(Random.nextInt(SMALL_FILE_SIZE_MAX))
val smallFileM = getRandomMediaFile(smallFileMBytes.size) val smallFileM = getRandomMediaFile(smallFileMBytes.size)
@ -119,12 +123,12 @@ internal class BackupRestoreTest {
val zipChunkOutputStream = ByteArrayOutputStream() val zipChunkOutputStream = ByteArrayOutputStream()
val mOutputStream = ByteArrayOutputStream() val mOutputStream = ByteArrayOutputStream()
val dOutputStream = ByteArrayOutputStream() val dOutputStream = ByteArrayOutputStream()
val snapshotTimestamp = slot<Long>() val snapshotHandle = slot<Snapshot>()
val snapshotOutputStream = ByteArrayOutputStream() val snapshotOutputStream = ByteArrayOutputStream()
// provide files and empty cache // provide files and empty cache
val availableChunks = emptyList<String>() val availableChunks = emptyList<String>()
coEvery { plugin.getAvailableChunkIds() } returns availableChunks coEvery { backend.list(any(), Blob::class, callback = any()) } just Runs
every { every {
chunksCache.areAllAvailableChunksCached(db, availableChunks.toHashSet()) chunksCache.areAllAvailableChunksCached(db, availableChunks.toHashSet())
} returns true } returns true
@ -152,16 +156,14 @@ internal class BackupRestoreTest {
} returns ByteArrayInputStream(fileDBytes) andThen ByteArrayInputStream(fileDBytes) } returns ByteArrayInputStream(fileDBytes) andThen ByteArrayInputStream(fileDBytes)
// output streams and caching // output streams and caching
coEvery { plugin.getChunkOutputStream(any()) } returnsMany listOf( coEvery { backend.save(any<Blob>()) } returnsMany listOf(
zipChunkOutputStream, mOutputStream, dOutputStream zipChunkOutputStream, mOutputStream, dOutputStream
) )
every { chunksCache.insert(any<CachedChunk>()) } just Runs every { chunksCache.insert(any<CachedChunk>()) } just Runs
every { filesCache.upsert(capture(cachedFiles)) } just Runs every { filesCache.upsert(capture(cachedFiles)) } just Runs
// snapshot writing // snapshot writing
coEvery { coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream
plugin.getBackupSnapshotOutputStream(capture(snapshotTimestamp))
} returns snapshotOutputStream
every { db.applyInParts<String>(any(), any()) } just Runs every { db.applyInParts<String>(any(), any()) } just Runs
backup.runBackup(null) backup.runBackup(null)
@ -181,16 +183,16 @@ internal class BackupRestoreTest {
// RESTORE // RESTORE
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured) val storedSnapshot = StoredSnapshot("$androidId.sv", snapshotHandle.captured.time)
val smallFileMOutputStream = ByteArrayOutputStream() val smallFileMOutputStream = ByteArrayOutputStream()
val smallFileDOutputStream = ByteArrayOutputStream() val smallFileDOutputStream = ByteArrayOutputStream()
val fileMOutputStream = ByteArrayOutputStream() val fileMOutputStream = ByteArrayOutputStream()
val fileDOutputStream = ByteArrayOutputStream() val fileDOutputStream = ByteArrayOutputStream()
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot) coEvery { backend.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery { coEvery {
plugin.getBackupSnapshotInputStream(storedSnapshot) backend.load(storedSnapshot.snapshotHandle)
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray()) } returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
// retrieve snapshots // retrieve snapshots
@ -200,21 +202,21 @@ internal class BackupRestoreTest {
assertEquals(2, snapshotResultList.size) assertEquals(2, snapshotResultList.size)
val snapshots = (snapshotResultList[1] as SnapshotResult.Success).snapshots val snapshots = (snapshotResultList[1] as SnapshotResult.Success).snapshots
assertEquals(1, snapshots.size) assertEquals(1, snapshots.size)
assertEquals(snapshotTimestamp.captured, snapshots[0].time) assertEquals(snapshotHandle.captured.time, snapshots[0].time)
val snapshot = snapshots[0].snapshot ?: error("snapshot was null") val snapshot = snapshots[0].snapshot ?: error("snapshot was null")
assertEquals(2, snapshot.mediaFilesList.size) assertEquals(2, snapshot.mediaFilesList.size)
assertEquals(2, snapshot.documentFilesList.size) assertEquals(2, snapshot.documentFilesList.size)
// pipe chunks back in // pipe chunks back in
coEvery { coEvery {
plugin.getChunkInputStream(storedSnapshot, cachedFiles[0].chunks[0]) backend.load(Blob(androidId, cachedFiles[0].chunks[0]))
} returns ByteArrayInputStream(zipChunkOutputStream.toByteArray()) } returns ByteArrayInputStream(zipChunkOutputStream.toByteArray())
// cachedFiles[0].chunks[1] is in previous zipChunk // cachedFiles[0].chunks[1] is in previous zipChunk
coEvery { coEvery {
plugin.getChunkInputStream(storedSnapshot, cachedFiles[2].chunks[0]) backend.load(Blob(androidId, cachedFiles[2].chunks[0]))
} returns ByteArrayInputStream(mOutputStream.toByteArray()) } returns ByteArrayInputStream(mOutputStream.toByteArray())
coEvery { coEvery {
plugin.getChunkInputStream(storedSnapshot, cachedFiles[3].chunks[0]) backend.load(Blob(androidId, cachedFiles[3].chunks[0]))
} returns ByteArrayInputStream(dOutputStream.toByteArray()) } returns ByteArrayInputStream(dOutputStream.toByteArray())
// provide file output streams for restore // provide file output streams for restore
@ -238,8 +240,16 @@ internal class BackupRestoreTest {
@Test @Test
fun testMultiChunks(): Unit = runBlocking { fun testMultiChunks(): Unit = runBlocking {
val backup = val backup = Backup(
Backup(context, db, fileScanner, pluginGetter, keyManager, cacheRepopulater, 4) context = context,
db = db,
fileScanner = fileScanner,
backendGetter = backendGetter,
androidId = androidId,
keyManager = keyManager,
cacheRepopulater = cacheRepopulater,
chunkSizeMax = 4,
)
val chunk1 = byteArrayOf(0x00, 0x01, 0x02, 0x03) val chunk1 = byteArrayOf(0x00, 0x01, 0x02, 0x03)
val chunk2 = byteArrayOf(0x04, 0x05, 0x06, 0x07) val chunk2 = byteArrayOf(0x04, 0x05, 0x06, 0x07)
@ -251,7 +261,7 @@ internal class BackupRestoreTest {
val file2 = getRandomDocFile(file2Bytes.size) val file2 = getRandomDocFile(file2Bytes.size)
val file1OutputStream = ByteArrayOutputStream() val file1OutputStream = ByteArrayOutputStream()
val file2OutputStream = ByteArrayOutputStream() val file2OutputStream = ByteArrayOutputStream()
val snapshotTimestamp = slot<Long>() val snapshotHandle = slot<Snapshot>()
val snapshotOutputStream = ByteArrayOutputStream() val snapshotOutputStream = ByteArrayOutputStream()
val scannedFiles = FileScannerResult( val scannedFiles = FileScannerResult(
@ -262,7 +272,7 @@ internal class BackupRestoreTest {
// provide files and empty cache // provide files and empty cache
val availableChunks = emptyList<String>() val availableChunks = emptyList<String>()
coEvery { plugin.getAvailableChunkIds() } returns availableChunks coEvery { backend.list(any(), Blob::class, callback = any()) } just Runs
every { every {
chunksCache.areAllAvailableChunksCached(db, availableChunks.toHashSet()) chunksCache.areAllAvailableChunksCached(db, availableChunks.toHashSet())
} returns true } returns true
@ -300,26 +310,38 @@ internal class BackupRestoreTest {
// output streams for deterministic chunks // output streams for deterministic chunks
val id040f32 = ByteArrayOutputStream() val id040f32 = ByteArrayOutputStream()
coEvery { coEvery {
plugin.getChunkOutputStream( backend.save(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
) )
} returns id040f32 } returns id040f32
val id901fbc = ByteArrayOutputStream() val id901fbc = ByteArrayOutputStream()
coEvery { coEvery {
plugin.getChunkOutputStream( backend.save(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
) )
} returns id901fbc } returns id901fbc
val id5adea3 = ByteArrayOutputStream() val id5adea3 = ByteArrayOutputStream()
coEvery { coEvery {
plugin.getChunkOutputStream( backend.save(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
) )
} returns id5adea3 } returns id5adea3
val id40d00c = ByteArrayOutputStream() val id40d00c = ByteArrayOutputStream()
coEvery { coEvery {
plugin.getChunkOutputStream( backend.save(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
) )
} returns id40d00c } returns id40d00c
@ -327,32 +349,46 @@ internal class BackupRestoreTest {
every { filesCache.upsert(capture(cachedFiles)) } just Runs every { filesCache.upsert(capture(cachedFiles)) } just Runs
// snapshot writing // snapshot writing
coEvery { coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream
plugin.getBackupSnapshotOutputStream(capture(snapshotTimestamp))
} returns snapshotOutputStream
every { db.applyInParts<String>(any(), any()) } just Runs every { db.applyInParts<String>(any(), any()) } just Runs
backup.runBackup(null) backup.runBackup(null)
// chunks were only written to storage once // chunks were only written to storage once
coVerify(exactly = 1) { coVerify(exactly = 1) {
plugin.getChunkOutputStream( backend.save(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3") Blob(
plugin.getChunkOutputStream( androidId = androidId,
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29") name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
plugin.getChunkOutputStream( )
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d") )
plugin.getChunkOutputStream( backend.save(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67") Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
)
backend.save(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
)
backend.save(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
)
} }
// RESTORE // RESTORE
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured) val storedSnapshot = StoredSnapshot("$androidId.sv", snapshotHandle.captured.time)
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot) coEvery { backend.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery { coEvery {
plugin.getBackupSnapshotInputStream(storedSnapshot) backend.load(storedSnapshot.snapshotHandle)
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray()) } returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
// retrieve snapshots // retrieve snapshots
@ -366,27 +402,35 @@ internal class BackupRestoreTest {
// pipe chunks back in // pipe chunks back in
coEvery { coEvery {
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
) )
} returns ByteArrayInputStream(id040f32.toByteArray()) } returns ByteArrayInputStream(id040f32.toByteArray())
coEvery { coEvery {
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
) )
} returns ByteArrayInputStream(id901fbc.toByteArray()) } returns ByteArrayInputStream(id901fbc.toByteArray())
coEvery { coEvery {
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
) )
} returns ByteArrayInputStream(id5adea3.toByteArray()) } returns ByteArrayInputStream(id5adea3.toByteArray())
coEvery { coEvery {
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
) )
} returns ByteArrayInputStream(id40d00c.toByteArray()) } returns ByteArrayInputStream(id40d00c.toByteArray())
@ -404,21 +448,29 @@ internal class BackupRestoreTest {
// chunks were only read from storage once // chunks were only read from storage once
coVerify(exactly = 1) { coVerify(exactly = 1) {
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3" androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
) )
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29" androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
) )
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d" androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
) )
plugin.getChunkInputStream( backend.load(
storedSnapshot, Blob(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67" androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
) )
} }
} }

View file

@ -12,13 +12,15 @@ import io.mockk.every
import io.mockk.just import io.mockk.just
import io.mockk.mockk import io.mockk.mockk
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.backup.Backup.Companion.VERSION import org.calyxos.backup.storage.backup.Backup.Companion.VERSION
import org.calyxos.backup.storage.crypto.Hkdf.KEY_SIZE_BYTES import org.calyxos.backup.storage.crypto.Hkdf.KEY_SIZE_BYTES
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.toHexString import org.calyxos.backup.storage.toHexString
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Test import org.junit.Test
@ -30,13 +32,20 @@ internal class ChunkWriterTest {
private val streamCrypto: StreamCrypto = mockk() private val streamCrypto: StreamCrypto = mockk()
private val chunksCache: ChunksCache = mockk() private val chunksCache: ChunksCache = mockk()
private val storagePlugin: StoragePlugin = mockk() private val backend: Backend = mockk()
private val androidId: String = getRandomString()
private val streamKey: ByteArray = Random.nextBytes(KEY_SIZE_BYTES) private val streamKey: ByteArray = Random.nextBytes(KEY_SIZE_BYTES)
private val ad1: ByteArray = Random.nextBytes(34) private val ad1: ByteArray = Random.nextBytes(34)
private val ad2: ByteArray = Random.nextBytes(34) private val ad2: ByteArray = Random.nextBytes(34)
private val ad3: ByteArray = Random.nextBytes(34) private val ad3: ByteArray = Random.nextBytes(34)
private val chunkWriter = private val chunkWriter = ChunkWriter(
ChunkWriter(streamCrypto, streamKey, chunksCache, storagePlugin, Random.nextInt(1, 42)) streamCrypto = streamCrypto,
streamKey = streamKey,
chunksCache = chunksCache,
backend = backend,
androidId = androidId,
bufferSize = Random.nextInt(1, 42),
)
private val chunkId1 = Random.nextBytes(KEY_SIZE_BYTES).toHexString() private val chunkId1 = Random.nextBytes(KEY_SIZE_BYTES).toHexString()
private val chunkId2 = Random.nextBytes(KEY_SIZE_BYTES).toHexString() private val chunkId2 = Random.nextBytes(KEY_SIZE_BYTES).toHexString()
@ -66,9 +75,9 @@ internal class ChunkWriterTest {
every { chunksCache.get(chunkId3) } returns null every { chunksCache.get(chunkId3) } returns null
// get the output streams for the chunks // get the output streams for the chunks
coEvery { storagePlugin.getChunkOutputStream(chunkId1) } returns chunk1Output coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output
coEvery { storagePlugin.getChunkOutputStream(chunkId2) } returns chunk2Output coEvery { backend.save(Blob(androidId, chunkId2)) } returns chunk2Output
coEvery { storagePlugin.getChunkOutputStream(chunkId3) } returns chunk3Output coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output
// get AD // get AD
every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1 every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1
@ -122,7 +131,7 @@ internal class ChunkWriterTest {
every { chunksCache.get(chunkId3) } returns null every { chunksCache.get(chunkId3) } returns null
// get and wrap the output stream for chunk that is missing // get and wrap the output stream for chunk that is missing
coEvery { storagePlugin.getChunkOutputStream(chunkId1) } returns chunk1Output coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output
every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1 every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1
every { every {
streamCrypto.newEncryptingStream(streamKey, chunk1Output, bytes(34)) streamCrypto.newEncryptingStream(streamKey, chunk1Output, bytes(34))
@ -132,7 +141,7 @@ internal class ChunkWriterTest {
every { chunksCache.insert(chunks[0].toCachedChunk()) } just Runs every { chunksCache.insert(chunks[0].toCachedChunk()) } just Runs
// get and wrap the output stream for chunk that isn't cached // get and wrap the output stream for chunk that isn't cached
coEvery { storagePlugin.getChunkOutputStream(chunkId3) } returns chunk3Output coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output
every { streamCrypto.getAssociatedDataForChunk(chunkId3) } returns ad3 every { streamCrypto.getAssociatedDataForChunk(chunkId3) } returns ad3
every { every {
streamCrypto.newEncryptingStream(streamKey, chunk3Output, bytes(34)) streamCrypto.newEncryptingStream(streamKey, chunk3Output, bytes(34))
@ -175,8 +184,8 @@ internal class ChunkWriterTest {
every { chunksCache.get(chunkId3) } returns null every { chunksCache.get(chunkId3) } returns null
// get the output streams for the chunks // get the output streams for the chunks
coEvery { storagePlugin.getChunkOutputStream(chunkId1) } returns chunk1Output coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output
coEvery { storagePlugin.getChunkOutputStream(chunkId3) } returns chunk3Output coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output
// get AD // get AD
every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1 every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1

View file

@ -11,16 +11,19 @@ import io.mockk.coVerify
import io.mockk.every import io.mockk.every
import io.mockk.just import io.mockk.just
import io.mockk.mockk import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.slot import io.mockk.slot
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.db.CachedChunk import org.calyxos.backup.storage.db.CachedChunk
import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.getRandomString import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.plugin.SnapshotRetriever import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue import org.junit.Assert.assertTrue
import org.junit.Test import org.junit.Test
@ -30,15 +33,22 @@ internal class ChunksCacheRepopulaterTest {
private val db: Db = mockk() private val db: Db = mockk()
private val chunksCache: ChunksCache = mockk() private val chunksCache: ChunksCache = mockk()
private val pluginGetter: () -> StoragePlugin = mockk() private val backendGetter: () -> Backend = mockk()
private val plugin: StoragePlugin = mockk() private val androidId: String = getRandomString()
private val backend: Backend = mockk()
private val snapshotRetriever: SnapshotRetriever = mockk() private val snapshotRetriever: SnapshotRetriever = mockk()
private val streamKey = "This is a backup key for testing".toByteArray() private val streamKey = "This is a backup key for testing".toByteArray()
private val cacheRepopulater = ChunksCacheRepopulater(db, pluginGetter, snapshotRetriever) private val cacheRepopulater = ChunksCacheRepopulater(
db = db,
storagePlugin = backendGetter,
androidId = androidId,
snapshotRetriever = snapshotRetriever,
)
init { init {
mockLog() mockLog()
every { pluginGetter() } returns plugin mockkStatic("org.calyxos.backup.storage.SnapshotRetrieverKt")
every { backendGetter() } returns backend
every { db.getChunksCache() } returns chunksCache every { db.getChunksCache() } returns chunksCache
} }
@ -73,7 +83,7 @@ internal class ChunksCacheRepopulaterTest {
) // chunk3 is not referenced and should get deleted ) // chunk3 is not referenced and should get deleted
val cachedChunksSlot = slot<Collection<CachedChunk>>() val cachedChunksSlot = slot<Collection<CachedChunk>>()
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots coEvery { backend.getCurrentBackupSnapshots(androidId) } returns storedSnapshots
coEvery { coEvery {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) snapshotRetriever.getSnapshot(streamKey, storedSnapshot1)
} returns snapshot1 } returns snapshot1
@ -81,14 +91,14 @@ internal class ChunksCacheRepopulaterTest {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot2) snapshotRetriever.getSnapshot(streamKey, storedSnapshot2)
} returns snapshot2 } returns snapshot2
every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs coEvery { backend.remove(Blob(androidId, chunk3)) } just Runs
cacheRepopulater.repopulate(streamKey, availableChunkIds) cacheRepopulater.repopulate(streamKey, availableChunkIds)
assertTrue(cachedChunksSlot.isCaptured) assertTrue(cachedChunksSlot.isCaptured)
assertEquals(cachedChunks.toSet(), cachedChunksSlot.captured.toSet()) assertEquals(cachedChunks.toSet(), cachedChunksSlot.captured.toSet())
coVerify { plugin.deleteChunks(listOf(chunk3)) } coVerify { backend.remove(Blob(androidId, chunk3)) }
} }
} }

View file

@ -14,7 +14,6 @@ import io.mockk.just
import io.mockk.mockk import io.mockk.mockk
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.BackupObserver import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.crypto.Hkdf.KEY_SIZE_BYTES import org.calyxos.backup.storage.crypto.Hkdf.KEY_SIZE_BYTES
import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.CachedChunk import org.calyxos.backup.storage.db.CachedChunk
@ -24,6 +23,7 @@ import org.calyxos.backup.storage.getRandomDocFile
import org.calyxos.backup.storage.getRandomString import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.toHexString import org.calyxos.backup.storage.toHexString
import org.calyxos.seedvault.core.backends.Backend
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Test import org.junit.Test
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
@ -39,13 +39,15 @@ internal class SmallFileBackupIntegrationTest {
private val filesCache: FilesCache = mockk() private val filesCache: FilesCache = mockk()
private val mac: Mac = mockk() private val mac: Mac = mockk()
private val chunksCache: ChunksCache = mockk() private val chunksCache: ChunksCache = mockk()
private val storagePlugin: StoragePlugin = mockk() private val backend: Backend = mockk()
private val androidId: String = getRandomString()
private val chunkWriter = ChunkWriter( private val chunkWriter = ChunkWriter(
streamCrypto = StreamCrypto, streamCrypto = StreamCrypto,
streamKey = Random.nextBytes(KEY_SIZE_BYTES), streamKey = Random.nextBytes(KEY_SIZE_BYTES),
chunksCache = chunksCache, chunksCache = chunksCache,
storagePlugin = storagePlugin, backend = backend,
androidId = androidId,
) )
private val zipChunker = ZipChunker( private val zipChunker = ZipChunker(
mac = mac, mac = mac,
@ -91,7 +93,7 @@ internal class SmallFileBackupIntegrationTest {
every { mac.doFinal(any<ByteArray>()) } returns chunkId every { mac.doFinal(any<ByteArray>()) } returns chunkId
every { chunksCache.get(any()) } returns null every { chunksCache.get(any()) } returns null
coEvery { storagePlugin.getChunkOutputStream(any()) } returns outputStream2 coEvery { backend.save(any()) } returns outputStream2
every { every {
chunksCache.insert(match<CachedChunk> { cachedChunk -> chunksCache.insert(match<CachedChunk> { cachedChunk ->
cachedChunk.id == chunkId.toHexString() && cachedChunk.id == chunkId.toHexString() &&

View file

@ -10,9 +10,9 @@ import io.mockk.coEvery
import io.mockk.every import io.mockk.every
import io.mockk.just import io.mockk.just
import io.mockk.mockk import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.slot import io.mockk.slot
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupDocumentFile import org.calyxos.backup.storage.backup.BackupDocumentFile
import org.calyxos.backup.storage.backup.BackupMediaFile import org.calyxos.backup.storage.backup.BackupMediaFile
@ -25,7 +25,10 @@ import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.getRandomString import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.plugin.SnapshotRetriever import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.calyxos.seedvault.core.crypto.KeyManager import org.calyxos.seedvault.core.crypto.KeyManager
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue import org.junit.Assert.assertTrue
@ -37,9 +40,10 @@ internal class PrunerTest {
private val db: Db = mockk() private val db: Db = mockk()
private val chunksCache: ChunksCache = mockk() private val chunksCache: ChunksCache = mockk()
private val pluginGetter: () -> StoragePlugin = mockk() private val backendGetter: () -> Backend = mockk()
private val androidId: String = getRandomString()
private val keyManager: KeyManager = mockk() private val keyManager: KeyManager = mockk()
private val plugin: StoragePlugin = mockk() private val backend: Backend = mockk()
private val snapshotRetriever: SnapshotRetriever = mockk() private val snapshotRetriever: SnapshotRetriever = mockk()
private val retentionManager: RetentionManager = mockk() private val retentionManager: RetentionManager = mockk()
private val streamCrypto: StreamCrypto = mockk() private val streamCrypto: StreamCrypto = mockk()
@ -48,14 +52,22 @@ internal class PrunerTest {
init { init {
mockLog(false) mockLog(false)
every { pluginGetter() } returns plugin mockkStatic("org.calyxos.backup.storage.SnapshotRetrieverKt")
every { backendGetter() } returns backend
every { db.getChunksCache() } returns chunksCache every { db.getChunksCache() } returns chunksCache
every { keyManager.getMainKey() } returns masterKey every { keyManager.getMainKey() } returns masterKey
every { streamCrypto.deriveStreamKey(masterKey) } returns streamKey every { streamCrypto.deriveStreamKey(masterKey) } returns streamKey
} }
private val pruner = private val pruner = Pruner(
Pruner(db, retentionManager, pluginGetter, keyManager, snapshotRetriever, streamCrypto) db = db,
retentionManager = retentionManager,
storagePluginGetter = backendGetter,
androidId = androidId,
keyManager = keyManager,
snapshotRetriever = snapshotRetriever,
streamCrypto = streamCrypto,
)
@Test @Test
fun test() = runBlocking { fun test() = runBlocking {
@ -84,12 +96,12 @@ internal class PrunerTest {
val actualChunks2 = slot<Collection<String>>() val actualChunks2 = slot<Collection<String>>()
val cachedChunk3 = CachedChunk(chunk3, 0, 0) val cachedChunk3 = CachedChunk(chunk3, 0, 0)
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots coEvery { backend.getCurrentBackupSnapshots(androidId) } returns storedSnapshots
every { every {
retentionManager.getSnapshotsToDelete(storedSnapshots) retentionManager.getSnapshotsToDelete(storedSnapshots)
} returns listOf(storedSnapshot1) } returns listOf(storedSnapshot1)
coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1 coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1
coEvery { plugin.deleteBackupSnapshot(storedSnapshot1) } just Runs coEvery { backend.remove(storedSnapshot1.snapshotHandle) } just Runs
every { every {
db.applyInParts(capture(actualChunks), captureLambda()) db.applyInParts(capture(actualChunks), captureLambda())
} answers { } answers {
@ -97,7 +109,7 @@ internal class PrunerTest {
} }
every { chunksCache.decrementRefCount(capture(actualChunks2)) } just Runs every { chunksCache.decrementRefCount(capture(actualChunks2)) } just Runs
every { chunksCache.getUnreferencedChunks() } returns listOf(cachedChunk3) every { chunksCache.getUnreferencedChunks() } returns listOf(cachedChunk3)
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs coEvery { backend.remove(Blob(androidId, chunk3)) } just Runs
every { chunksCache.deleteChunks(listOf(cachedChunk3)) } just Runs every { chunksCache.deleteChunks(listOf(cachedChunk3)) } just Runs
pruner.prune(null) pruner.prune(null)