mobi/seedvault
1
0
Fork 0
Code Pull requests Activity

Use new Backend directly in storage lib

Browse source
This commit is contained in:
Torsten Grote 2024-08-27 17:28:03 -03:00
parent 0c1dfb316d
commit 58d58415c5
No known key found for this signature in database
GPG key ID: 3E5F77D92CF891FF
24 changed files with 446 additions and 406 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/src/main/java/com/stevesoltys/seedvault/storage/StorageModule.kt
View file

@ -11,5 +11,5 @@ import org.calyxos.backup.storage.api.StorageBackup
import org.koin.dsl.module
val storageModule = module {
single { StorageBackup(get(), { get<StoragePluginManager>().filesPlugin }, get<KeyManager>()) }
single { StorageBackup(get(), { get<StoragePluginManager>().backend }, get<KeyManager>()) }
}

4
storage/demo/src/main/java/de/grobox/storagebackuptester/App.kt
View file

@ -10,7 +10,7 @@ import android.os.StrictMode
import android.os.StrictMode.VmPolicy
import android.util.Log
import de.grobox.storagebackuptester.crypto.KeyManager
import de.grobox.storagebackuptester.plugin.TestSafStoragePlugin
import de.grobox.storagebackuptester.plugin.TestSafBackend
import de.grobox.storagebackuptester.settings.SettingsManager
import org.calyxos.backup.storage.api.StorageBackup
import org.calyxos.backup.storage.ui.restore.FileSelectionManager
@ -19,7 +19,7 @@ class App : Application() {
val settingsManager: SettingsManager by lazy { SettingsManager(applicationContext) }
val storageBackup: StorageBackup by lazy {
val plugin = TestSafStoragePlugin(this) { settingsManager.getBackupLocation() }
val plugin = TestSafBackend(this) { settingsManager.getBackupLocation() }
StorageBackup(this, { plugin }, KeyManager)
}
val fileSelectionManager: FileSelectionManager get() = FileSelectionManager()

70
storage/demo/src/main/java/de/grobox/storagebackuptester/plugin/TestSafBackend.kt Normal file
View file

@ -0,0 +1,70 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package de.grobox.storagebackuptester.plugin
import android.content.Context
import android.net.Uri
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileHandle
import org.calyxos.seedvault.core.backends.FileInfo
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.backends.saf.SafBackend
import org.calyxos.seedvault.core.backends.saf.SafConfig
import java.io.InputStream
import java.io.OutputStream
import kotlin.reflect.KClass
class TestSafBackend(
private val appContext: Context,
private val getLocationUri: () -> Uri?,
) : Backend {
private val safConfig
get() = SafConfig(
config = getLocationUri() ?: error("no uri"),
name = "foo",
isUsb = false,
requiresNetwork = false,
rootId = "bar",
)
private val delegate: SafBackend get() = SafBackend(appContext, safConfig)
private val nullStream = object : OutputStream() {
override fun write(b: Int) {
// oops
}
}
override suspend fun test(): Boolean = delegate.test()
override suspend fun getFreeSpace(): Long? = delegate.getFreeSpace()
override suspend fun save(handle: FileHandle): OutputStream {
if (getLocationUri() == null) return nullStream
return delegate.save(handle)
}
override suspend fun load(handle: FileHandle): InputStream {
return delegate.load(handle)
}
override suspend fun list(
topLevelFolder: TopLevelFolder?,
vararg fileTypes: KClass<out FileHandle>,
callback: (FileInfo) -> Unit,
) = delegate.list(topLevelFolder, *fileTypes, callback = callback)
override suspend fun remove(handle: FileHandle) = delegate.remove(handle)
override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) {
delegate.rename(from, to)
}
override suspend fun removeAll() = delegate.removeAll()
override val providerPackageName: String? get() = delegate.providerPackageName
}

47
storage/demo/src/main/java/de/grobox/storagebackuptester/plugin/TestSafStoragePlugin.kt
View file

@ -1,47 +0,0 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package de.grobox.storagebackuptester.plugin
import android.content.Context
import android.net.Uri
import org.calyxos.backup.storage.plugin.saf.SafStoragePlugin
import org.calyxos.seedvault.core.backends.saf.SafBackend
import org.calyxos.seedvault.core.backends.saf.SafConfig
import java.io.IOException
import java.io.OutputStream
class TestSafStoragePlugin(
private val appContext: Context,
private val getLocationUri: () -> Uri?,
) : SafStoragePlugin(appContext) {
private val safConfig
get() = SafConfig(
config = getLocationUri() ?: error("no uri"),
name = "foo",
isUsb = false,
requiresNetwork = false,
rootId = "bar",
)
override val delegate: SafBackend get() = SafBackend(appContext, safConfig)
private val nullStream = object : OutputStream() {
override fun write(b: Int) {
// oops
}
}
@Throws(IOException::class)
override suspend fun getChunkOutputStream(chunkId: String): OutputStream {
if (getLocationUri() == null) return nullStream
return super.getChunkOutputStream(chunkId)
}
override suspend fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
return super.getBackupSnapshotOutputStream(timestamp)
}
}

67
storage/lib/src/main/java/org/calyxos/backup/storage/SnapshotRetriever.kt Normal file
View file

@ -0,0 +1,67 @@
/*
* SPDX-FileCopyrightText: 2024 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage
import com.google.protobuf.InvalidProtocolBufferException
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.restore.readVersion
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import java.io.IOException
import java.security.GeneralSecurityException
internal class SnapshotRetriever(
private val storagePlugin: () -> Backend,
private val streamCrypto: StreamCrypto = StreamCrypto,
) {
@Throws(
IOException::class,
GeneralSecurityException::class,
InvalidProtocolBufferException::class,
)
suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot {
return storagePlugin().load(storedSnapshot.snapshotHandle).use { inputStream ->
val version = inputStream.readVersion()
val timestamp = storedSnapshot.timestamp
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
BackupSnapshot.parseFrom(decryptedStream)
}
}
}
}
@Throws(IOException::class)
internal suspend fun Backend.getCurrentBackupSnapshots(androidId: String): List<StoredSnapshot> {
val topLevelFolder = TopLevelFolder("$androidId.sv")
val snapshots = ArrayList<StoredSnapshot>()
list(topLevelFolder, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
internal suspend fun Backend.getBackupSnapshotsForRestore(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
list(null, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}

9
storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt
View file

@ -6,6 +6,8 @@
package org.calyxos.backup.storage.api
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
public data class SnapshotItem(
public val storedSnapshot: StoredSnapshot,
@ -21,7 +23,7 @@ public sealed class SnapshotResult {
public data class StoredSnapshot(
/**
* The unique ID of the current device/user combination chosen by the [StoragePlugin].
* The unique ID of the current device/user combination chosen by the [Backend].
* It may include an '.sv' extension.
*/
public val userId: String,
@ -31,6 +33,11 @@ public data class StoredSnapshot(
public val timestamp: Long,
) {
public val androidId: String = userId.substringBefore(".sv")
public val snapshotHandle: FileBackupFileType.Snapshot
get() = FileBackupFileType.Snapshot(
androidId = androidId,
time = timestamp,
)
}
/**

40
storage/lib/src/main/java/org/calyxos/backup/storage/api/StorageBackup.kt
View file

@ -5,10 +5,13 @@
package org.calyxos.backup.storage.api
import android.annotation.SuppressLint
import android.content.Context
import android.net.Uri
import android.provider.DocumentsContract.isTreeUri
import android.provider.MediaStore
import android.provider.Settings
import android.provider.Settings.Secure.ANDROID_ID
import android.util.Log
import androidx.annotation.WorkerThread
import androidx.room.Room
@ -16,13 +19,14 @@ import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.withContext
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.backup.ChunksCacheRepopulater
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.backup.storage.getDocumentPath
import org.calyxos.backup.storage.getMediaType
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.prune.Pruner
import org.calyxos.backup.storage.prune.RetentionManager
import org.calyxos.backup.storage.restore.FileRestore
@ -31,6 +35,8 @@ import org.calyxos.backup.storage.scanner.DocumentScanner
import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.MediaScanner
import org.calyxos.backup.storage.toStoredUri
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException
import java.util.concurrent.atomic.AtomicBoolean
@ -39,7 +45,7 @@ private const val TAG = "StorageBackup"
public class StorageBackup(
private val context: Context,
private val pluginGetter: () -> StoragePlugin,
private val pluginGetter: () -> Backend,
private val keyManager: KeyManager,
private val dispatcher: CoroutineDispatcher = Dispatchers.IO,
) {
@ -50,13 +56,29 @@ public class StorageBackup(
}
private val uriStore by lazy { db.getUriStore() }
@SuppressLint("HardwareIds")
private val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID)
private val mediaScanner by lazy { MediaScanner(context) }
private val snapshotRetriever = SnapshotRetriever(pluginGetter)
private val chunksCacheRepopulater = ChunksCacheRepopulater(db, pluginGetter, snapshotRetriever)
private val chunksCacheRepopulater = ChunksCacheRepopulater(
db = db,
storagePlugin = pluginGetter,
androidId = androidId,
snapshotRetriever = snapshotRetriever,
)
private val backup by lazy {
val documentScanner = DocumentScanner(context)
val fileScanner = FileScanner(uriStore, mediaScanner, documentScanner)
Backup(context, db, fileScanner, pluginGetter, keyManager, chunksCacheRepopulater)
Backup(
context = context,
db = db,
fileScanner = fileScanner,
backendGetter = pluginGetter,
androidId = androidId,
keyManager = keyManager,
cacheRepopulater = chunksCacheRepopulater
)
}
private val restore by lazy {
val fileRestore = FileRestore(context, mediaScanner)
@ -64,7 +86,7 @@ public class StorageBackup(
}
private val retention = RetentionManager(context)
private val pruner by lazy {
Pruner(db, retention, pluginGetter, keyManager, snapshotRetriever)
Pruner(db, retention, pluginGetter, androidId, keyManager, snapshotRetriever)
}
private val backupRunning = AtomicBoolean(false)
@ -113,7 +135,6 @@ public class StorageBackup(
* (see [deleteAllSnapshots]) as well as clears local cache (see [clearCache]).
*/
public suspend fun init() {
pluginGetter().init()
deleteAllSnapshots()
clearCache()
}
@ -123,13 +144,14 @@ public class StorageBackup(
* (potentially encrypted with an old key) laying around.
* Using a storage location with existing data is not supported.
* Using the same root folder for storage on different devices or user profiles is fine though
* as the [StoragePlugin] should isolate storage per [StoredSnapshot.userId].
* as the [Backend] should isolate storage per [StoredSnapshot.userId].
*/
public suspend fun deleteAllSnapshots(): Unit = withContext(dispatcher) {
try {
pluginGetter().getCurrentBackupSnapshots().forEach {
pluginGetter().getCurrentBackupSnapshots(androidId).forEach {
val handle = FileBackupFileType.Snapshot(androidId, it.timestamp)
try {
pluginGetter().deleteBackupSnapshot(it)
pluginGetter().remove(handle)
} catch (e: IOException) {
Log.e(TAG, "Error deleting snapshot $it", e)
}

21
storage/lib/src/main/java/org/calyxos/backup/storage/backup/Backup.kt
View file

@ -12,13 +12,15 @@ import android.os.Build
import android.text.format.Formatter
import android.util.Log
import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.crypto.ChunkCrypto
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.FileScannerResult
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException
import java.security.GeneralSecurityException
@ -42,7 +44,8 @@ internal class Backup(
private val context: Context,
private val db: Db,
private val fileScanner: FileScanner,
private val storagePluginGetter: () -> StoragePlugin,
private val backendGetter: () -> Backend,
private val androidId: String,
keyManager: KeyManager,
private val cacheRepopulater: ChunksCacheRepopulater,
chunkSizeMax: Int = CHUNK_SIZE_MAX,
@ -57,7 +60,7 @@ internal class Backup(
}
private val contentResolver = context.contentResolver
private val storagePlugin get() = storagePluginGetter()
private val backend get() = backendGetter()
private val filesCache = db.getFilesCache()
private val chunksCache = db.getChunksCache()
@ -71,7 +74,7 @@ internal class Backup(
} catch (e: GeneralSecurityException) {
throw AssertionError(e)
}
private val chunkWriter = ChunkWriter(streamCrypto, streamKey, chunksCache, storagePlugin)
private val chunkWriter = ChunkWriter(streamCrypto, streamKey, chunksCache, backend, androidId)
private val hasMediaAccessPerm =
context.checkSelfPermission(ACCESS_MEDIA_LOCATION) == PERMISSION_GRANTED
private val fileBackup = FileBackup(
@ -95,7 +98,12 @@ internal class Backup(
try {
// get available chunks, so we do not need to rely solely on local cache
// for checking if a chunk already exists on storage
val availableChunkIds = storagePlugin.getAvailableChunkIds().toHashSet()
val chunkIds = ArrayList<String>()
val topLevelFolder = TopLevelFolder.fromAndroidId(androidId)
backend.list(topLevelFolder, FileBackupFileType.Blob::class) { fileInfo ->
chunkIds.add(fileInfo.fileHandle.name)
}
val availableChunkIds = chunkIds.toHashSet()
if (!chunksCache.areAllAvailableChunksCached(db, availableChunkIds)) {
cacheRepopulater.repopulate(streamKey, availableChunkIds)
}
@ -154,7 +162,8 @@ internal class Backup(
.setTimeStart(startTime)
.setTimeEnd(endTime)
.build()
storagePlugin.getBackupSnapshotOutputStream(startTime).use { outputStream ->
val fileHandle = FileBackupFileType.Snapshot(androidId, startTime)
backend.save(fileHandle).use { outputStream ->
outputStream.write(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForSnapshot(startTime)
streamCrypto.newEncryptingStream(streamKey, outputStream, ad)

9
storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunkWriter.kt
View file

@ -6,10 +6,11 @@
package org.calyxos.backup.storage.backup
import android.util.Log
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.backup.Backup.Companion.VERSION
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.io.InputStream
@ -30,7 +31,8 @@ internal class ChunkWriter(
private val streamCrypto: StreamCrypto,
private val streamKey: ByteArray,
private val chunksCache: ChunksCache,
private val storagePlugin: StoragePlugin,
private val backend: Backend,
private val androidId: String,
private val bufferSize: Int = DEFAULT_BUFFER_SIZE,
) {
@ -68,7 +70,8 @@ internal class ChunkWriter(
@Throws(IOException::class, GeneralSecurityException::class)
private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit) {
storagePlugin.getChunkOutputStream(chunkId).use { chunkStream ->
val handle = FileBackupFileType.Blob(androidId, chunkId)
backend.save(handle).use { chunkStream ->
chunkStream.write(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForChunk(chunkId)
streamCrypto.newEncryptingStream(streamKey, chunkStream, ad).use { encryptingStream ->

33
storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulater.kt
View file

@ -6,22 +6,24 @@
package org.calyxos.backup.storage.backup
import android.util.Log
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.db.CachedChunk
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import java.io.IOException
import java.security.GeneralSecurityException
import kotlin.time.DurationUnit.MILLISECONDS
import kotlin.time.ExperimentalTime
import kotlin.time.toDuration
private const val TAG = "ChunksCacheRepopulater"
internal class ChunksCacheRepopulater(
private val db: Db,
private val storagePlugin: () -> StoragePlugin,
private val storagePlugin: () -> Backend,
private val androidId: String,
private val snapshotRetriever: SnapshotRetriever,
) {
@ -36,20 +38,20 @@ internal class ChunksCacheRepopulater(
}
@Throws(IOException::class)
@OptIn(ExperimentalTime::class)
private suspend fun repopulateInternal(
streamKey: ByteArray,
availableChunkIds: HashSet<String>,
) {
val start = System.currentTimeMillis()
val snapshots = storagePlugin().getCurrentBackupSnapshots().mapNotNull { storedSnapshot ->
try {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
} catch (e: GeneralSecurityException) {
Log.w(TAG, "Error fetching snapshot $storedSnapshot", e)
null
val snapshots =
storagePlugin().getCurrentBackupSnapshots(androidId).mapNotNull { storedSnapshot ->
try {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot)
} catch (e: GeneralSecurityException) {
Log.w(TAG, "Error fetching snapshot $storedSnapshot", e)
null
}
}
}
val snapshotDuration = (System.currentTimeMillis() - start).toDuration(MILLISECONDS)
Log.i(TAG, "Retrieving and parsing all snapshots took $snapshotDuration")
@ -60,9 +62,12 @@ internal class ChunksCacheRepopulater(
Log.i(TAG, "Repopulating chunks cache took $repopulateDuration")
// delete chunks that are not references by any snapshot anymore
val chunksToDelete = availableChunkIds.subtract(cachedChunks.map { it.id })
val chunksToDelete = availableChunkIds.subtract(cachedChunks.map { it.id }.toSet())
val deletionDuration = measure {
storagePlugin().deleteChunks(chunksToDelete.toList())
chunksToDelete.forEach { chunkId ->
val handle = FileBackupFileType.Blob(androidId, chunkId)
storagePlugin().remove(handle)
}
}
Log.i(TAG, "Deleting ${chunksToDelete.size} chunks took $deletionDuration")
}

18
storage/lib/src/main/java/org/calyxos/backup/storage/plugin/PluginConstants.kt
View file

@ -1,18 +0,0 @@
/*
* SPDX-FileCopyrightText: 2024 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage.plugin
public object PluginConstants {
public const val SNAPSHOT_EXT: String = ".SeedSnap"
public val folderRegex: Regex = Regex("^[a-f0-9]{16}\\.sv$")
public val chunkFolderRegex: Regex = Regex("[a-f0-9]{2}")
public val chunkRegex: Regex = Regex("[a-f0-9]{64}")
public val snapshotRegex: Regex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
public const val MIME_TYPE: String = "application/octet-stream"
public const val CHUNK_FOLDER_COUNT: Int = 256
}

38
storage/lib/src/main/java/org/calyxos/backup/storage/plugin/SnapshotRetriever.kt
View file

@ -1,38 +0,0 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage.plugin
import com.google.protobuf.InvalidProtocolBufferException
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.restore.readVersion
import java.io.IOException
import java.security.GeneralSecurityException
internal class SnapshotRetriever(
private val storagePlugin: () -> StoragePlugin,
private val streamCrypto: StreamCrypto = StreamCrypto,
) {
@Throws(
IOException::class,
GeneralSecurityException::class,
InvalidProtocolBufferException::class,
)
suspend fun getSnapshot(streamKey: ByteArray, storedSnapshot: StoredSnapshot): BackupSnapshot {
return storagePlugin().getBackupSnapshotInputStream(storedSnapshot).use { inputStream ->
val version = inputStream.readVersion()
val timestamp = storedSnapshot.timestamp
val ad = streamCrypto.getAssociatedDataForSnapshot(timestamp, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->
BackupSnapshot.parseFrom(decryptedStream)
}
}
}
}

130
storage/lib/src/main/java/org/calyxos/backup/storage/plugin/saf/SafStoragePlugin.kt
View file

@ -1,130 +0,0 @@
/*
* SPDX-FileCopyrightText: 2021 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package org.calyxos.backup.storage.plugin.saf
import android.annotation.SuppressLint
import android.content.Context
import android.provider.Settings
import android.provider.Settings.Secure.ANDROID_ID
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.backends.saf.SafBackend
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
/**
* @param appContext application context provided by the storage module
*/
public abstract class SafStoragePlugin(
private val appContext: Context,
) : StoragePlugin {
protected abstract val delegate: SafBackend
private val androidId: String by lazy {
@SuppressLint("HardwareIds")
// This is unique to each combination of app-signing key, user, and device
// so we don't leak anything by not hashing this and can use it as is.
// Note: Use [appContext] here to not get the wrong ID for a different user.
val androidId = Settings.Secure.getString(appContext.contentResolver, ANDROID_ID)
androidId
}
private val topLevelFolder: TopLevelFolder by lazy {
// the folder name is our user ID
val folderName = "$androidId.sv"
TopLevelFolder(folderName)
}
override suspend fun init() {
// no-op as we are getting [root] created from super class
}
@Throws(IOException::class)
override suspend fun getAvailableChunkIds(): List<String> {
val chunkIds = ArrayList<String>()
delegate.list(topLevelFolder, FileBackupFileType.Blob::class) { fileInfo ->
chunkIds.add(fileInfo.fileHandle.name)
}
return chunkIds
}
@Throws(IOException::class)
override suspend fun getChunkOutputStream(chunkId: String): OutputStream {
val fileHandle = FileBackupFileType.Blob(androidId, chunkId)
return delegate.save(fileHandle)
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
val fileHandle = FileBackupFileType.Snapshot(androidId, timestamp)
return delegate.save(fileHandle)
}
/************************* Restore *******************************/
@Throws(IOException::class)
override suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
delegate.list(null, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream {
val androidId = storedSnapshot.androidId
val handle = FileBackupFileType.Snapshot(androidId, storedSnapshot.timestamp)
return delegate.load(handle)
}
@Throws(IOException::class)
override suspend fun getChunkInputStream(
snapshot: StoredSnapshot,
chunkId: String,
): InputStream {
val handle = FileBackupFileType.Blob(snapshot.androidId, chunkId)
return delegate.load(handle)
}
/************************* Pruning *******************************/
@Throws(IOException::class)
override suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot> {
val snapshots = ArrayList<StoredSnapshot>()
delegate.list(topLevelFolder, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) {
val androidId = storedSnapshot.androidId
val handle = FileBackupFileType.Snapshot(androidId, storedSnapshot.timestamp)
delegate.remove(handle)
}
@Throws(IOException::class)
override suspend fun deleteChunks(chunkIds: List<String>) {
chunkIds.forEach { chunkId ->
val androidId = topLevelFolder.name.substringBefore(".sv")
val handle = FileBackupFileType.Blob(androidId, chunkId)
delegate.remove(handle)
}
}
}

21
storage/lib/src/main/java/org/calyxos/backup/storage/prune/Pruner.kt
View file

@ -7,29 +7,31 @@ package org.calyxos.backup.storage.prune
import android.util.Log
import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException
import java.security.GeneralSecurityException
import kotlin.time.ExperimentalTime
private val TAG = Pruner::class.java.simpleName
internal class Pruner(
private val db: Db,
private val retentionManager: RetentionManager,
private val storagePluginGetter: () -> StoragePlugin,
private val storagePluginGetter: () -> Backend,
private val androidId: String,
keyManager: KeyManager,
private val snapshotRetriever: SnapshotRetriever,
streamCrypto: StreamCrypto = StreamCrypto,
) {
private val storagePlugin get() = storagePluginGetter()
private val backend get() = storagePluginGetter()
private val chunksCache = db.getChunksCache()
private val streamKey = try {
streamCrypto.deriveStreamKey(keyManager.getMainKey())
@ -37,11 +39,10 @@ internal class Pruner(
throw AssertionError(e)
}
@OptIn(ExperimentalTime::class)
@Throws(IOException::class)
suspend fun prune(backupObserver: BackupObserver?) {
val duration = measure {
val storedSnapshots = storagePlugin.getCurrentBackupSnapshots()
val storedSnapshots = backend.getCurrentBackupSnapshots(androidId)
val toDelete = retentionManager.getSnapshotsToDelete(storedSnapshots)
backupObserver?.onPruneStart(toDelete.map { it.timestamp })
for (snapshot in toDelete) {
@ -66,7 +67,7 @@ internal class Pruner(
val chunks = HashSet<String>()
snapshot.mediaFilesList.forEach { chunks.addAll(it.chunkIdsList) }
snapshot.documentFilesList.forEach { chunks.addAll(it.chunkIdsList) }
storagePlugin.deleteBackupSnapshot(storedSnapshot)
backend.remove(storedSnapshot.snapshotHandle)
db.applyInParts(chunks) {
chunksCache.decrementRefCount(it)
}
@ -80,7 +81,9 @@ internal class Pruner(
it.id
}
backupObserver?.onPruneSnapshot(storedSnapshot.timestamp, chunkIdsToDelete.size, size)
storagePlugin.deleteChunks(chunkIdsToDelete)
chunkIdsToDelete.forEach { chunkId ->
backend.remove(FileBackupFileType.Blob(androidId, chunkId))
}
chunksCache.deleteChunks(cachedChunksToDelete)
}

9
storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt
View file

@ -6,22 +6,23 @@
package org.calyxos.backup.storage.restore
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.security.GeneralSecurityException
internal abstract class AbstractChunkRestore(
private val storagePluginGetter: () -> StoragePlugin,
private val backendGetter: () -> Backend,
private val fileRestore: FileRestore,
private val streamCrypto: StreamCrypto,
private val streamKey: ByteArray,
) {
private val storagePlugin get() = storagePluginGetter()
private val backend get() = backendGetter()
@Throws(IOException::class, GeneralSecurityException::class)
protected suspend fun getAndDecryptChunk(
@ -30,7 +31,7 @@ internal abstract class AbstractChunkRestore(
chunkId: String,
streamReader: suspend (InputStream) -> Unit,
) {
storagePlugin.getChunkInputStream(storedSnapshot, chunkId).use { inputStream ->
backend.load(Blob(storedSnapshot.androidId, chunkId)).use { inputStream ->
inputStream.readVersion(version)
val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version.toByte())
streamCrypto.newDecryptingStream(streamKey, inputStream, ad).use { decryptedStream ->

6
storage/lib/src/main/java/org/calyxos/backup/storage/restore/MultiChunkRestore.kt
View file

@ -8,9 +8,9 @@ package org.calyxos.backup.storage.restore
import android.content.Context
import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
@ -24,11 +24,11 @@ private const val TAG = "MultiChunkRestore"
@Suppress("BlockingMethodInNonBlockingContext")
internal class MultiChunkRestore(
private val context: Context,
storagePlugin: () -> StoragePlugin,
backendGetter: () -> Backend,
fileRestore: FileRestore,
streamCrypto: StreamCrypto,
streamKey: ByteArray,
) : AbstractChunkRestore(storagePlugin, fileRestore, streamCrypto, streamKey) {
) : AbstractChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey) {
suspend fun restore(
version: Int,

19
storage/lib/src/main/java/org/calyxos/backup/storage/restore/Restore.kt
View file

@ -12,13 +12,14 @@ import kotlinx.coroutines.flow.flow
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.SnapshotItem
import org.calyxos.backup.storage.api.SnapshotResult
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.BackupSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getBackupSnapshotsForRestore
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.IOException
import java.io.InputStream
@ -28,16 +29,16 @@ private const val TAG = "Restore"
internal class Restore(
context: Context,
private val storagePluginGetter: () -> StoragePlugin,
private val backendGetter: () -> Backend,
private val keyManager: KeyManager,
private val snapshotRetriever: SnapshotRetriever,
fileRestore: FileRestore,
streamCrypto: StreamCrypto = StreamCrypto,
) {
private val storagePlugin get() = storagePluginGetter()
private val backend get() = backendGetter()
private val streamKey by lazy {
// This class might get instantiated before the StoragePlugin had time to provide the key
// This class might get instantiated before the Backend had time to provide the key
// so we need to get it lazily here to prevent crashes. We can still crash later,
// if the plugin is not providing a key as it should when performing calls into this class.
try {
@ -49,13 +50,13 @@ internal class Restore(
// lazily instantiate these, so they don't try to get the streamKey too early
private val zipChunkRestore by lazy {
ZipChunkRestore(storagePluginGetter, fileRestore, streamCrypto, streamKey)
ZipChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey)
}
private val singleChunkRestore by lazy {
SingleChunkRestore(storagePluginGetter, fileRestore, streamCrypto, streamKey)
SingleChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey)
}
private val multiChunkRestore by lazy {
MultiChunkRestore(context, storagePluginGetter, fileRestore, streamCrypto, streamKey)
MultiChunkRestore(context, backendGetter, fileRestore, streamCrypto, streamKey)
}
fun getBackupSnapshots(): Flow<SnapshotResult> = flow {
@ -63,7 +64,7 @@ internal class Restore(
val time = measure {
val list = try {
// get all available backups, they may not be usable
storagePlugin.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot ->
backend.getBackupSnapshotsForRestore().sortedByDescending { storedSnapshot ->
storedSnapshot.timestamp
}.map { storedSnapshot ->
// as long as snapshot is null, it can't be used for restore

6
storage/lib/src/main/java/org/calyxos/backup/storage/restore/SingleChunkRestore.kt
View file

@ -7,18 +7,18 @@ package org.calyxos.backup.storage.restore
import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
private const val TAG = "SingleChunkRestore"
internal class SingleChunkRestore(
storagePlugin: () -> StoragePlugin,
backendGetter: () -> Backend,
fileRestore: FileRestore,
streamCrypto: StreamCrypto,
streamKey: ByteArray,
) : AbstractChunkRestore(storagePlugin, fileRestore, streamCrypto, streamKey) {
) : AbstractChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey) {
suspend fun restore(
version: Int,

6
storage/lib/src/main/java/org/calyxos/backup/storage/restore/ZipChunkRestore.kt
View file

@ -7,9 +7,9 @@ package org.calyxos.backup.storage.restore
import android.util.Log
import org.calyxos.backup.storage.api.RestoreObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.seedvault.core.backends.Backend
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
@ -18,11 +18,11 @@ import java.util.zip.ZipInputStream
private const val TAG = "ZipChunkRestore"
internal class ZipChunkRestore(
storagePlugin: () -> StoragePlugin,
backendGetter: () -> Backend,
fileRestore: FileRestore,
streamCrypto: StreamCrypto,
streamKey: ByteArray,
) : AbstractChunkRestore(storagePlugin, fileRestore, streamCrypto, streamKey) {
) : AbstractChunkRestore(backendGetter, fileRestore, streamCrypto, streamKey) {
/**
* Assumes that files in [zipChunks] are sorted by zipIndex with no duplicate indices.

196
storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt
View file

@ -22,7 +22,6 @@ import io.mockk.slot
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.SnapshotResult
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.Backup
import org.calyxos.backup.storage.backup.Backup.Companion.CHUNK_SIZE_MAX
@ -40,12 +39,14 @@ import org.calyxos.backup.storage.db.CachedFile
import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.db.FilesCache
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.restore.FileRestore
import org.calyxos.backup.storage.restore.RestorableFile
import org.calyxos.backup.storage.restore.Restore
import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.FileScannerResult
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.calyxos.seedvault.core.backends.FileBackupFileType.Snapshot
import org.calyxos.seedvault.core.crypto.KeyManager
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
@ -71,22 +72,23 @@ internal class BackupRestoreTest {
private val contentResolver: ContentResolver = mockk()
private val fileScanner: FileScanner = mockk()
private val pluginGetter: () -> StoragePlugin = mockk()
private val backendGetter: () -> Backend = mockk()
private val androidId: String = getRandomString()
private val keyManager: KeyManager = mockk()
private val plugin: StoragePlugin = mockk()
private val backend: Backend = mockk()
private val fileRestore: FileRestore = mockk()
private val snapshotRetriever = SnapshotRetriever(pluginGetter)
private val snapshotRetriever = SnapshotRetriever(backendGetter)
private val cacheRepopulater: ChunksCacheRepopulater = mockk()
init {
mockLog()
mockkStatic("org.calyxos.backup.storage.SnapshotRetrieverKt")
mockkStatic(Formatter::class)
every { Formatter.formatShortFileSize(any(), any()) } returns ""
mockkStatic("org.calyxos.backup.storage.UriUtilsKt")
every { pluginGetter() } returns plugin
every { backendGetter() } returns backend
every { db.getFilesCache() } returns filesCache
every { db.getChunksCache() } returns chunksCache
every { keyManager.getMainKey() } returns SecretKeySpec(
@ -97,11 +99,13 @@ internal class BackupRestoreTest {
every { context.contentResolver } returns contentResolver
}
private val restore = Restore(context, pluginGetter, keyManager, snapshotRetriever, fileRestore)
private val restore =
Restore(context, backendGetter, keyManager, snapshotRetriever, fileRestore)
@Test
fun testZipAndSingleRandom(): Unit = runBlocking {
val backup = Backup(context, db, fileScanner, pluginGetter, keyManager, cacheRepopulater)
val backup =
Backup(context, db, fileScanner, backendGetter, androidId, keyManager, cacheRepopulater)
val smallFileMBytes = Random.nextBytes(Random.nextInt(SMALL_FILE_SIZE_MAX))
val smallFileM = getRandomMediaFile(smallFileMBytes.size)
@ -119,12 +123,12 @@ internal class BackupRestoreTest {
val zipChunkOutputStream = ByteArrayOutputStream()
val mOutputStream = ByteArrayOutputStream()
val dOutputStream = ByteArrayOutputStream()
val snapshotTimestamp = slot<Long>()
val snapshotHandle = slot<Snapshot>()
val snapshotOutputStream = ByteArrayOutputStream()
// provide files and empty cache
val availableChunks = emptyList<String>()
coEvery { plugin.getAvailableChunkIds() } returns availableChunks
coEvery { backend.list(any(), Blob::class, callback = any()) } just Runs
every {
chunksCache.areAllAvailableChunksCached(db, availableChunks.toHashSet())
} returns true
@ -152,16 +156,14 @@ internal class BackupRestoreTest {
} returns ByteArrayInputStream(fileDBytes) andThen ByteArrayInputStream(fileDBytes)
// output streams and caching
coEvery { plugin.getChunkOutputStream(any()) } returnsMany listOf(
coEvery { backend.save(any<Blob>()) } returnsMany listOf(
zipChunkOutputStream, mOutputStream, dOutputStream
)
every { chunksCache.insert(any<CachedChunk>()) } just Runs
every { filesCache.upsert(capture(cachedFiles)) } just Runs
// snapshot writing
coEvery {
plugin.getBackupSnapshotOutputStream(capture(snapshotTimestamp))
} returns snapshotOutputStream
coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream
every { db.applyInParts<String>(any(), any()) } just Runs
backup.runBackup(null)
@ -181,16 +183,16 @@ internal class BackupRestoreTest {
// RESTORE
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured)
val storedSnapshot = StoredSnapshot("$androidId.sv", snapshotHandle.captured.time)
val smallFileMOutputStream = ByteArrayOutputStream()
val smallFileDOutputStream = ByteArrayOutputStream()
val fileMOutputStream = ByteArrayOutputStream()
val fileDOutputStream = ByteArrayOutputStream()
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery { backend.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery {
plugin.getBackupSnapshotInputStream(storedSnapshot)
backend.load(storedSnapshot.snapshotHandle)
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
// retrieve snapshots
@ -200,21 +202,21 @@ internal class BackupRestoreTest {
assertEquals(2, snapshotResultList.size)
val snapshots = (snapshotResultList[1] as SnapshotResult.Success).snapshots
assertEquals(1, snapshots.size)
assertEquals(snapshotTimestamp.captured, snapshots[0].time)
assertEquals(snapshotHandle.captured.time, snapshots[0].time)
val snapshot = snapshots[0].snapshot ?: error("snapshot was null")
assertEquals(2, snapshot.mediaFilesList.size)
assertEquals(2, snapshot.documentFilesList.size)
// pipe chunks back in
coEvery {
plugin.getChunkInputStream(storedSnapshot, cachedFiles[0].chunks[0])
backend.load(Blob(androidId, cachedFiles[0].chunks[0]))
} returns ByteArrayInputStream(zipChunkOutputStream.toByteArray())
// cachedFiles[0].chunks[1] is in previous zipChunk
coEvery {
plugin.getChunkInputStream(storedSnapshot, cachedFiles[2].chunks[0])
backend.load(Blob(androidId, cachedFiles[2].chunks[0]))
} returns ByteArrayInputStream(mOutputStream.toByteArray())
coEvery {
plugin.getChunkInputStream(storedSnapshot, cachedFiles[3].chunks[0])
backend.load(Blob(androidId, cachedFiles[3].chunks[0]))
} returns ByteArrayInputStream(dOutputStream.toByteArray())
// provide file output streams for restore
@ -238,8 +240,16 @@ internal class BackupRestoreTest {
@Test
fun testMultiChunks(): Unit = runBlocking {
val backup =
Backup(context, db, fileScanner, pluginGetter, keyManager, cacheRepopulater, 4)
val backup = Backup(
context = context,
db = db,
fileScanner = fileScanner,
backendGetter = backendGetter,
androidId = androidId,
keyManager = keyManager,
cacheRepopulater = cacheRepopulater,
chunkSizeMax = 4,
)
val chunk1 = byteArrayOf(0x00, 0x01, 0x02, 0x03)
val chunk2 = byteArrayOf(0x04, 0x05, 0x06, 0x07)
@ -251,7 +261,7 @@ internal class BackupRestoreTest {
val file2 = getRandomDocFile(file2Bytes.size)
val file1OutputStream = ByteArrayOutputStream()
val file2OutputStream = ByteArrayOutputStream()
val snapshotTimestamp = slot<Long>()
val snapshotHandle = slot<Snapshot>()
val snapshotOutputStream = ByteArrayOutputStream()
val scannedFiles = FileScannerResult(
@ -262,7 +272,7 @@ internal class BackupRestoreTest {
// provide files and empty cache
val availableChunks = emptyList<String>()
coEvery { plugin.getAvailableChunkIds() } returns availableChunks
coEvery { backend.list(any(), Blob::class, callback = any()) } just Runs
every {
chunksCache.areAllAvailableChunksCached(db, availableChunks.toHashSet())
} returns true
@ -300,26 +310,38 @@ internal class BackupRestoreTest {
// output streams for deterministic chunks
val id040f32 = ByteArrayOutputStream()
coEvery {
plugin.getChunkOutputStream(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
backend.save(
Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
)
} returns id040f32
val id901fbc = ByteArrayOutputStream()
coEvery {
plugin.getChunkOutputStream(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
backend.save(
Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
)
} returns id901fbc
val id5adea3 = ByteArrayOutputStream()
coEvery {
plugin.getChunkOutputStream(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
backend.save(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
)
} returns id5adea3
val id40d00c = ByteArrayOutputStream()
coEvery {
plugin.getChunkOutputStream(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
backend.save(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
)
} returns id40d00c
@ -327,32 +349,46 @@ internal class BackupRestoreTest {
every { filesCache.upsert(capture(cachedFiles)) } just Runs
// snapshot writing
coEvery {
plugin.getBackupSnapshotOutputStream(capture(snapshotTimestamp))
} returns snapshotOutputStream
coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream
every { db.applyInParts<String>(any(), any()) } just Runs
backup.runBackup(null)
// chunks were only written to storage once
coVerify(exactly = 1) {
plugin.getChunkOutputStream(
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3")
plugin.getChunkOutputStream(
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29")
plugin.getChunkOutputStream(
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d")
plugin.getChunkOutputStream(
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67")
backend.save(
Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
)
backend.save(
Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
)
backend.save(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
)
backend.save(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
)
}
// RESTORE
val storedSnapshot = StoredSnapshot("test", snapshotTimestamp.captured)
val storedSnapshot = StoredSnapshot("$androidId.sv", snapshotHandle.captured.time)
coEvery { plugin.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery { backend.getBackupSnapshotsForRestore() } returns listOf(storedSnapshot)
coEvery {
plugin.getBackupSnapshotInputStream(storedSnapshot)
backend.load(storedSnapshot.snapshotHandle)
} returns ByteArrayInputStream(snapshotOutputStream.toByteArray())
// retrieve snapshots
@ -366,27 +402,35 @@ internal class BackupRestoreTest {
// pipe chunks back in
coEvery {
plugin.getChunkInputStream(
storedSnapshot,
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
backend.load(
Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
)
} returns ByteArrayInputStream(id040f32.toByteArray())
coEvery {
plugin.getChunkInputStream(
storedSnapshot,
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
backend.load(
Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
)
} returns ByteArrayInputStream(id901fbc.toByteArray())
coEvery {
plugin.getChunkInputStream(
storedSnapshot,
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
backend.load(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
)
} returns ByteArrayInputStream(id5adea3.toByteArray())
coEvery {
plugin.getChunkInputStream(
storedSnapshot,
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
backend.load(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
)
} returns ByteArrayInputStream(id40d00c.toByteArray())
@ -404,21 +448,29 @@ internal class BackupRestoreTest {
// chunks were only read from storage once
coVerify(exactly = 1) {
plugin.getChunkInputStream(
storedSnapshot,
"040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3"
backend.load(
Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
)
plugin.getChunkInputStream(
storedSnapshot,
"901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29"
backend.load(
Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
)
plugin.getChunkInputStream(
storedSnapshot,
"5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d"
backend.load(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
)
plugin.getChunkInputStream(
storedSnapshot,
"40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67"
backend.load(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
)
}
}

31
storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunkWriterTest.kt
View file

@ -12,13 +12,15 @@ import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.backup.Backup.Companion.VERSION
import org.calyxos.backup.storage.crypto.Hkdf.KEY_SIZE_BYTES
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.toHexString
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Test
@ -30,13 +32,20 @@ internal class ChunkWriterTest {
private val streamCrypto: StreamCrypto = mockk()
private val chunksCache: ChunksCache = mockk()
private val storagePlugin: StoragePlugin = mockk()
private val backend: Backend = mockk()
private val androidId: String = getRandomString()
private val streamKey: ByteArray = Random.nextBytes(KEY_SIZE_BYTES)
private val ad1: ByteArray = Random.nextBytes(34)
private val ad2: ByteArray = Random.nextBytes(34)
private val ad3: ByteArray = Random.nextBytes(34)
private val chunkWriter =
ChunkWriter(streamCrypto, streamKey, chunksCache, storagePlugin, Random.nextInt(1, 42))
private val chunkWriter = ChunkWriter(
streamCrypto = streamCrypto,
streamKey = streamKey,
chunksCache = chunksCache,
backend = backend,
androidId = androidId,
bufferSize = Random.nextInt(1, 42),
)
private val chunkId1 = Random.nextBytes(KEY_SIZE_BYTES).toHexString()
private val chunkId2 = Random.nextBytes(KEY_SIZE_BYTES).toHexString()
@ -66,9 +75,9 @@ internal class ChunkWriterTest {
every { chunksCache.get(chunkId3) } returns null
// get the output streams for the chunks
coEvery { storagePlugin.getChunkOutputStream(chunkId1) } returns chunk1Output
coEvery { storagePlugin.getChunkOutputStream(chunkId2) } returns chunk2Output
coEvery { storagePlugin.getChunkOutputStream(chunkId3) } returns chunk3Output
coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output
coEvery { backend.save(Blob(androidId, chunkId2)) } returns chunk2Output
coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output
// get AD
every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1
@ -122,7 +131,7 @@ internal class ChunkWriterTest {
every { chunksCache.get(chunkId3) } returns null
// get and wrap the output stream for chunk that is missing
coEvery { storagePlugin.getChunkOutputStream(chunkId1) } returns chunk1Output
coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output
every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1
every {
streamCrypto.newEncryptingStream(streamKey, chunk1Output, bytes(34))
@ -132,7 +141,7 @@ internal class ChunkWriterTest {
every { chunksCache.insert(chunks[0].toCachedChunk()) } just Runs
// get and wrap the output stream for chunk that isn't cached
coEvery { storagePlugin.getChunkOutputStream(chunkId3) } returns chunk3Output
coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output
every { streamCrypto.getAssociatedDataForChunk(chunkId3) } returns ad3
every {
streamCrypto.newEncryptingStream(streamKey, chunk3Output, bytes(34))
@ -175,8 +184,8 @@ internal class ChunkWriterTest {
every { chunksCache.get(chunkId3) } returns null
// get the output streams for the chunks
coEvery { storagePlugin.getChunkOutputStream(chunkId1) } returns chunk1Output
coEvery { storagePlugin.getChunkOutputStream(chunkId3) } returns chunk3Output
coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output
coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output
// get AD
every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1

28
storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunksCacheRepopulaterTest.kt
View file

@ -11,16 +11,19 @@ import io.mockk.coVerify
import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.slot
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.db.CachedChunk
import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
@ -30,15 +33,22 @@ internal class ChunksCacheRepopulaterTest {
private val db: Db = mockk()
private val chunksCache: ChunksCache = mockk()
private val pluginGetter: () -> StoragePlugin = mockk()
private val plugin: StoragePlugin = mockk()
private val backendGetter: () -> Backend = mockk()
private val androidId: String = getRandomString()
private val backend: Backend = mockk()
private val snapshotRetriever: SnapshotRetriever = mockk()
private val streamKey = "This is a backup key for testing".toByteArray()
private val cacheRepopulater = ChunksCacheRepopulater(db, pluginGetter, snapshotRetriever)
private val cacheRepopulater = ChunksCacheRepopulater(
db = db,
storagePlugin = backendGetter,
androidId = androidId,
snapshotRetriever = snapshotRetriever,
)
init {
mockLog()
every { pluginGetter() } returns plugin
mockkStatic("org.calyxos.backup.storage.SnapshotRetrieverKt")
every { backendGetter() } returns backend
every { db.getChunksCache() } returns chunksCache
}
@ -73,7 +83,7 @@ internal class ChunksCacheRepopulaterTest {
) // chunk3 is not referenced and should get deleted
val cachedChunksSlot = slot<Collection<CachedChunk>>()
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots
coEvery { backend.getCurrentBackupSnapshots(androidId) } returns storedSnapshots
coEvery {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot1)
} returns snapshot1
@ -81,14 +91,14 @@ internal class ChunksCacheRepopulaterTest {
snapshotRetriever.getSnapshot(streamKey, storedSnapshot2)
} returns snapshot2
every { chunksCache.clearAndRepopulate(db, capture(cachedChunksSlot)) } just Runs
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs
coEvery { backend.remove(Blob(androidId, chunk3)) } just Runs
cacheRepopulater.repopulate(streamKey, availableChunkIds)
assertTrue(cachedChunksSlot.isCaptured)
assertEquals(cachedChunks.toSet(), cachedChunksSlot.captured.toSet())
coVerify { plugin.deleteChunks(listOf(chunk3)) }
coVerify { backend.remove(Blob(androidId, chunk3)) }
}
}

10
storage/lib/src/test/java/org/calyxos/backup/storage/backup/SmallFileBackupIntegrationTest.kt
View file

@ -14,7 +14,6 @@ import io.mockk.just
import io.mockk.mockk
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.BackupObserver
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.crypto.Hkdf.KEY_SIZE_BYTES
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.CachedChunk
@ -24,6 +23,7 @@ import org.calyxos.backup.storage.getRandomDocFile
import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.toHexString
import org.calyxos.seedvault.core.backends.Backend
import org.junit.Assert.assertEquals
import org.junit.Test
import java.io.ByteArrayInputStream
@ -39,13 +39,15 @@ internal class SmallFileBackupIntegrationTest {
private val filesCache: FilesCache = mockk()
private val mac: Mac = mockk()
private val chunksCache: ChunksCache = mockk()
private val storagePlugin: StoragePlugin = mockk()
private val backend: Backend = mockk()
private val androidId: String = getRandomString()
private val chunkWriter = ChunkWriter(
streamCrypto = StreamCrypto,
streamKey = Random.nextBytes(KEY_SIZE_BYTES),
chunksCache = chunksCache,
storagePlugin = storagePlugin,
backend = backend,
androidId = androidId,
)
private val zipChunker = ZipChunker(
mac = mac,
@ -91,7 +93,7 @@ internal class SmallFileBackupIntegrationTest {
every { mac.doFinal(any<ByteArray>()) } returns chunkId
every { chunksCache.get(any()) } returns null
coEvery { storagePlugin.getChunkOutputStream(any()) } returns outputStream2
coEvery { backend.save(any()) } returns outputStream2
every {
chunksCache.insert(match<CachedChunk> { cachedChunk ->
cachedChunk.id == chunkId.toHexString() &&

32
storage/lib/src/test/java/org/calyxos/backup/storage/prune/PrunerTest.kt
View file

@ -10,9 +10,9 @@ import io.mockk.coEvery
import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.slot
import kotlinx.coroutines.runBlocking
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.backup.BackupDocumentFile
import org.calyxos.backup.storage.backup.BackupMediaFile
@ -25,7 +25,10 @@ import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.getRandomString
import org.calyxos.backup.storage.mockLog
import org.calyxos.backup.storage.plugin.SnapshotRetriever
import org.calyxos.backup.storage.SnapshotRetriever
import org.calyxos.backup.storage.getCurrentBackupSnapshots
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.calyxos.seedvault.core.crypto.KeyManager
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
@ -37,9 +40,10 @@ internal class PrunerTest {
private val db: Db = mockk()
private val chunksCache: ChunksCache = mockk()
private val pluginGetter: () -> StoragePlugin = mockk()
private val backendGetter: () -> Backend = mockk()
private val androidId: String = getRandomString()
private val keyManager: KeyManager = mockk()
private val plugin: StoragePlugin = mockk()
private val backend: Backend = mockk()
private val snapshotRetriever: SnapshotRetriever = mockk()
private val retentionManager: RetentionManager = mockk()
private val streamCrypto: StreamCrypto = mockk()
@ -48,14 +52,22 @@ internal class PrunerTest {
init {
mockLog(false)
every { pluginGetter() } returns plugin
mockkStatic("org.calyxos.backup.storage.SnapshotRetrieverKt")
every { backendGetter() } returns backend
every { db.getChunksCache() } returns chunksCache
every { keyManager.getMainKey() } returns masterKey
every { streamCrypto.deriveStreamKey(masterKey) } returns streamKey
}
private val pruner =
Pruner(db, retentionManager, pluginGetter, keyManager, snapshotRetriever, streamCrypto)
private val pruner = Pruner(
db = db,
retentionManager = retentionManager,
storagePluginGetter = backendGetter,
androidId = androidId,
keyManager = keyManager,
snapshotRetriever = snapshotRetriever,
streamCrypto = streamCrypto,
)
@Test
fun test() = runBlocking {
@ -84,12 +96,12 @@ internal class PrunerTest {
val actualChunks2 = slot<Collection<String>>()
val cachedChunk3 = CachedChunk(chunk3, 0, 0)
coEvery { plugin.getCurrentBackupSnapshots() } returns storedSnapshots
coEvery { backend.getCurrentBackupSnapshots(androidId) } returns storedSnapshots
every {
retentionManager.getSnapshotsToDelete(storedSnapshots)
} returns listOf(storedSnapshot1)
coEvery { snapshotRetriever.getSnapshot(streamKey, storedSnapshot1) } returns snapshot1
coEvery { plugin.deleteBackupSnapshot(storedSnapshot1) } just Runs
coEvery { backend.remove(storedSnapshot1.snapshotHandle) } just Runs
every {
db.applyInParts(capture(actualChunks), captureLambda())
} answers {
@ -97,7 +109,7 @@ internal class PrunerTest {
}
every { chunksCache.decrementRefCount(capture(actualChunks2)) } just Runs
every { chunksCache.getUnreferencedChunks() } returns listOf(cachedChunk3)
coEvery { plugin.deleteChunks(listOf(chunk3)) } just Runs
coEvery { backend.remove(Blob(androidId, chunk3)) } just Runs
every { chunksCache.deleteChunks(listOf(cachedChunk3)) } just Runs
pruner.prune(null)