Add prototype plumbing for new v2 app backup

.idea/codeStyles/Project.xml

@@ -1,12 +1,7 @@
 <component name="ProjectCodeStyleConfiguration">
   <code_scheme name="Project" version="173">
     <JetCodeStyleSettings>
-      <option name="PACKAGES_TO_USE_STAR_IMPORTS">
+      <option name="LINE_BREAK_AFTER_MULTILINE_WHEN_ENTRY" value="false" />
-        <value />
-      </option>
-      <option name="NAME_COUNT_TO_USE_STAR_IMPORT" value="2147483647" />
-      <option name="NAME_COUNT_TO_USE_STAR_IMPORT_FOR_MEMBERS" value="2147483647" />
-      <option name="ALLOW_TRAILING_COMMA" value="true" />
       <option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
     </JetCodeStyleSettings>
     <codeStyleSettings language="XML">

Android.bp

@@ -32,6 +32,9 @@ android_app {
         "com.google.android.material_material",
         "kotlinx-coroutines-android",
         "kotlinx-coroutines-core",
+        // app backup related libs
+        "seedvault-lib-kotlin-logging-jvm",
+        "seedvault-lib-chunker",
         "seedvault-lib-zstd-jni",
         // our own gradle module libs
         "seedvault-lib-core",

app/build.gradle.kts

@@ -157,6 +157,8 @@ dependencies {
 
     implementation(libs.google.protobuf.javalite)
     implementation(libs.google.tink.android)
+    implementation(libs.kotlin.logging)
+    implementation(libs.squareup.okio)
 
     /**
      * Storage Dependencies
@@ -175,6 +177,7 @@ dependencies {
     implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.jar"))
     implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.aar"))
 
+    implementation(fileTree("${rootProject.rootDir}/libs").include("seedvault-chunker-0.1.jar"))
     implementation(fileTree("${rootProject.rootDir}/libs").include("zstd-jni-1.5.6-5.aar"))
     implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar"))
 
@@ -188,6 +191,7 @@ dependencies {
     // anything less than 'implementation' fails tests run with gradlew
     testImplementation(aospLibs)
     testImplementation("androidx.test.ext:junit:1.1.5")
+    testImplementation("org.slf4j:slf4j-simple:2.0.3")
     testImplementation("org.robolectric:robolectric:4.12.2")
     testImplementation("org.hamcrest:hamcrest:2.2")
     testImplementation("org.junit.jupiter:junit-jupiter-api:${libs.versions.junit5.get()}")
@@ -198,6 +202,7 @@ dependencies {
     )
     testImplementation("app.cash.turbine:turbine:1.0.0")
     testImplementation("org.bitcoinj:bitcoinj-core:0.16.2")
+    testImplementation("com.github.luben:zstd-jni:1.5.6-5")
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${libs.versions.junit5.get()}")
     testRuntimeOnly("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}")
 

app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt

@@ -7,8 +7,13 @@ package com.stevesoltys.seedvault.restore
 
 import com.stevesoltys.seedvault.metadata.BackupMetadata
 import com.stevesoltys.seedvault.metadata.PackageMetadataMap
+import com.stevesoltys.seedvault.proto.Snapshot
 
-data class RestorableBackup(val backupMetadata: BackupMetadata) {
+data class RestorableBackup(
+    val backupMetadata: BackupMetadata,
+    val repoId: String? = null,
+    val snapshot: Snapshot? = null,
+) {
 
     val name: String
         get() = backupMetadata.deviceName

app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt

@@ -0,0 +1,83 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport
+
+import com.github.luben.zstd.ZstdOutputStream
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.header.VERSION
+import com.stevesoltys.seedvault.proto.Snapshot
+import com.stevesoltys.seedvault.transport.restore.Loader
+import io.github.oshai.kotlinlogging.KotlinLogging
+import okio.Buffer
+import okio.buffer
+import okio.sink
+import org.calyxos.seedvault.core.backends.AppBackupFileType
+import org.calyxos.seedvault.core.backends.TopLevelFolder
+
+internal class SnapshotManager(
+    private val crypto: Crypto,
+    private val loader: Loader,
+    private val backendManager: BackendManager,
+) {
+
+    private val log = KotlinLogging.logger {}
+
+    /**
+     * The latest [Snapshot]. May be stale if [loadSnapshots] has not returned
+     * or wasn't called since new snapshots have been created.
+     */
+    var latestSnapshot: Snapshot? = null
+        private set
+
+    suspend fun loadSnapshots(callback: (Snapshot) -> Unit) {
+        log.info { "Loading snapshots..." }
+        val handles = mutableListOf<AppBackupFileType.Snapshot>()
+        backendManager.backend.list(
+            topLevelFolder = TopLevelFolder(crypto.repoId),
+            AppBackupFileType.Snapshot::class,
+        ) { fileInfo ->
+            fileInfo.fileHandle as AppBackupFileType.Snapshot
+            handles.add(fileInfo.fileHandle as AppBackupFileType.Snapshot)
+        }
+        handles.forEach { fileHandle ->
+            // TODO is it a fatal error when one snapshot is corrupted or couldn't get loaded?
+            val snapshot = onSnapshotFound(fileHandle)
+            callback(snapshot)
+        }
+    }
+
+    private suspend fun onSnapshotFound(snapshotHandle: AppBackupFileType.Snapshot): Snapshot {
+        // TODO set up local snapshot cache, so we don't need to download those all the time
+        val snapshot = loader.loadFile(snapshotHandle).use { inputStream ->
+            Snapshot.parseFrom(inputStream)
+        }
+        // update latest snapshot if this one is more recent
+        if (snapshot.token > (latestSnapshot?.token ?: 0)) latestSnapshot = snapshot
+        return snapshot
+    }
+
+    suspend fun saveSnapshot(snapshot: Snapshot) {
+        val buffer = Buffer()
+        val bufferStream = buffer.outputStream()
+        bufferStream.write(VERSION.toInt())
+        crypto.newEncryptingStream(bufferStream, crypto.getAdForVersion()).use { cryptoStream ->
+            ZstdOutputStream(cryptoStream).use { zstdOutputStream ->
+                snapshot.writeTo(zstdOutputStream)
+            }
+        }
+        val sha256ByteString = buffer.sha256()
+        val handle = AppBackupFileType.Snapshot(crypto.repoId, sha256ByteString.hex())
+        // TODO exception handling
+        backendManager.backend.save(handle).use { outputStream ->
+            outputStream.sink().buffer().apply {
+                writeAll(buffer)
+                flush() // needs flushing
+            }
+        }
+    }
+
+}

app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt

@@ -0,0 +1,51 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.backup
+
+import com.stevesoltys.seedvault.transport.SnapshotManager
+import io.github.oshai.kotlinlogging.KotlinLogging
+import kotlinx.coroutines.delay
+
+internal class AppBackupManager(
+    private val blobsCache: BlobsCache,
+    private val snapshotManager: SnapshotManager,
+    private val snapshotCreatorFactory: SnapshotCreatorFactory,
+) {
+
+    private val log = KotlinLogging.logger {}
+    var snapshotCreator: SnapshotCreator? = null
+        private set
+
+    suspend fun beforeBackup() {
+        log.info { "Before backup" }
+        snapshotCreator = snapshotCreatorFactory.createSnapshotCreator()
+        blobsCache.populateCache()
+    }
+
+    suspend fun afterBackupFinished() {
+        log.info { "After backup finished" }
+        blobsCache.clear()
+        val snapshot = snapshotCreator?.finalizeSnapshot() ?: error("Had no snapshotCreator")
+        keepTrying {
+            snapshotManager.saveSnapshot(snapshot)
+        }
+        snapshotCreator = null
+    }
+
+    private suspend fun keepTrying(n: Int = 3, block: suspend () -> Unit) {
+        for (i in 1..n) {
+            try {
+                block()
+                return
+            } catch (e: Exception) {
+                if (i == n) throw e
+                log.error(e) { "Error (#$i), we'll keep trying" }
+                delay(1000)
+            }
+        }
+    }
+
+}

app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt

@@ -5,11 +5,17 @@
 
 package com.stevesoltys.seedvault.transport.backup
 
+import com.stevesoltys.seedvault.transport.SnapshotManager
 import org.koin.android.ext.koin.androidContext
 import org.koin.dsl.module
 
 val backupModule = module {
     single { BackupInitializer(get()) }
+    single { BackupReceiver(get(), get(), get()) }
+    single { BlobsCache(get(), get(), get()) }
+    single { BlobCreator(get(), get()) }
+    single { SnapshotManager(get(), get(), get()) }
+    single { SnapshotCreatorFactory(androidContext(), get(), get(), get()) }
     single { InputFactory() }
     single {
         PackageService(

app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt

@@ -0,0 +1,92 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.backup
+
+import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.proto.Snapshot.Blob
+import org.calyxos.seedvault.chunker.Chunk
+import org.calyxos.seedvault.chunker.Chunker
+import org.calyxos.seedvault.chunker.GearTableCreator
+import org.calyxos.seedvault.core.toHexString
+import java.io.InputStream
+
+data class BackupData(
+    val chunks: List<String>,
+    val chunkMap: Map<String, Blob>,
+)
+
+internal class BackupReceiver(
+    private val blobsCache: BlobsCache,
+    private val blobCreator: BlobCreator,
+    private val crypto: Crypto,
+    private val replaceableChunker: Chunker? = null,
+) {
+
+    private val chunker: Chunker by lazy {
+        // crypto.gearTableKey is not available at creation time, so use lazy instantiation
+        replaceableChunker ?: Chunker(
+            minSize = 1536 * 1024, // 1.5 MB
+            avgSize = 3 * 1024 * 1024, // 3.0 MB
+            maxSize = 7680 * 1024, // 7.5 MB
+            normalization = 1,
+            gearTable = GearTableCreator.create(crypto.gearTableKey),
+            hashFunction = { bytes ->
+                crypto.sha256(bytes).toHexString()
+            },
+        )
+    }
+    private val chunks = mutableListOf<String>()
+    private val chunkMap = mutableMapOf<String, Blob>()
+
+    suspend fun addBytes(bytes: ByteArray) {
+        chunker.addBytes(bytes).forEach { chunk ->
+            onNewChunk(chunk)
+        }
+    }
+
+    suspend fun readFromStream(inputStream: InputStream) {
+        try {
+            val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
+            var bytes = inputStream.read(buffer)
+            while (bytes >= 0) {
+                if (bytes == buffer.size) {
+                    addBytes(buffer)
+                } else {
+                    addBytes(buffer.copyOfRange(0, bytes))
+                }
+                bytes = inputStream.read(buffer)
+            }
+        } catch (e: Exception) {
+            finalize()
+            throw e
+        }
+    }
+
+    suspend fun finalize(): BackupData {
+        chunker.finalize().forEach { chunk ->
+            onNewChunk(chunk)
+        }
+        // copy chunks and chunkMap before clearing
+        val backupData = BackupData(chunks.toList(), chunkMap.toMap())
+        chunks.clear()
+        chunkMap.clear()
+        return backupData
+    }
+
+    private suspend fun onNewChunk(chunk: Chunk) {
+        chunks.add(chunk.hash)
+
+        val existingBlob = blobsCache.getBlob(chunk.hash)
+        if (existingBlob == null) {
+            val blob = blobCreator.createNewBlob(chunk)
+            chunkMap[chunk.hash] = blob
+            blobsCache.saveNewBlob(chunk.hash, blob)
+        } else {
+            chunkMap[chunk.hash] = existingBlob
+        }
+    }
+
+}

app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.backup
+
+import com.github.luben.zstd.ZstdOutputStream
+import com.google.protobuf.ByteString
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.header.VERSION
+import com.stevesoltys.seedvault.proto.Snapshot.Blob
+import okio.Buffer
+import okio.buffer
+import okio.sink
+import org.calyxos.seedvault.chunker.Chunk
+import org.calyxos.seedvault.core.backends.AppBackupFileType
+
+internal class BlobCreator(
+    private val crypto: Crypto,
+    private val backendManager: BackendManager,
+) {
+
+    private val buffer = Buffer()
+
+    suspend fun createNewBlob(chunk: Chunk): Blob {
+        buffer.clear()
+        val bufferStream = buffer.outputStream()
+        bufferStream.write(VERSION.toInt())
+        crypto.newEncryptingStream(bufferStream, crypto.getAdForVersion()).use { cryptoStream ->
+            ZstdOutputStream(cryptoStream).use { zstdOutputStream ->
+                zstdOutputStream.write(chunk.data)
+            }
+        }
+        val sha256ByteString = buffer.sha256()
+        val handle = AppBackupFileType.Blob(crypto.repoId, sha256ByteString.hex())
+        // TODO exception handling and retries
+        val size = backendManager.backend.save(handle).use { outputStream ->
+            val outputBuffer = outputStream.sink().buffer()
+            val length = outputBuffer.writeAll(buffer)
+            // flushing is important here, otherwise data doesn't get fully written!
+            outputBuffer.flush()
+            length
+        }
+        return Blob.newBuilder()
+            .setId(ByteString.copyFrom(sha256ByteString.asByteBuffer()))
+            .setLength(size.toInt())
+            .setUncompressedLength(chunk.length)
+            .build()
+    }
+}

app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt

@@ -0,0 +1,69 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.backup
+
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.proto.Snapshot.Blob
+import com.stevesoltys.seedvault.transport.SnapshotManager
+import io.github.oshai.kotlinlogging.KotlinLogging
+import org.calyxos.seedvault.core.backends.AppBackupFileType
+import org.calyxos.seedvault.core.backends.TopLevelFolder
+
+internal class BlobsCache(
+    private val crypto: Crypto,
+    private val backendManager: BackendManager,
+    private val snapshotManager: SnapshotManager,
+) {
+
+    private val log = KotlinLogging.logger {}
+    private val blobMap = mutableMapOf<String, Blob>()
+
+    /**
+     * This must be called before saving files to the backend to avoid uploading duplicate blobs.
+     */
+    suspend fun populateCache() {
+        log.info { "Getting all blobs from backend..." }
+        blobMap.clear()
+        val blobs = mutableSetOf<String>()
+        backendManager.backend.list(
+            topLevelFolder = TopLevelFolder(crypto.repoId),
+            AppBackupFileType.Blob::class,
+        ) { fileInfo ->
+            fileInfo.fileHandle as AppBackupFileType.Blob
+            // TODO we could save size info here and later check it is as expected
+            blobs.add(fileInfo.fileHandle.name)
+        }
+        snapshotManager.loadSnapshots { snapshot ->
+            snapshot.blobsMap.forEach { (chunkId, blob) ->
+                // check if referenced blob still exists on backend
+                if (blobs.contains(blob.id.hexFromProto())) {
+                    // only add blob to our mapping, if it still exists
+                    blobMap.putIfAbsent(chunkId, blob)?.let { previous ->
+                        if (previous.id != blob.id) log.warn {
+                            "Chunk ID ${chunkId.substring(0..5)} had more than one blob"
+                        }
+                    }
+                } else log.warn {
+                    "Blob ${blob.id.hexFromProto()} referenced in snapshot ${snapshot.token}"
+                }
+            }
+        }
+    }
+
+    fun getBlob(hash: String): Blob? = blobMap[hash]
+
+    fun saveNewBlob(chunkId: String, blob: Blob) {
+        blobMap[chunkId] = blob
+        // TODO persist this blob locally in case backup gets interrupted
+    }
+
+    fun clear() {
+        log.info { "Clearing cache..." }
+        blobMap.clear()
+    }
+
+}

app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt

@@ -0,0 +1,132 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.backup
+
+import android.annotation.SuppressLint
+import android.content.Context
+import android.content.pm.PackageInfo
+import android.content.pm.PackageManager.PERMISSION_GRANTED
+import android.os.Build
+import android.os.UserManager
+import android.provider.Settings
+import android.provider.Settings.Secure.ANDROID_ID
+import com.google.protobuf.ByteString
+import com.stevesoltys.seedvault.Clock
+import com.stevesoltys.seedvault.metadata.BackupType
+import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA
+import com.stevesoltys.seedvault.proto.Snapshot
+import com.stevesoltys.seedvault.proto.Snapshot.Apk
+import com.stevesoltys.seedvault.proto.Snapshot.App
+import com.stevesoltys.seedvault.proto.Snapshot.Blob
+import com.stevesoltys.seedvault.settings.SettingsManager
+import org.calyxos.seedvault.core.toHexString
+
+internal class SnapshotCreatorFactory(
+    private val context: Context,
+    private val clock: Clock,
+    private val packageService: PackageService,
+    private val settingsManager: SettingsManager,
+) {
+    fun createSnapshotCreator() = SnapshotCreator(context, clock, packageService, settingsManager)
+}
+
+internal class SnapshotCreator(
+    private val context: Context,
+    private val clock: Clock,
+    private val packageService: PackageService,
+    private val settingsManager: SettingsManager,
+) {
+
+    private val snapshotBuilder = Snapshot.newBuilder()
+        .setToken(clock.time())
+    private val appBuilderMap = mutableMapOf<String, App.Builder>()
+    private val blobsMap = mutableMapOf<String, Blob>()
+
+    private val launchableSystemApps by lazy {
+        packageService.launchableSystemApps.map { it.activityInfo.packageName }.toSet()
+    }
+
+    fun onApkBackedUp(
+        packageName: String,
+        apk: Apk,
+        chunkMap: Map<String, Blob>,
+    ) {
+        val appBuilder = appBuilderMap.getOrPut(packageName) {
+            App.newBuilder()
+        }
+        appBuilder.setApk(apk)
+        blobsMap.putAll(chunkMap)
+    }
+
+    fun onPackageBackedUp(
+        packageInfo: PackageInfo,
+        type: BackupType,
+        backupData: BackupData,
+    ) {
+        val packageName = packageInfo.packageName
+        val builder = appBuilderMap.getOrPut(packageName) {
+            App.newBuilder()
+        }
+        val isSystemApp = packageInfo.isSystemApp()
+        val chunkIds = backupData.chunks.forProto()
+        blobsMap.putAll(backupData.chunkMap)
+        builder
+            .setTime(clock.time())
+            .setState(APK_AND_DATA.name)
+            .setType(type.forSnapshot())
+            .setName(packageInfo.applicationInfo?.loadLabel(context.packageManager)?.toString())
+            .setSystem(isSystemApp)
+            .setLaunchableSystemApp(isSystemApp && launchableSystemApps.contains(packageName))
+            .addAllChunkIds(chunkIds)
+    }
+
+    fun onIconsBackedUp(backupData: BackupData) {
+        snapshotBuilder.addAllIconChunkIds(backupData.chunks.forProto())
+        blobsMap.putAll(backupData.chunkMap)
+    }
+
+    fun finalizeSnapshot(): Snapshot {
+        val userName = getUserName()
+        val deviceName = if (userName == null) {
+            "${Build.MANUFACTURER} ${Build.MODEL}"
+        } else {
+            "${Build.MANUFACTURER} ${Build.MODEL} - $userName"
+        }
+
+        @SuppressLint("HardwareIds")
+        val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID)
+        val snapshot = snapshotBuilder
+            .setName(deviceName)
+            .setAndroidId(androidId)
+            .setSdkInt(Build.VERSION.SDK_INT)
+            .setAndroidIncremental(Build.VERSION.INCREMENTAL)
+            .setD2D(settingsManager.d2dBackupsEnabled())
+            .putAllApps(appBuilderMap.mapValues { it.value.build() })
+            .putAllBlobs(blobsMap)
+            .build()
+        appBuilderMap.clear()
+        snapshotBuilder.clear()
+        return snapshot
+    }
+
+    private fun getUserName(): String? {
+        val perm = "android.permission.QUERY_USERS"
+        return if (context.checkSelfPermission(perm) == PERMISSION_GRANTED) {
+            val userManager = context.getSystemService(UserManager::class.java) ?: return null
+            userManager.userName
+        } else null
+    }
+
+    private fun BackupType.forSnapshot(): Snapshot.BackupType = when (this) {
+        BackupType.KV -> Snapshot.BackupType.KV
+        BackupType.FULL -> Snapshot.BackupType.FULL
+    }
+
+}
+
+fun Iterable<String>.forProto() = map { ByteString.fromHex(it) }
+fun Iterable<ByteString>.hexFromProto() = map { it.toByteArray().toHexString() }
+fun ByteString.hexFromProto() = toByteArray().toHexString()

app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt

@@ -0,0 +1,71 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.restore
+
+import com.github.luben.zstd.ZstdInputStream
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.header.UnsupportedVersionException
+import com.stevesoltys.seedvault.header.VERSION
+import kotlinx.coroutines.runBlocking
+import org.calyxos.seedvault.core.backends.AppBackupFileType
+import org.calyxos.seedvault.core.toHexString
+import java.io.ByteArrayInputStream
+import java.io.InputStream
+import java.io.SequenceInputStream
+import java.security.GeneralSecurityException
+import java.util.Enumeration
+
+internal class Loader(
+    private val crypto: Crypto,
+    private val backendManager: BackendManager,
+) {
+
+    /**
+     * The responsibility with closing the returned stream lies with the caller.
+     */
+    suspend fun loadFile(handle: AppBackupFileType): InputStream {
+        // We load the entire ciphertext into memory,
+        // so we can check the SHA-256 hash before decrypting and parsing the data.
+        val cipherText = backendManager.backend.load(handle).use { inputStream ->
+            inputStream.readAllBytes()
+        }
+        // check SHA-256 hash first thing
+        val sha256 = crypto.sha256(cipherText).toHexString()
+        val expectedHash = when (handle) {
+            is AppBackupFileType.Snapshot -> handle.hash
+            is AppBackupFileType.Blob -> handle.name
+        }
+        if (sha256 != expectedHash) {
+            throw GeneralSecurityException("File had wrong SHA-256 hash: $handle")
+        }
+        // check that we can handle the version of that snapshot
+        val version = cipherText[0]
+        if (version <= 1) throw GeneralSecurityException("Unexpected version: $version")
+        if (version > VERSION) throw UnsupportedVersionException(version)
+        // get associated data for version, used for authenticated decryption
+        val ad = crypto.getAdForVersion(version)
+        // skip first version byte when creating cipherText stream
+        val inputStream = ByteArrayInputStream(cipherText, 1, cipherText.size - 1)
+        // decrypt and decompress cipherText stream and parse snapshot
+        return ZstdInputStream(crypto.newDecryptingStream(inputStream, ad))
+    }
+
+    suspend fun loadFiles(handles: List<AppBackupFileType>): InputStream {
+        val enumeration: Enumeration<InputStream> = object : Enumeration<InputStream> {
+            val iterator = handles.iterator()
+
+            override fun hasMoreElements(): Boolean {
+                return iterator.hasNext()
+            }
+
+            override fun nextElement(): InputStream {
+                return runBlocking { loadFile(iterator.next()) }
+            }
+        }
+        return SequenceInputStream(enumeration)
+    }
+}

app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt

@@ -10,6 +10,7 @@ import org.koin.dsl.module
 
 val restoreModule = module {
     single { OutputFactory() }
+    single { Loader(get(), get()) }
     single { KVRestore(get(), get(), get(), get(), get(), get()) }
     single { FullRestore(get(), get(), get(), get(), get()) }
     single {

app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt

@@ -5,6 +5,7 @@
 
 package com.stevesoltys.seedvault.worker
 
+import com.stevesoltys.seedvault.transport.backup.AppBackupManager
 import org.koin.android.ext.koin.androidContext
 import org.koin.dsl.module
 
@@ -23,6 +24,7 @@ val workerModule = module {
             crypto = get(),
         )
     }
+    single { AppBackupManager(get(), get(), get()) }
     single {
         ApkBackup(
             pm = androidContext().packageManager,

app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt

@@ -0,0 +1,100 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport
+
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.crypto.Crypto
+import com.stevesoltys.seedvault.getRandomString
+import com.stevesoltys.seedvault.proto.Snapshot
+import com.stevesoltys.seedvault.transport.restore.Loader
+import io.mockk.coEvery
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.slot
+import kotlinx.coroutines.runBlocking
+import org.calyxos.seedvault.core.backends.AppBackupFileType
+import org.calyxos.seedvault.core.backends.Backend
+import org.calyxos.seedvault.core.backends.FileInfo
+import org.calyxos.seedvault.core.backends.TopLevelFolder
+import org.calyxos.seedvault.core.toByteArrayFromHex
+import org.calyxos.seedvault.core.toHexString
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Test
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.security.MessageDigest
+import kotlin.random.Random
+
+class SnapshotManagerTest {
+
+    private val crypto: Crypto = mockk()
+    private val backendManager: BackendManager = mockk()
+    private val backend: Backend = mockk()
+
+    private val loader = Loader(crypto, backendManager) // need a real loader
+    private val snapshotManager = SnapshotManager(crypto, loader, backendManager)
+
+    private val ad = Random.nextBytes(1)
+    private val passThroughOutputStream = slot<OutputStream>()
+    private val passThroughInputStream = slot<InputStream>()
+    private val repoId = Random.nextBytes(32).toHexString()
+    private val snapshotHandle = slot<AppBackupFileType.Snapshot>()
+    private val snapshot = Snapshot.newBuilder()
+        .setToken(Random.nextLong())
+        .setName(getRandomString())
+        .setSdkInt(Random.nextInt())
+        .putAllBlobs(mapOf(getRandomString() to Snapshot.Blob.getDefaultInstance()))
+        .build()
+
+    @Test
+    fun `test saving and loading`() = runBlocking {
+        val outputStream = ByteArrayOutputStream()
+
+        every { crypto.getAdForVersion() } returns ad
+        every { crypto.newEncryptingStream(capture(passThroughOutputStream), ad) } answers {
+            passThroughOutputStream.captured // not really encrypting here
+        }
+        every { crypto.repoId } returns repoId
+        every { backendManager.backend } returns backend
+        coEvery { backend.save(capture(snapshotHandle)) } returns outputStream
+
+        snapshotManager.saveSnapshot(snapshot)
+
+        // check that file content hash matches snapshot hash
+        val messageDigest = MessageDigest.getInstance("SHA-256")
+        assertEquals(
+            messageDigest.digest(outputStream.toByteArray()).toHexString(),
+            snapshotHandle.captured.hash,
+        )
+
+        val fileInfo = FileInfo(snapshotHandle.captured, Random.nextLong())
+        assertTrue(outputStream.size() > 0)
+        val inputStream = ByteArrayInputStream(outputStream.toByteArray())
+        coEvery {
+            backend.list(
+                topLevelFolder = TopLevelFolder(repoId),
+                AppBackupFileType.Snapshot::class,
+                callback = captureLambda<(FileInfo) -> Unit>()
+            )
+        } answers {
+            lambda<(FileInfo) -> Unit>().captured.invoke(fileInfo)
+        }
+        coEvery { backend.load(snapshotHandle.captured) } returns inputStream
+        every {
+            crypto.sha256(outputStream.toByteArray())
+        } returns snapshotHandle.captured.hash.toByteArrayFromHex()
+        every { crypto.newDecryptingStream(capture(passThroughInputStream), ad) } answers {
+            passThroughInputStream.captured
+        }
+
+        var loadedSnapshot: Snapshot? = null
+        snapshotManager.loadSnapshots { loadedSnapshot = it }
+        assertEquals(snapshot, loadedSnapshot)
+    }
+}

app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCreatorTest.kt

@@ -0,0 +1,71 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.transport.backup
+
+import com.stevesoltys.seedvault.backend.BackendManager
+import com.stevesoltys.seedvault.transport.TransportTest
+import io.mockk.coEvery
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.slot
+import kotlinx.coroutines.runBlocking
+import org.calyxos.seedvault.chunker.Chunk
+import org.calyxos.seedvault.core.backends.AppBackupFileType
+import org.calyxos.seedvault.core.backends.Backend
+import org.calyxos.seedvault.core.toHexString
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+import java.io.ByteArrayOutputStream
+import java.io.OutputStream
+import java.security.MessageDigest
+import kotlin.random.Random
+
+internal class BlobCreatorTest : TransportTest() {
+
+    private val backendManager: BackendManager = mockk()
+    private val backend: Backend = mockk()
+    private val blobCreator = BlobCreator(crypto, backendManager)
+
+    private val ad = Random.nextBytes(1)
+    private val passThroughOutputStream = slot<OutputStream>()
+    private val blobHandle = slot<AppBackupFileType.Blob>()
+
+    @Test
+    fun `test re-use for hashing two chunks`() = runBlocking {
+        val data1 = Random.nextBytes(1337)
+        val data2 = Random.nextBytes(2342)
+        val chunk1 = Chunk(0L, data1.size, data1, "doesn't matter here")
+        val chunk2 = Chunk(0L, data2.size, data2, "doesn't matter here")
+        val outputStream1 = ByteArrayOutputStream()
+        val outputStream2 = ByteArrayOutputStream()
+
+        every { crypto.getAdForVersion() } returns ad
+        every { crypto.newEncryptingStream(capture(passThroughOutputStream), ad) } answers {
+            passThroughOutputStream.captured // not really encrypting here
+        }
+        every { crypto.repoId } returns repoId
+        every { backendManager.backend } returns backend
+        coEvery { backend.save(capture(blobHandle)) } returns outputStream1
+
+        blobCreator.createNewBlob(chunk1)
+        // check that file content hash matches snapshot hash
+        val messageDigest = MessageDigest.getInstance("SHA-256")
+        assertEquals(
+            messageDigest.digest(outputStream1.toByteArray()).toHexString(),
+            blobHandle.captured.name,
+        )
+
+        // use same BlobCreator to create another blob, because we re-use a single buffer
+        // and need to check clearing that does work as expected
+        coEvery { backend.save(capture(blobHandle)) } returns outputStream2
+        blobCreator.createNewBlob(chunk2)
+        // check that file content hash matches snapshot hash
+        assertEquals(
+            messageDigest.digest(outputStream2.toByteArray()).toHexString(),
+            blobHandle.captured.name,
+        )
+    }
+}

libs/Android.bp

@@ -16,6 +16,12 @@ java_import {
     sdk_version: "current",
 }
 
+java_import {
+    name: "seedvault-lib-chunker",
+    jars: ["seedvault-chunker-0.1.jar"],
+    sdk_version: "current",
+}
+
 java_import {
     name: "seedvault-lib-kotlin-logging-jvm",
     jars: ["kotlin-logging-jvm-6.0.3.jar"],