diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 29332138..027ef2be 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -1,12 +1,7 @@ - - diff --git a/Android.bp b/Android.bp index db349e26..e5cfe7a0 100644 --- a/Android.bp +++ b/Android.bp @@ -32,6 +32,9 @@ android_app { "com.google.android.material_material", "kotlinx-coroutines-android", "kotlinx-coroutines-core", + // app backup related libs + "seedvault-lib-kotlin-logging-jvm", + "seedvault-lib-chunker", "seedvault-lib-zstd-jni", // our own gradle module libs "seedvault-lib-core", diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8af4a16a..22e6af61 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -157,6 +157,8 @@ dependencies { implementation(libs.google.protobuf.javalite) implementation(libs.google.tink.android) + implementation(libs.kotlin.logging) + implementation(libs.squareup.okio) /** * Storage Dependencies @@ -175,6 +177,7 @@ dependencies { implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.jar")) implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.aar")) + implementation(fileTree("${rootProject.rootDir}/libs").include("seedvault-chunker-0.1.jar")) implementation(fileTree("${rootProject.rootDir}/libs").include("zstd-jni-1.5.6-5.aar")) implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar")) @@ -188,6 +191,7 @@ dependencies { // anything less than 'implementation' fails tests run with gradlew testImplementation(aospLibs) testImplementation("androidx.test.ext:junit:1.1.5") + testImplementation("org.slf4j:slf4j-simple:2.0.3") testImplementation("org.robolectric:robolectric:4.12.2") testImplementation("org.hamcrest:hamcrest:2.2") testImplementation("org.junit.jupiter:junit-jupiter-api:${libs.versions.junit5.get()}") @@ -198,6 +202,7 @@ dependencies { ) testImplementation("app.cash.turbine:turbine:1.0.0") testImplementation("org.bitcoinj:bitcoinj-core:0.16.2") + testImplementation("com.github.luben:zstd-jni:1.5.6-5") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${libs.versions.junit5.get()}") testRuntimeOnly("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}") diff --git a/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt b/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt index a895f91a..f47c7a81 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/restore/RestorableBackup.kt @@ -7,8 +7,13 @@ package com.stevesoltys.seedvault.restore import com.stevesoltys.seedvault.metadata.BackupMetadata import com.stevesoltys.seedvault.metadata.PackageMetadataMap +import com.stevesoltys.seedvault.proto.Snapshot -data class RestorableBackup(val backupMetadata: BackupMetadata) { +data class RestorableBackup( + val backupMetadata: BackupMetadata, + val repoId: String? = null, + val snapshot: Snapshot? = null, +) { val name: String get() = backupMetadata.deviceName diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt new file mode 100644 index 00000000..a6480dd3 --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/SnapshotManager.kt @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport + +import com.github.luben.zstd.ZstdOutputStream +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.header.VERSION +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.restore.Loader +import io.github.oshai.kotlinlogging.KotlinLogging +import okio.Buffer +import okio.buffer +import okio.sink +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.TopLevelFolder + +internal class SnapshotManager( + private val crypto: Crypto, + private val loader: Loader, + private val backendManager: BackendManager, +) { + + private val log = KotlinLogging.logger {} + + /** + * The latest [Snapshot]. May be stale if [loadSnapshots] has not returned + * or wasn't called since new snapshots have been created. + */ + var latestSnapshot: Snapshot? = null + private set + + suspend fun loadSnapshots(callback: (Snapshot) -> Unit) { + log.info { "Loading snapshots..." } + val handles = mutableListOf() + backendManager.backend.list( + topLevelFolder = TopLevelFolder(crypto.repoId), + AppBackupFileType.Snapshot::class, + ) { fileInfo -> + fileInfo.fileHandle as AppBackupFileType.Snapshot + handles.add(fileInfo.fileHandle as AppBackupFileType.Snapshot) + } + handles.forEach { fileHandle -> + // TODO is it a fatal error when one snapshot is corrupted or couldn't get loaded? + val snapshot = onSnapshotFound(fileHandle) + callback(snapshot) + } + } + + private suspend fun onSnapshotFound(snapshotHandle: AppBackupFileType.Snapshot): Snapshot { + // TODO set up local snapshot cache, so we don't need to download those all the time + val snapshot = loader.loadFile(snapshotHandle).use { inputStream -> + Snapshot.parseFrom(inputStream) + } + // update latest snapshot if this one is more recent + if (snapshot.token > (latestSnapshot?.token ?: 0)) latestSnapshot = snapshot + return snapshot + } + + suspend fun saveSnapshot(snapshot: Snapshot) { + val buffer = Buffer() + val bufferStream = buffer.outputStream() + bufferStream.write(VERSION.toInt()) + crypto.newEncryptingStream(bufferStream, crypto.getAdForVersion()).use { cryptoStream -> + ZstdOutputStream(cryptoStream).use { zstdOutputStream -> + snapshot.writeTo(zstdOutputStream) + } + } + val sha256ByteString = buffer.sha256() + val handle = AppBackupFileType.Snapshot(crypto.repoId, sha256ByteString.hex()) + // TODO exception handling + backendManager.backend.save(handle).use { outputStream -> + outputStream.sink().buffer().apply { + writeAll(buffer) + flush() // needs flushing + } + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt new file mode 100644 index 00000000..7949403e --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/AppBackupManager.kt @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.transport.SnapshotManager +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.delay + +internal class AppBackupManager( + private val blobsCache: BlobsCache, + private val snapshotManager: SnapshotManager, + private val snapshotCreatorFactory: SnapshotCreatorFactory, +) { + + private val log = KotlinLogging.logger {} + var snapshotCreator: SnapshotCreator? = null + private set + + suspend fun beforeBackup() { + log.info { "Before backup" } + snapshotCreator = snapshotCreatorFactory.createSnapshotCreator() + blobsCache.populateCache() + } + + suspend fun afterBackupFinished() { + log.info { "After backup finished" } + blobsCache.clear() + val snapshot = snapshotCreator?.finalizeSnapshot() ?: error("Had no snapshotCreator") + keepTrying { + snapshotManager.saveSnapshot(snapshot) + } + snapshotCreator = null + } + + private suspend fun keepTrying(n: Int = 3, block: suspend () -> Unit) { + for (i in 1..n) { + try { + block() + return + } catch (e: Exception) { + if (i == n) throw e + log.error(e) { "Error (#$i), we'll keep trying" } + delay(1000) + } + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt index bc18e0cb..206bee0d 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupModule.kt @@ -5,11 +5,17 @@ package com.stevesoltys.seedvault.transport.backup +import com.stevesoltys.seedvault.transport.SnapshotManager import org.koin.android.ext.koin.androidContext import org.koin.dsl.module val backupModule = module { single { BackupInitializer(get()) } + single { BackupReceiver(get(), get(), get()) } + single { BlobsCache(get(), get(), get()) } + single { BlobCreator(get(), get()) } + single { SnapshotManager(get(), get(), get()) } + single { SnapshotCreatorFactory(androidContext(), get(), get(), get()) } single { InputFactory() } single { PackageService( diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt new file mode 100644 index 00000000..38292fb2 --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupReceiver.kt @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import org.calyxos.seedvault.chunker.Chunk +import org.calyxos.seedvault.chunker.Chunker +import org.calyxos.seedvault.chunker.GearTableCreator +import org.calyxos.seedvault.core.toHexString +import java.io.InputStream + +data class BackupData( + val chunks: List, + val chunkMap: Map, +) + +internal class BackupReceiver( + private val blobsCache: BlobsCache, + private val blobCreator: BlobCreator, + private val crypto: Crypto, + private val replaceableChunker: Chunker? = null, +) { + + private val chunker: Chunker by lazy { + // crypto.gearTableKey is not available at creation time, so use lazy instantiation + replaceableChunker ?: Chunker( + minSize = 1536 * 1024, // 1.5 MB + avgSize = 3 * 1024 * 1024, // 3.0 MB + maxSize = 7680 * 1024, // 7.5 MB + normalization = 1, + gearTable = GearTableCreator.create(crypto.gearTableKey), + hashFunction = { bytes -> + crypto.sha256(bytes).toHexString() + }, + ) + } + private val chunks = mutableListOf() + private val chunkMap = mutableMapOf() + + suspend fun addBytes(bytes: ByteArray) { + chunker.addBytes(bytes).forEach { chunk -> + onNewChunk(chunk) + } + } + + suspend fun readFromStream(inputStream: InputStream) { + try { + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var bytes = inputStream.read(buffer) + while (bytes >= 0) { + if (bytes == buffer.size) { + addBytes(buffer) + } else { + addBytes(buffer.copyOfRange(0, bytes)) + } + bytes = inputStream.read(buffer) + } + } catch (e: Exception) { + finalize() + throw e + } + } + + suspend fun finalize(): BackupData { + chunker.finalize().forEach { chunk -> + onNewChunk(chunk) + } + // copy chunks and chunkMap before clearing + val backupData = BackupData(chunks.toList(), chunkMap.toMap()) + chunks.clear() + chunkMap.clear() + return backupData + } + + private suspend fun onNewChunk(chunk: Chunk) { + chunks.add(chunk.hash) + + val existingBlob = blobsCache.getBlob(chunk.hash) + if (existingBlob == null) { + val blob = blobCreator.createNewBlob(chunk) + chunkMap[chunk.hash] = blob + blobsCache.saveNewBlob(chunk.hash, blob) + } else { + chunkMap[chunk.hash] = existingBlob + } + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt new file mode 100644 index 00000000..1a86158b --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobCreator.kt @@ -0,0 +1,52 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.github.luben.zstd.ZstdOutputStream +import com.google.protobuf.ByteString +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.header.VERSION +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import okio.Buffer +import okio.buffer +import okio.sink +import org.calyxos.seedvault.chunker.Chunk +import org.calyxos.seedvault.core.backends.AppBackupFileType + +internal class BlobCreator( + private val crypto: Crypto, + private val backendManager: BackendManager, +) { + + private val buffer = Buffer() + + suspend fun createNewBlob(chunk: Chunk): Blob { + buffer.clear() + val bufferStream = buffer.outputStream() + bufferStream.write(VERSION.toInt()) + crypto.newEncryptingStream(bufferStream, crypto.getAdForVersion()).use { cryptoStream -> + ZstdOutputStream(cryptoStream).use { zstdOutputStream -> + zstdOutputStream.write(chunk.data) + } + } + val sha256ByteString = buffer.sha256() + val handle = AppBackupFileType.Blob(crypto.repoId, sha256ByteString.hex()) + // TODO exception handling and retries + val size = backendManager.backend.save(handle).use { outputStream -> + val outputBuffer = outputStream.sink().buffer() + val length = outputBuffer.writeAll(buffer) + // flushing is important here, otherwise data doesn't get fully written! + outputBuffer.flush() + length + } + return Blob.newBuilder() + .setId(ByteString.copyFrom(sha256ByteString.asByteBuffer())) + .setLength(size.toInt()) + .setUncompressedLength(chunk.length) + .build() + } +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt new file mode 100644 index 00000000..9d7f715e --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/BlobsCache.kt @@ -0,0 +1,69 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import com.stevesoltys.seedvault.transport.SnapshotManager +import io.github.oshai.kotlinlogging.KotlinLogging +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.TopLevelFolder + +internal class BlobsCache( + private val crypto: Crypto, + private val backendManager: BackendManager, + private val snapshotManager: SnapshotManager, +) { + + private val log = KotlinLogging.logger {} + private val blobMap = mutableMapOf() + + /** + * This must be called before saving files to the backend to avoid uploading duplicate blobs. + */ + suspend fun populateCache() { + log.info { "Getting all blobs from backend..." } + blobMap.clear() + val blobs = mutableSetOf() + backendManager.backend.list( + topLevelFolder = TopLevelFolder(crypto.repoId), + AppBackupFileType.Blob::class, + ) { fileInfo -> + fileInfo.fileHandle as AppBackupFileType.Blob + // TODO we could save size info here and later check it is as expected + blobs.add(fileInfo.fileHandle.name) + } + snapshotManager.loadSnapshots { snapshot -> + snapshot.blobsMap.forEach { (chunkId, blob) -> + // check if referenced blob still exists on backend + if (blobs.contains(blob.id.hexFromProto())) { + // only add blob to our mapping, if it still exists + blobMap.putIfAbsent(chunkId, blob)?.let { previous -> + if (previous.id != blob.id) log.warn { + "Chunk ID ${chunkId.substring(0..5)} had more than one blob" + } + } + } else log.warn { + "Blob ${blob.id.hexFromProto()} referenced in snapshot ${snapshot.token}" + } + } + } + } + + fun getBlob(hash: String): Blob? = blobMap[hash] + + fun saveNewBlob(chunkId: String, blob: Blob) { + blobMap[chunkId] = blob + // TODO persist this blob locally in case backup gets interrupted + } + + fun clear() { + log.info { "Clearing cache..." } + blobMap.clear() + } + +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt new file mode 100644 index 00000000..adbf64eb --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/backup/SnapshotCreator.kt @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageInfo +import android.content.pm.PackageManager.PERMISSION_GRANTED +import android.os.Build +import android.os.UserManager +import android.provider.Settings +import android.provider.Settings.Secure.ANDROID_ID +import com.google.protobuf.ByteString +import com.stevesoltys.seedvault.Clock +import com.stevesoltys.seedvault.metadata.BackupType +import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.proto.Snapshot.Apk +import com.stevesoltys.seedvault.proto.Snapshot.App +import com.stevesoltys.seedvault.proto.Snapshot.Blob +import com.stevesoltys.seedvault.settings.SettingsManager +import org.calyxos.seedvault.core.toHexString + +internal class SnapshotCreatorFactory( + private val context: Context, + private val clock: Clock, + private val packageService: PackageService, + private val settingsManager: SettingsManager, +) { + fun createSnapshotCreator() = SnapshotCreator(context, clock, packageService, settingsManager) +} + +internal class SnapshotCreator( + private val context: Context, + private val clock: Clock, + private val packageService: PackageService, + private val settingsManager: SettingsManager, +) { + + private val snapshotBuilder = Snapshot.newBuilder() + .setToken(clock.time()) + private val appBuilderMap = mutableMapOf() + private val blobsMap = mutableMapOf() + + private val launchableSystemApps by lazy { + packageService.launchableSystemApps.map { it.activityInfo.packageName }.toSet() + } + + fun onApkBackedUp( + packageName: String, + apk: Apk, + chunkMap: Map, + ) { + val appBuilder = appBuilderMap.getOrPut(packageName) { + App.newBuilder() + } + appBuilder.setApk(apk) + blobsMap.putAll(chunkMap) + } + + fun onPackageBackedUp( + packageInfo: PackageInfo, + type: BackupType, + backupData: BackupData, + ) { + val packageName = packageInfo.packageName + val builder = appBuilderMap.getOrPut(packageName) { + App.newBuilder() + } + val isSystemApp = packageInfo.isSystemApp() + val chunkIds = backupData.chunks.forProto() + blobsMap.putAll(backupData.chunkMap) + builder + .setTime(clock.time()) + .setState(APK_AND_DATA.name) + .setType(type.forSnapshot()) + .setName(packageInfo.applicationInfo?.loadLabel(context.packageManager)?.toString()) + .setSystem(isSystemApp) + .setLaunchableSystemApp(isSystemApp && launchableSystemApps.contains(packageName)) + .addAllChunkIds(chunkIds) + } + + fun onIconsBackedUp(backupData: BackupData) { + snapshotBuilder.addAllIconChunkIds(backupData.chunks.forProto()) + blobsMap.putAll(backupData.chunkMap) + } + + fun finalizeSnapshot(): Snapshot { + val userName = getUserName() + val deviceName = if (userName == null) { + "${Build.MANUFACTURER} ${Build.MODEL}" + } else { + "${Build.MANUFACTURER} ${Build.MODEL} - $userName" + } + + @SuppressLint("HardwareIds") + val androidId = Settings.Secure.getString(context.contentResolver, ANDROID_ID) + val snapshot = snapshotBuilder + .setName(deviceName) + .setAndroidId(androidId) + .setSdkInt(Build.VERSION.SDK_INT) + .setAndroidIncremental(Build.VERSION.INCREMENTAL) + .setD2D(settingsManager.d2dBackupsEnabled()) + .putAllApps(appBuilderMap.mapValues { it.value.build() }) + .putAllBlobs(blobsMap) + .build() + appBuilderMap.clear() + snapshotBuilder.clear() + return snapshot + } + + private fun getUserName(): String? { + val perm = "android.permission.QUERY_USERS" + return if (context.checkSelfPermission(perm) == PERMISSION_GRANTED) { + val userManager = context.getSystemService(UserManager::class.java) ?: return null + userManager.userName + } else null + } + + private fun BackupType.forSnapshot(): Snapshot.BackupType = when (this) { + BackupType.KV -> Snapshot.BackupType.KV + BackupType.FULL -> Snapshot.BackupType.FULL + } + +} + +fun Iterable.forProto() = map { ByteString.fromHex(it) } +fun Iterable.hexFromProto() = map { it.toByteArray().toHexString() } +fun ByteString.hexFromProto() = toByteArray().toHexString() diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt new file mode 100644 index 00000000..66a3246e --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/Loader.kt @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.restore + +import com.github.luben.zstd.ZstdInputStream +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.header.UnsupportedVersionException +import com.stevesoltys.seedvault.header.VERSION +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.toHexString +import java.io.ByteArrayInputStream +import java.io.InputStream +import java.io.SequenceInputStream +import java.security.GeneralSecurityException +import java.util.Enumeration + +internal class Loader( + private val crypto: Crypto, + private val backendManager: BackendManager, +) { + + /** + * The responsibility with closing the returned stream lies with the caller. + */ + suspend fun loadFile(handle: AppBackupFileType): InputStream { + // We load the entire ciphertext into memory, + // so we can check the SHA-256 hash before decrypting and parsing the data. + val cipherText = backendManager.backend.load(handle).use { inputStream -> + inputStream.readAllBytes() + } + // check SHA-256 hash first thing + val sha256 = crypto.sha256(cipherText).toHexString() + val expectedHash = when (handle) { + is AppBackupFileType.Snapshot -> handle.hash + is AppBackupFileType.Blob -> handle.name + } + if (sha256 != expectedHash) { + throw GeneralSecurityException("File had wrong SHA-256 hash: $handle") + } + // check that we can handle the version of that snapshot + val version = cipherText[0] + if (version <= 1) throw GeneralSecurityException("Unexpected version: $version") + if (version > VERSION) throw UnsupportedVersionException(version) + // get associated data for version, used for authenticated decryption + val ad = crypto.getAdForVersion(version) + // skip first version byte when creating cipherText stream + val inputStream = ByteArrayInputStream(cipherText, 1, cipherText.size - 1) + // decrypt and decompress cipherText stream and parse snapshot + return ZstdInputStream(crypto.newDecryptingStream(inputStream, ad)) + } + + suspend fun loadFiles(handles: List): InputStream { + val enumeration: Enumeration = object : Enumeration { + val iterator = handles.iterator() + + override fun hasMoreElements(): Boolean { + return iterator.hasNext() + } + + override fun nextElement(): InputStream { + return runBlocking { loadFile(iterator.next()) } + } + } + return SequenceInputStream(enumeration) + } +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt index 869e1b0c..fd1d835d 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/transport/restore/RestoreModule.kt @@ -10,6 +10,7 @@ import org.koin.dsl.module val restoreModule = module { single { OutputFactory() } + single { Loader(get(), get()) } single { KVRestore(get(), get(), get(), get(), get(), get()) } single { FullRestore(get(), get(), get(), get(), get()) } single { diff --git a/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt b/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt index dea0b941..a41d1155 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/worker/WorkerModule.kt @@ -5,6 +5,7 @@ package com.stevesoltys.seedvault.worker +import com.stevesoltys.seedvault.transport.backup.AppBackupManager import org.koin.android.ext.koin.androidContext import org.koin.dsl.module @@ -23,6 +24,7 @@ val workerModule = module { crypto = get(), ) } + single { AppBackupManager(get(), get(), get()) } single { ApkBackup( pm = androidContext().packageManager, diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt new file mode 100644 index 00000000..e1a1652b --- /dev/null +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/SnapshotManagerTest.kt @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport + +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.crypto.Crypto +import com.stevesoltys.seedvault.getRandomString +import com.stevesoltys.seedvault.proto.Snapshot +import com.stevesoltys.seedvault.transport.restore.Loader +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.FileInfo +import org.calyxos.seedvault.core.backends.TopLevelFolder +import org.calyxos.seedvault.core.toByteArrayFromHex +import org.calyxos.seedvault.core.toHexString +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.io.OutputStream +import java.security.MessageDigest +import kotlin.random.Random + +class SnapshotManagerTest { + + private val crypto: Crypto = mockk() + private val backendManager: BackendManager = mockk() + private val backend: Backend = mockk() + + private val loader = Loader(crypto, backendManager) // need a real loader + private val snapshotManager = SnapshotManager(crypto, loader, backendManager) + + private val ad = Random.nextBytes(1) + private val passThroughOutputStream = slot() + private val passThroughInputStream = slot() + private val repoId = Random.nextBytes(32).toHexString() + private val snapshotHandle = slot() + private val snapshot = Snapshot.newBuilder() + .setToken(Random.nextLong()) + .setName(getRandomString()) + .setSdkInt(Random.nextInt()) + .putAllBlobs(mapOf(getRandomString() to Snapshot.Blob.getDefaultInstance())) + .build() + + @Test + fun `test saving and loading`() = runBlocking { + val outputStream = ByteArrayOutputStream() + + every { crypto.getAdForVersion() } returns ad + every { crypto.newEncryptingStream(capture(passThroughOutputStream), ad) } answers { + passThroughOutputStream.captured // not really encrypting here + } + every { crypto.repoId } returns repoId + every { backendManager.backend } returns backend + coEvery { backend.save(capture(snapshotHandle)) } returns outputStream + + snapshotManager.saveSnapshot(snapshot) + + // check that file content hash matches snapshot hash + val messageDigest = MessageDigest.getInstance("SHA-256") + assertEquals( + messageDigest.digest(outputStream.toByteArray()).toHexString(), + snapshotHandle.captured.hash, + ) + + val fileInfo = FileInfo(snapshotHandle.captured, Random.nextLong()) + assertTrue(outputStream.size() > 0) + val inputStream = ByteArrayInputStream(outputStream.toByteArray()) + coEvery { + backend.list( + topLevelFolder = TopLevelFolder(repoId), + AppBackupFileType.Snapshot::class, + callback = captureLambda<(FileInfo) -> Unit>() + ) + } answers { + lambda<(FileInfo) -> Unit>().captured.invoke(fileInfo) + } + coEvery { backend.load(snapshotHandle.captured) } returns inputStream + every { + crypto.sha256(outputStream.toByteArray()) + } returns snapshotHandle.captured.hash.toByteArrayFromHex() + every { crypto.newDecryptingStream(capture(passThroughInputStream), ad) } answers { + passThroughInputStream.captured + } + + var loadedSnapshot: Snapshot? = null + snapshotManager.loadSnapshots { loadedSnapshot = it } + assertEquals(snapshot, loadedSnapshot) + } +} diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCreatorTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCreatorTest.kt new file mode 100644 index 00000000..3ec7fa8d --- /dev/null +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/backup/BlobCreatorTest.kt @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.transport.backup + +import com.stevesoltys.seedvault.backend.BackendManager +import com.stevesoltys.seedvault.transport.TransportTest +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.chunker.Chunk +import org.calyxos.seedvault.core.backends.AppBackupFileType +import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.toHexString +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.io.ByteArrayOutputStream +import java.io.OutputStream +import java.security.MessageDigest +import kotlin.random.Random + +internal class BlobCreatorTest : TransportTest() { + + private val backendManager: BackendManager = mockk() + private val backend: Backend = mockk() + private val blobCreator = BlobCreator(crypto, backendManager) + + private val ad = Random.nextBytes(1) + private val passThroughOutputStream = slot() + private val blobHandle = slot() + + @Test + fun `test re-use for hashing two chunks`() = runBlocking { + val data1 = Random.nextBytes(1337) + val data2 = Random.nextBytes(2342) + val chunk1 = Chunk(0L, data1.size, data1, "doesn't matter here") + val chunk2 = Chunk(0L, data2.size, data2, "doesn't matter here") + val outputStream1 = ByteArrayOutputStream() + val outputStream2 = ByteArrayOutputStream() + + every { crypto.getAdForVersion() } returns ad + every { crypto.newEncryptingStream(capture(passThroughOutputStream), ad) } answers { + passThroughOutputStream.captured // not really encrypting here + } + every { crypto.repoId } returns repoId + every { backendManager.backend } returns backend + coEvery { backend.save(capture(blobHandle)) } returns outputStream1 + + blobCreator.createNewBlob(chunk1) + // check that file content hash matches snapshot hash + val messageDigest = MessageDigest.getInstance("SHA-256") + assertEquals( + messageDigest.digest(outputStream1.toByteArray()).toHexString(), + blobHandle.captured.name, + ) + + // use same BlobCreator to create another blob, because we re-use a single buffer + // and need to check clearing that does work as expected + coEvery { backend.save(capture(blobHandle)) } returns outputStream2 + blobCreator.createNewBlob(chunk2) + // check that file content hash matches snapshot hash + assertEquals( + messageDigest.digest(outputStream2.toByteArray()).toHexString(), + blobHandle.captured.name, + ) + } +} diff --git a/libs/Android.bp b/libs/Android.bp index 36158313..ae63f75a 100644 --- a/libs/Android.bp +++ b/libs/Android.bp @@ -16,6 +16,12 @@ java_import { sdk_version: "current", } +java_import { + name: "seedvault-lib-chunker", + jars: ["seedvault-chunker-0.1.jar"], + sdk_version: "current", +} + java_import { name: "seedvault-lib-kotlin-logging-jvm", jars: ["kotlin-logging-jvm-6.0.3.jar"], diff --git a/libs/seedvault-chunker-0.1.jar b/libs/seedvault-chunker-0.1.jar new file mode 100644 index 00000000..a452dfe5 Binary files /dev/null and b/libs/seedvault-chunker-0.1.jar differ