diff --git a/app/build.gradle.kts b/app/build.gradle.kts index d2e6a243..bea343c3 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -106,19 +106,7 @@ android { } dependencies { - - val aospLibs = fileTree("$projectDir/libs") { - // For more information about this module: - // https://android.googlesource.com/platform/frameworks/base/+/refs/tags/android-11.0.0_r3/Android.bp#507 - // framework_intermediates/classes-header.jar works for gradle build as well, - // but not unit tests, so we use the actual classes (without updatable modules). - // - // out/target/common/obj/JAVA_LIBRARIES/framework-minus-apex_intermediates/classes.jar - include("android.jar") - // out/target/common/obj/JAVA_LIBRARIES/core-libart.com.android.art_intermediates/classes.jar - include("libcore.jar") - } - + val aospLibs: FileTree by rootProject.extra compileOnly(aospLibs) /** @@ -149,6 +137,7 @@ dependencies { /** * Storage Dependencies */ + implementation(project(":core")) implementation(project(":storage:lib")) /** @@ -188,6 +177,7 @@ dependencies { testRuntimeOnly("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}") androidTestImplementation(aospLibs) + androidTestImplementation(kotlin("test")) androidTestImplementation("androidx.test:runner:1.4.0") androidTestImplementation("androidx.test:rules:1.4.0") androidTestImplementation("androidx.test.ext:junit:1.1.3") diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/plugins/saf/SafBackendTest.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/plugins/saf/SafBackendTest.kt new file mode 100644 index 00000000..8442332a --- /dev/null +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/plugins/saf/SafBackendTest.kt @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.plugins.saf + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.filters.MediumTest +import androidx.test.platform.app.InstrumentationRegistry +import com.stevesoltys.seedvault.settings.SettingsManager +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.BackendTest +import org.calyxos.seedvault.core.backends.saf.SafBackend +import org.calyxos.seedvault.core.backends.saf.SafConfig +import org.junit.Test +import org.junit.runner.RunWith +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject + +@RunWith(AndroidJUnit4::class) +@MediumTest +class SafBackendTest : BackendTest(), KoinComponent { + + private val context = InstrumentationRegistry.getInstrumentation().targetContext + private val settingsManager by inject() + override val plugin: Backend + get() { + val safStorage = settingsManager.getSafStorage() ?: error("No SAF storage") + val safConfig = SafConfig( + config = safStorage.config, + name = safStorage.name, + isUsb = safStorage.isUsb, + requiresNetwork = safStorage.requiresNetwork, + rootId = safStorage.rootId, + ) + return SafBackend(context, safConfig, ".SeedvaultTest") + } + + @Test + fun test(): Unit = runBlocking { + testWriteListReadRenameDelete() + } +} diff --git a/build.gradle.kts b/build.gradle.kts index ae47a822..dda10d46 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -13,6 +13,21 @@ plugins { alias(libs.plugins.jetbrains.kotlin.android) apply false alias(libs.plugins.jetbrains.dokka) apply false alias(libs.plugins.jlleitschuh.ktlint) apply false + alias(libs.plugins.jetbrains.kotlin.jvm) apply false +} + +val aospLibs by extra { + fileTree("$rootDir/app/libs") { // TODO move libs to root + // For more information about this module: + // https://android.googlesource.com/platform/frameworks/base/+/refs/tags/android-11.0.0_r3/Android.bp#507 + // framework_intermediates/classes-header.jar works for gradle build as well, + // but not unit tests, so we use the actual classes (without updatable modules). + // + // out/target/common/obj/JAVA_LIBRARIES/framework-minus-apex_intermediates/classes.jar + include("android.jar") + // out/target/common/obj/JAVA_LIBRARIES/core-libart.com.android.art_intermediates/classes.jar + include("libcore.jar") + } } subprojects { diff --git a/core/.gitignore b/core/.gitignore new file mode 100644 index 00000000..42afabfd --- /dev/null +++ b/core/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/core/build.gradle.kts b/core/build.gradle.kts new file mode 100644 index 00000000..f4993325 --- /dev/null +++ b/core/build.gradle.kts @@ -0,0 +1,51 @@ +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.jetbrains.kotlin.android) +} + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +android { + namespace = "org.calyxos.seedvault.core" + compileSdk = libs.versions.compileSdk.get().toInt() + + defaultConfig { + minSdk = libs.versions.minSdk.get().toInt() + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunnerArguments["disableAnalytics"] = "true" + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + kotlinOptions { + jvmTarget = JavaVersion.VERSION_17.toString() + languageVersion = "1.8" + freeCompilerArgs += listOf( + "-opt-in=kotlin.RequiresOptIn", + "-Xexplicit-api=strict" + ) + } +} + +dependencies { + val aospLibs: FileTree by rootProject.extra + compileOnly(aospLibs) + compileOnly("org.ogce:xpp3:1.1.6") + compileOnly(kotlin("test")) + implementation(libs.bundles.kotlin) + implementation(libs.bundles.coroutines) + implementation(libs.androidx.documentfile) + // implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("okio-jvm-3.7.0.jar")) + implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("*.jar")) + implementation("io.github.oshai:kotlin-logging-jvm:6.0.3") + implementation("org.slf4j:slf4j-simple:2.0.3") + + testImplementation(kotlin("test")) + testImplementation("org.ogce:xpp3:1.1.6") +} diff --git a/core/src/main/AndroidManifest.xml b/core/src/main/AndroidManifest.xml new file mode 100644 index 00000000..6c244e77 --- /dev/null +++ b/core/src/main/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + diff --git a/core/src/main/java/org/calyxos/seedvault/core/ByteArrayUtils.kt b/core/src/main/java/org/calyxos/seedvault/core/ByteArrayUtils.kt new file mode 100644 index 00000000..bc8d65c6 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/ByteArrayUtils.kt @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: 2021 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core + +internal fun ByteArray.toHexString() = joinToString("") { "%02x".format(it) } + +internal fun String.toByteArrayFromHex() = chunked(2).map { it.toInt(16).toByte() }.toByteArray() diff --git a/core/src/main/java/org/calyxos/seedvault/core/Utils.kt b/core/src/main/java/org/calyxos/seedvault/core/Utils.kt new file mode 100644 index 00000000..28982dfa --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/Utils.kt @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core + +import android.Manifest.permission.INTERACT_ACROSS_USERS_FULL +import android.content.Context +import android.content.pm.PackageManager.PERMISSION_GRANTED +import android.os.UserManager + +/** + * Hack to allow other profiles access to USB backend. + * @return the context of the device's main user, so use with great care! + */ +@Suppress("MissingPermission") +public fun Context.getBackendContext(isUsbStorage: () -> Boolean): Context { + if (checkSelfPermission(INTERACT_ACROSS_USERS_FULL) == PERMISSION_GRANTED && isUsbStorage()) { + UserManager.get(this).getProfileParent(user) + ?.let { parent -> return createContextAsUser(parent, 0) } + } + return this +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/Backend.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/Backend.kt new file mode 100644 index 00000000..cd275b69 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/Backend.kt @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends + +import okio.BufferedSink +import okio.BufferedSource +import kotlin.reflect.KClass + +public interface Backend { + + public suspend fun save(handle: FileHandle): BufferedSink + + public suspend fun load(handle: FileHandle): BufferedSource + + public suspend fun list( + topLevelFolder: TopLevelFolder?, + vararg fileTypes: KClass, + callback: (FileInfo) -> Unit, + ) + + public suspend fun remove(handle: FileHandle) + + public suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) + + // TODO really all? + public suspend fun removeAll() + +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/BackendProperties.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/BackendProperties.kt new file mode 100644 index 00000000..f4d4acc6 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/BackendProperties.kt @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends + +import android.annotation.WorkerThread +import android.content.Context +import android.net.ConnectivityManager +import android.net.NetworkCapabilities.NET_CAPABILITY_INTERNET +import at.bitfire.dav4jvm.exception.HttpException +import java.io.IOException + +public abstract class BackendProperties { + public abstract val config: T + public abstract val name: String + public abstract val isUsb: Boolean + public abstract val requiresNetwork: Boolean + + @WorkerThread + public abstract fun isUnavailableUsb(context: Context): Boolean + + /** + * Returns true if this is storage that requires network access, + * but it isn't available right now. + */ + public fun isUnavailableNetwork(context: Context, allowMetered: Boolean): Boolean { + return requiresNetwork && !hasUnmeteredInternet(context, allowMetered) + } + + private fun hasUnmeteredInternet(context: Context, allowMetered: Boolean): Boolean { + val cm = context.getSystemService(ConnectivityManager::class.java) ?: return false + val isMetered = cm.isActiveNetworkMetered + val capabilities = cm.getNetworkCapabilities(cm.activeNetwork) ?: return false + return capabilities.hasCapability(NET_CAPABILITY_INTERNET) && (allowMetered || !isMetered) + } +} + +public fun Exception.isOutOfSpace(): Boolean { + return when (this) { + is IOException -> message?.contains("No space left on device") == true || + (cause as? HttpException)?.code == 507 + + is HttpException -> code == 507 + + else -> false + } +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/BackendTest.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/BackendTest.kt new file mode 100644 index 00000000..f925b3d8 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/BackendTest.kt @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends + +import androidx.annotation.VisibleForTesting +import at.bitfire.dav4jvm.exception.HttpException +import org.calyxos.seedvault.core.toHexString +import org.junit.Assert.assertArrayEquals +import kotlin.random.Random +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.fail + +@VisibleForTesting +public abstract class BackendTest { + + public abstract val plugin: Backend + + protected suspend fun testWriteListReadRenameDelete() { + try { + plugin.removeAll() + } catch (e: HttpException) { + if (e.code != 404) fail(e.message, e) + } + + val now = System.currentTimeMillis() + val bytes1 = Random.nextBytes(1337) + val bytes2 = Random.nextBytes(1337 * 8) + plugin.save(LegacyAppBackupFile.Metadata(now)).use { + it.write(bytes1) + } + + plugin.save(FileBackupFileType.Snapshot("0123456789abcdef", now)).use { + it.write(bytes2) + } + + var metadata: LegacyAppBackupFile.Metadata? = null + var snapshot: FileBackupFileType.Snapshot? = null + plugin.list( + null, + FileBackupFileType.Snapshot::class, + FileBackupFileType.Blob::class, + LegacyAppBackupFile.Metadata::class, + ) { fileInfo -> + val handle = fileInfo.fileHandle + if (handle is LegacyAppBackupFile.Metadata && handle.token == now) { + metadata = handle + } else if (handle is FileBackupFileType.Snapshot && handle.time == now) { + snapshot = handle + } + } + assertNotNull(metadata) + assertNotNull(snapshot) + + assertArrayEquals(bytes1, plugin.load(metadata as FileHandle).readByteArray()) + assertArrayEquals(bytes2, plugin.load(snapshot as FileHandle).readByteArray()) + + val blobName = Random.nextBytes(32).toHexString() + var blob: FileBackupFileType.Blob? = null + val bytes3 = Random.nextBytes(1337 * 16) + plugin.save(FileBackupFileType.Blob("0123456789abcdef", blobName)).use { + it.write(bytes3) + } + plugin.list( + null, + FileBackupFileType.Snapshot::class, + FileBackupFileType.Blob::class, + LegacyAppBackupFile.Metadata::class, + ) { fileInfo -> + val handle = fileInfo.fileHandle + if (handle is FileBackupFileType.Blob && handle.name == blobName) { + blob = handle + } + } + assertNotNull(blob) + assertArrayEquals(bytes3, plugin.load(blob as FileHandle).readByteArray()) + + // try listing with top-level folder, should find two files of FileBackupFileType in there + var numFiles = 0 + plugin.list( + snapshot!!.topLevelFolder, + FileBackupFileType.Snapshot::class, + FileBackupFileType.Blob::class, + LegacyAppBackupFile.Metadata::class, + ) { numFiles++ } + assertEquals(2, numFiles) + + plugin.remove(snapshot as FileHandle) + + // rename snapshots + val snapshotNewFolder = TopLevelFolder("0123456789abcdee.sv") + plugin.rename(snapshot!!.topLevelFolder, snapshotNewFolder) + + // rename to existing folder should fail + val e = assertFailsWith { + plugin.rename(snapshotNewFolder, metadata!!.topLevelFolder) + } + println(e) + + plugin.remove(metadata!!.topLevelFolder) + plugin.remove(snapshotNewFolder) + } + +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/Constants.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/Constants.kt new file mode 100644 index 00000000..d006b0da --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/Constants.kt @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends + +public object Constants { + + public const val DIRECTORY_ROOT: String = ".SeedVaultAndroidBackup" + internal const val FILE_BACKUP_METADATA = ".backup.metadata" + internal const val FILE_BACKUP_ICONS = ".backup.icons" + public val tokenRegex: Regex = Regex("([0-9]{13})") // good until the year 2286 + public const val SNAPSHOT_EXT: String = ".SeedSnap" + public val folderRegex: Regex = Regex("^[a-f0-9]{16}\\.sv$") + public val chunkFolderRegex: Regex = Regex("[a-f0-9]{2}") + public val chunkRegex: Regex = Regex("[a-f0-9]{64}") + public val snapshotRegex: Regex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286 + public const val MIME_TYPE: String = "application/octet-stream" + public const val CHUNK_FOLDER_COUNT: Int = 256 + +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/FileHandle.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/FileHandle.kt new file mode 100644 index 00000000..88e1321d --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/FileHandle.kt @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends + +import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_ICONS +import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA +import org.calyxos.seedvault.core.backends.Constants.SNAPSHOT_EXT + +public sealed class FileHandle { + public abstract val name: String + + /** + * The relative path relative to the storage root without prepended or trailing slash (/). + */ + public abstract val relativePath: String +} + +public data class TopLevelFolder(override val name: String) : FileHandle() { + override val relativePath: String = name +} + +public sealed class LegacyAppBackupFile : FileHandle() { + public abstract val token: Long + public val topLevelFolder: TopLevelFolder get() = TopLevelFolder(token.toString()) + override val relativePath: String get() = "$token/$name" + + public data class Metadata(override val token: Long) : LegacyAppBackupFile() { + override val name: String = FILE_BACKUP_METADATA + } + + public data class IconsFile(override val token: Long) : LegacyAppBackupFile() { + override val name: String = FILE_BACKUP_ICONS + } + + public data class Blob( + override val token: Long, + override val name: String, + ) : LegacyAppBackupFile() +} + +public sealed class FileBackupFileType : FileHandle() { + public abstract val androidId: String + public val topLevelFolder: TopLevelFolder get() = TopLevelFolder("$androidId.sv") + + public data class Blob( + override val androidId: String, + override val name: String, + ) : FileBackupFileType() { + override val relativePath: String get() = "$androidId.sv/${name.substring(0, 2)}/$name" + } + + public data class Snapshot( + override val androidId: String, + val time: Long, + ) : FileBackupFileType() { + override val name: String = "$time$SNAPSHOT_EXT" + override val relativePath: String get() = "$androidId.sv/$name" + } +} + +public data class FileInfo( + val fileHandle: FileHandle, + val size: Long, +) diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/saf/DocumentFileCache.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/DocumentFileCache.kt new file mode 100644 index 00000000..123bc07a --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/DocumentFileCache.kt @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.saf + +import android.content.Context +import androidx.documentfile.provider.DocumentFile +import org.calyxos.seedvault.core.backends.FileBackupFileType +import org.calyxos.seedvault.core.backends.FileHandle +import org.calyxos.seedvault.core.backends.LegacyAppBackupFile +import org.calyxos.seedvault.core.backends.TopLevelFolder + +internal class DocumentFileCache( + private val context: Context, + private val baseFile: DocumentFile, + private val root: String, +) { + + private val cache = mutableMapOf() + + internal suspend fun getRootFile(): DocumentFile { + return cache.getOrPut(root) { + baseFile.getOrCreateDirectory(context, root) + } + } + + internal suspend fun getFile(fh: FileHandle): DocumentFile = when (fh) { + is TopLevelFolder -> cache.getOrPut("$root/${fh.relativePath}") { + getRootFile().getOrCreateDirectory(context, fh.name) + } + + is LegacyAppBackupFile -> cache.getOrPut("$root/${fh.relativePath}") { + getFile(fh.topLevelFolder).getOrCreateFile(context, fh.name) + } + + is FileBackupFileType.Blob -> { + val subFolderName = fh.name.substring(0, 2) + cache.getOrPut("$root/${fh.topLevelFolder.name}/$subFolderName") { + getFile(fh.topLevelFolder).getOrCreateDirectory(context, subFolderName) + }.getOrCreateFile(context, fh.name) + } + + is FileBackupFileType.Snapshot -> { + getFile(fh.topLevelFolder).getOrCreateFile(context, fh.name) + } + } +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafBackend.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafBackend.kt new file mode 100644 index 00000000..e1a70632 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafBackend.kt @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.saf + +import android.content.Context +import android.provider.DocumentsContract.renameDocument +import androidx.documentfile.provider.DocumentFile +import io.github.oshai.kotlinlogging.KotlinLogging +import okio.BufferedSink +import okio.BufferedSource +import okio.buffer +import okio.sink +import okio.source +import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.Constants.DIRECTORY_ROOT +import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA +import org.calyxos.seedvault.core.backends.Constants.chunkFolderRegex +import org.calyxos.seedvault.core.backends.Constants.chunkRegex +import org.calyxos.seedvault.core.backends.Constants.folderRegex +import org.calyxos.seedvault.core.backends.Constants.snapshotRegex +import org.calyxos.seedvault.core.backends.Constants.tokenRegex +import org.calyxos.seedvault.core.backends.FileBackupFileType +import org.calyxos.seedvault.core.backends.FileHandle +import org.calyxos.seedvault.core.backends.FileInfo +import org.calyxos.seedvault.core.backends.LegacyAppBackupFile +import org.calyxos.seedvault.core.backends.TopLevelFolder +import org.calyxos.seedvault.core.getBackendContext +import java.io.IOException +import kotlin.reflect.KClass + +public class SafBackend( + private val appContext: Context, + private val safConfig: SafConfig, + root: String = DIRECTORY_ROOT, +) : Backend { + + private val log = KotlinLogging.logger {} + + /** + * Attention: This context might be from a different user. Use with care. + */ + private val context: Context get() = appContext.getBackendContext { safConfig.isUsb } + private val cache = DocumentFileCache(context, safConfig.getDocumentFile(context), root) + + override suspend fun save(handle: FileHandle): BufferedSink { + val file = cache.getFile(handle) + return file.getOutputStream(context.contentResolver).sink().buffer() + } + + override suspend fun load(handle: FileHandle): BufferedSource { + val file = cache.getFile(handle) + return file.getInputStream(context.contentResolver).source().buffer() + } + + override suspend fun list( + topLevelFolder: TopLevelFolder?, + vararg fileTypes: KClass, + callback: (FileInfo) -> Unit, + ) { + if (TopLevelFolder::class in fileTypes) throw UnsupportedOperationException() + if (LegacyAppBackupFile::class in fileTypes) throw UnsupportedOperationException() + if (LegacyAppBackupFile.IconsFile::class in fileTypes) throw UnsupportedOperationException() + if (LegacyAppBackupFile.Blob::class in fileTypes) throw UnsupportedOperationException() + + val folder = if (topLevelFolder == null) { + cache.getRootFile() + } else { + cache.getFile(topLevelFolder) + } + // limit depth based on wanted types and if top-level folder is given + var depth = if (FileBackupFileType.Blob::class in fileTypes) 3 else 2 + if (topLevelFolder != null) depth -= 1 + + folder.listFilesRecursive(depth) { file -> + if (!file.isFile) return@listFilesRecursive + val parentName = file.parentFile?.name ?: return@listFilesRecursive + val name = file.name ?: return@listFilesRecursive + if (LegacyAppBackupFile.Metadata::class in fileTypes && name == FILE_BACKUP_METADATA && + parentName.matches(tokenRegex) + ) { + val metadata = LegacyAppBackupFile.Metadata(parentName.toLong()) + callback(FileInfo(metadata, file.length())) + } + if (FileBackupFileType.Snapshot::class in fileTypes || + FileBackupFileType::class in fileTypes + ) { + val match = snapshotRegex.matchEntire(name) + if (match != null) { + val snapshot = FileBackupFileType.Snapshot( + androidId = parentName.substringBefore('.'), + time = match.groupValues[1].toLong(), + ) + callback(FileInfo(snapshot, file.length())) + } + } + if ((FileBackupFileType.Blob::class in fileTypes || + FileBackupFileType::class in fileTypes) + ) { + val androidIdSv = file.parentFile?.parentFile?.name ?: "" + if (folderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) { + if (chunkRegex.matches(name)) { + val blob = FileBackupFileType.Blob( + androidId = androidIdSv.substringBefore('.'), + name = name, + ) + callback(FileInfo(blob, file.length())) + } + } + } + } + } + + private suspend fun DocumentFile.listFilesRecursive( + depth: Int, + callback: (DocumentFile) -> Unit, + ) { + if (depth <= 0) return + listFilesBlocking(context).forEach { file -> + callback(file) + if (file.isDirectory) file.listFilesRecursive(depth - 1, callback) + } + } + + override suspend fun remove(handle: FileHandle) { + val file = cache.getFile(handle) + if (!file.delete()) throw IOException("could not delete ${handle.relativePath}") + } + + override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) { + val fromFile = cache.getFile(from) + // don't use fromFile.renameTo(to.name) as that creates "${to.name} (1)" + val newUri = renameDocument(context.contentResolver, fromFile.uri, to.name) + ?: throw IOException("could not rename ${from.relativePath}") + val toFile = DocumentFile.fromTreeUri(context, newUri) + ?: throw IOException("renamed URI invalid: $newUri") + if (toFile.name != to.name) { + toFile.delete() + throw IOException("renamed to ${toFile.name}, but expected ${to.name}") + } + } + + override suspend fun removeAll() { + cache.getRootFile().listFilesBlocking(context).forEach { + it.delete() + } + } + +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafConfig.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafConfig.kt new file mode 100644 index 00000000..8d0d924a --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafConfig.kt @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.saf + +import android.content.Context +import android.net.Uri +import android.provider.DocumentsContract.Root.COLUMN_ROOT_ID +import androidx.annotation.WorkerThread +import androidx.documentfile.provider.DocumentFile +import org.calyxos.seedvault.core.backends.BackendProperties + +public data class SafConfig( + override val config: Uri, + override val name: String, + override val isUsb: Boolean, + override val requiresNetwork: Boolean, + /** + * The [COLUMN_ROOT_ID] for the [uri]. + * This is only nullable for historic reasons, because we didn't always store it. + */ + val rootId: String?, +) : BackendProperties() { + + internal val uri: Uri = config + + public fun getDocumentFile(context: Context): DocumentFile = + DocumentFile.fromTreeUri(context, config) + ?: throw AssertionError("Should only happen on API < 21.") + + /** + * Returns true if this is USB storage that is not available, false otherwise. + * + * Must be run off UI thread (ideally I/O). + */ + @WorkerThread + override fun isUnavailableUsb(context: Context): Boolean { + return isUsb && !getDocumentFile(context).isDirectory + } +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafHelper.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafHelper.kt new file mode 100644 index 00000000..46682e45 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafHelper.kt @@ -0,0 +1,217 @@ +/* + * SPDX-FileCopyrightText: 2021 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.saf + +import android.annotation.SuppressLint +import android.content.ContentResolver +import android.content.Context +import android.database.ContentObserver +import android.database.Cursor +import android.net.Uri +import android.provider.DocumentsContract.Document.COLUMN_DOCUMENT_ID +import android.provider.DocumentsContract.EXTRA_LOADING +import android.provider.DocumentsContract.buildChildDocumentsUriUsingTree +import android.provider.DocumentsContract.buildDocumentUriUsingTree +import android.provider.DocumentsContract.getDocumentId +import android.util.Log +import androidx.annotation.VisibleForTesting +import androidx.documentfile.provider.DocumentFile +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeout +import org.calyxos.seedvault.core.backends.Constants.MIME_TYPE +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import kotlin.coroutines.resume + +private const val TAG = "SafHelper" + +@Throws(IOException::class) +public fun DocumentFile.getInputStream(contentResolver: ContentResolver): InputStream { + return uri.openInputStream(contentResolver) +} + +@Throws(IOException::class) +public fun DocumentFile.getOutputStream(contentResolver: ContentResolver): OutputStream { + return uri.openOutputStream(contentResolver) +} + +/** + * Checks if a file exists and if not, creates it. + * + * If we were trying to create it right away, some providers create "filename (1)". + */ +@Throws(IOException::class) +internal suspend fun DocumentFile.getOrCreateFile(context: Context, name: String): DocumentFile { + return try { + findFileBlocking(context, name) ?: createFileOrThrow(name, MIME_TYPE) + } catch (e: Exception) { + // SAF can throw all sorts of exceptions, so wrap it in IOException. + // E.g. IllegalArgumentException can be thrown by FileSystemProvider#isChildDocument() + // when flash drive is not plugged-in: + // http://aosp.opersys.com/xref/android-11.0.0_r8/xref/frameworks/base/core/java/com/android/internal/content/FileSystemProvider.java#135 + if (e is IOException) throw e + else throw IOException(e) + } +} + +@Throws(IOException::class) +internal fun DocumentFile.createFileOrThrow( + name: String, + mimeType: String = MIME_TYPE, +): DocumentFile { + val file = createFile(mimeType, name) ?: throw IOException("Unable to create file: $name") + if (file.name != name) { + file.delete() + if (file.name == null) { // this happens when file existed already + // try to find the original file we were looking for + val foundFile = findFile(name) + if (foundFile?.name == name) return foundFile + } + throw IOException("Wanted to create $name, but got ${file.name}") + } + return file +} + +/** + * Checks if a directory already exists and if not, creates it. + */ +@Throws(IOException::class) +public suspend fun DocumentFile.getOrCreateDirectory(context: Context, name: String): DocumentFile { + return findFileBlocking(context, name) ?: createDirectoryOrThrow(name) +} + +@Throws(IOException::class) +public fun DocumentFile.createDirectoryOrThrow(name: String): DocumentFile { + val directory = createDirectory(name) + ?: throw IOException("Unable to create directory: $name") + if (directory.name != name) { + directory.delete() + throw IOException("Wanted to directory $name, but got ${directory.name}") + } + return directory +} + +/** + * Works like [DocumentFile.listFiles] except + * that it waits until the DocumentProvider has a result. + * This prevents getting an empty list even though there are children to be listed. + */ +@Throws(IOException::class) +public suspend fun DocumentFile.listFilesBlocking(context: Context): List { + val resolver = context.contentResolver + val childrenUri = buildChildDocumentsUriUsingTree(uri, getDocumentId(uri)) + val projection = arrayOf(COLUMN_DOCUMENT_ID) + val result = ArrayList() + + try { + getLoadedCursor { + resolver.query(childrenUri, projection, null, null, null) + } + } catch (e: TimeoutCancellationException) { + throw IOException(e) + }.use { cursor -> + while (cursor.moveToNext()) { + val documentId = cursor.getString(0) + val documentUri = buildDocumentUriUsingTree(uri, documentId) + result.add(getTreeDocumentFile(this, context, documentUri)) + } + } + return result +} + +/** + * An extremely dirty reflection hack to instantiate a TreeDocumentFile with a parent. + * + * All other public ways to get a TreeDocumentFile only work from [Uri]s + * (e.g. [DocumentFile.fromTreeUri]) and always set parent to null. + * + * We have a test for this method to ensure CI will alert us when this reflection breaks. + * Also, [DocumentFile] is part of AndroidX, so we control the dependency and notice when it fails. + */ +@VisibleForTesting +@SuppressLint("CheckedExceptions") +public fun getTreeDocumentFile( + parent: DocumentFile, + context: Context, + uri: Uri, +): DocumentFile { + @SuppressWarnings("MagicNumber") + val constructor = parent.javaClass.declaredConstructors.find { + it.name == "androidx.documentfile.provider.TreeDocumentFile" && it.parameterCount == 3 + } + check(constructor != null) { "Could not find constructor for TreeDocumentFile" } + constructor.isAccessible = true + return constructor.newInstance(parent, context, uri) as DocumentFile +} + +/** + * Same as [DocumentFile.findFile] only that it re-queries when the first result was stale. + * + * Most documents providers including Nextcloud are listing the full directory content + * when querying for a specific file in a directory, + * so there is no point in trying to optimize the query by not listing all children. + */ +public suspend fun DocumentFile.findFileBlocking( + context: Context, + displayName: String, +): DocumentFile? { + val files = try { + listFilesBlocking(context) + } catch (e: IOException) { + Log.e(TAG, "Error finding file blocking", e) + return null + } + for (doc in files) { + if (displayName == doc.name) return doc + } + return null +} + +/** + * Returns a cursor for the given query while ensuring that the cursor was loaded. + * + * When the SAF backend is a cloud storage provider (e.g. Nextcloud), + * it can happen that the query returns an outdated (e.g. empty) cursor + * which will only be updated in response to this query. + * + * See: https://commonsware.com/blog/2019/12/14/scoped-storage-stories-listfiles-woe.html + * + * This method uses a [suspendCancellableCoroutine] to wait for the result of a [ContentObserver] + * registered on the cursor in case the cursor is still loading ([EXTRA_LOADING]). + * If the cursor is not loading, it will be returned right away. + * + * @param timeout an optional time-out in milliseconds + * @throws TimeoutCancellationException if there was no result before the time-out + * @throws IOException if the query returns null + */ +@VisibleForTesting +@Throws(IOException::class, TimeoutCancellationException::class) +public suspend fun getLoadedCursor(timeout: Long = 15_000, query: () -> Cursor?): Cursor = + withTimeout(timeout) { + suspendCancellableCoroutine { cont -> + val cursor = query() ?: throw IOException() + cont.invokeOnCancellation { cursor.close() } + val loading = cursor.extras.getBoolean(EXTRA_LOADING, false) + if (loading) { + Log.d(TAG, "Wait for children to get loaded...") + cursor.registerContentObserver(object : ContentObserver(null) { + override fun onChange(selfChange: Boolean, uri: Uri?) { + Log.d(TAG, "Children loaded. Continue...") + cursor.close() + val newCursor = query() + if (newCursor == null) { + cont.cancel(IOException("query returned no results")) + } else cont.resume(newCursor) + } + }) + } else { + // not loading, return cursor right away + cont.resume(cursor) + } + } + } diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/saf/UriUtils.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/UriUtils.kt new file mode 100644 index 00000000..07d0d3e4 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/saf/UriUtils.kt @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: 2021 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.saf + +import android.content.ContentResolver +import android.net.Uri +import android.provider.MediaStore +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream + +public fun Uri.getDocumentPath(): String? { + return lastPathSegment?.split(':')?.getOrNull(1) +} + +public fun Uri.getVolume(): String? { + val volume = lastPathSegment?.split(':')?.getOrNull(0) + return if (volume == "primary") MediaStore.VOLUME_EXTERNAL_PRIMARY else volume +} + +@Throws(IOException::class) +public fun Uri.openInputStream(contentResolver: ContentResolver): InputStream { + return try { + contentResolver.openInputStream(this) + } catch (e: IllegalArgumentException) { + // This is necessary, because contrary to the documentation, files that have been deleted + // after we retrieved their Uri, will throw an IllegalArgumentException + throw IOException(e) + } ?: throw IOException("Stream for $this returned null") +} + +@Throws(IOException::class) +public fun Uri.openOutputStream(contentResolver: ContentResolver): OutputStream { + return try { + contentResolver.openOutputStream(this, "wt") + } catch (e: IllegalArgumentException) { + // This is necessary, because contrary to the documentation, files that have been deleted + // after we retrieved their Uri, will throw an IllegalArgumentException + throw IOException(e) + } ?: throw IOException("Stream for $this returned null") +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/GetLastModified.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/GetLastModified.kt new file mode 100644 index 00000000..46c78f58 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/GetLastModified.kt @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +import at.bitfire.dav4jvm.Property +import at.bitfire.dav4jvm.PropertyFactory +import at.bitfire.dav4jvm.property.webdav.NS_WEBDAV +import org.xmlpull.v1.XmlPullParser + +/** + * A fake version of [at.bitfire.dav4jvm.property.webdav.GetLastModified] which we register + * so we don't need to depend on `org.apache.commons.lang3` which is used for date parsing. + */ +internal class GetLastModified : Property { + companion object { + @JvmField + val NAME = Property.Name(NS_WEBDAV, "getlastmodified") + } + + object Factory : PropertyFactory { + override fun getName() = NAME + override fun create(parser: XmlPullParser): GetLastModified? = null + } +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/PipedCloseActionOutputStream.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/PipedCloseActionOutputStream.kt new file mode 100644 index 00000000..377a8c3e --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/PipedCloseActionOutputStream.kt @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +import java.io.IOException +import java.io.PipedInputStream +import java.io.PipedOutputStream + +internal class PipedCloseActionOutputStream( + inputStream: PipedInputStream, +) : PipedOutputStream(inputStream) { + + private var onClose: (() -> Unit)? = null + + override fun write(b: Int) { + try { + super.write(b) + } catch (e: Exception) { + try { + onClose?.invoke() + } catch (closeException: Exception) { + e.addSuppressed(closeException) + } + throw e + } + } + + override fun write(b: ByteArray, off: Int, len: Int) { + try { + super.write(b, off, len) + } catch (e: Exception) { + try { + onClose?.invoke() + } catch (closeException: Exception) { + e.addSuppressed(closeException) + } + throw e + } + } + + @Throws(IOException::class) + override fun close() { + super.close() + try { + onClose?.invoke() + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } + } + + fun doOnClose(function: () -> Unit) { + this.onClose = function + } +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackend.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackend.kt new file mode 100644 index 00000000..b479a733 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackend.kt @@ -0,0 +1,281 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +import at.bitfire.dav4jvm.BasicDigestAuthHandler +import at.bitfire.dav4jvm.DavCollection +import at.bitfire.dav4jvm.PropertyRegistry +import at.bitfire.dav4jvm.Response.HrefRelation.SELF +import at.bitfire.dav4jvm.exception.HttpException +import at.bitfire.dav4jvm.exception.NotFoundException +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import okhttp3.ConnectionSpec +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.RequestBody +import okio.BufferedSink +import okio.BufferedSource +import okio.buffer +import okio.sink +import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.Constants.DIRECTORY_ROOT +import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA +import org.calyxos.seedvault.core.backends.Constants.chunkFolderRegex +import org.calyxos.seedvault.core.backends.Constants.chunkRegex +import org.calyxos.seedvault.core.backends.Constants.folderRegex +import org.calyxos.seedvault.core.backends.Constants.snapshotRegex +import org.calyxos.seedvault.core.backends.Constants.tokenRegex +import org.calyxos.seedvault.core.backends.FileBackupFileType +import org.calyxos.seedvault.core.backends.FileHandle +import org.calyxos.seedvault.core.backends.FileInfo +import org.calyxos.seedvault.core.backends.LegacyAppBackupFile +import org.calyxos.seedvault.core.backends.TopLevelFolder +import java.io.IOException +import java.io.PipedInputStream +import java.util.concurrent.TimeUnit +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine +import kotlin.reflect.KClass + +private const val DEBUG_LOG = true + +@OptIn(DelicateCoroutinesApi::class) +internal class WebDavBackend( + webDavConfig: WebDavConfig, + root: String = DIRECTORY_ROOT, +) : Backend { + + private val log = KotlinLogging.logger {} + + private val authHandler = BasicDigestAuthHandler( + domain = null, // Optional, to only authenticate against hosts with this domain. + username = webDavConfig.username, + password = webDavConfig.password, + ) + private val okHttpClient = OkHttpClient.Builder() + .followRedirects(false) + .authenticator(authHandler) + .addNetworkInterceptor(authHandler) + .connectTimeout(30, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .readTimeout(240, TimeUnit.SECONDS) + .pingInterval(45, TimeUnit.SECONDS) + .connectionSpecs(listOf(ConnectionSpec.MODERN_TLS)) + .retryOnConnectionFailure(true) + .build() + + private val url = "${webDavConfig.url}/$root" + private val folders = mutableSetOf() // cache for existing/created folders + + init { + PropertyRegistry.register(GetLastModified.Factory) + } + + override suspend fun save(handle: FileHandle): BufferedSink { + val location = handle.toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + davCollection.ensureFoldersExist(log, folders) + + val pipedInputStream = PipedInputStream() + val pipedOutputStream = PipedCloseActionOutputStream(pipedInputStream) + + val body = object : RequestBody() { + override fun isOneShot(): Boolean = true + override fun contentType() = "application/octet-stream".toMediaType() + override fun writeTo(sink: BufferedSink) { + pipedInputStream.use { inputStream -> + sink.outputStream().use { outputStream -> + inputStream.copyTo(outputStream) + } + } + } + } + val deferred = GlobalScope.async(Dispatchers.IO) { + davCollection.put(body) { response -> + log.debugLog { "save($location) = $response" } + } + } + pipedOutputStream.doOnClose { + runBlocking { // blocking i/o wait + deferred.await() + } + } + return pipedOutputStream.sink().buffer() + } + + override suspend fun load(handle: FileHandle): BufferedSource { + val location = handle.toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + val response = davCollection.get(accept = "", headers = null) + log.debugLog { "load($location) = $response" } + if (response.code / 100 != 2) throw IOException("HTTP error ${response.code}") + return response.body?.source() ?: throw IOException() + } + + override suspend fun list( + topLevelFolder: TopLevelFolder?, + vararg fileTypes: KClass, + callback: (FileInfo) -> Unit, + ) { + if (TopLevelFolder::class in fileTypes) throw UnsupportedOperationException() + if (LegacyAppBackupFile::class in fileTypes) throw UnsupportedOperationException() + if (LegacyAppBackupFile.IconsFile::class in fileTypes) throw UnsupportedOperationException() + if (LegacyAppBackupFile.Blob::class in fileTypes) throw UnsupportedOperationException() + + // limit depth based on wanted types and if top-level folder is given + var depth = if (FileBackupFileType.Blob::class in fileTypes) 3 else 2 + if (topLevelFolder != null) depth -= 1 + + val location = if (topLevelFolder == null) { + "$url/".toHttpUrl() + } else { + "$url/${topLevelFolder.name}/".toHttpUrl() + } + val davCollection = DavCollection(okHttpClient, location) + val tokenFolders = mutableSetOf() + davCollection.propfindDepthInfinity(depth) { response, relation -> + log.debugLog { "list() = $response" } + + // work around nginx's inability to find files starting with . + if (relation != SELF && LegacyAppBackupFile.Metadata::class in fileTypes && + response.isFolder() && response.hrefName().matches(tokenRegex) + ) { + tokenFolders.add(response.href) + } + if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2) { + val name = response.hrefName() + val parentName = response.href.pathSegments[response.href.pathSegments.size - 2] + + if (LegacyAppBackupFile.Metadata::class in fileTypes) { + if (name == FILE_BACKUP_METADATA && parentName.matches(tokenRegex)) { + val metadata = LegacyAppBackupFile.Metadata(parentName.toLong()) + val size = response.properties.contentLength() + callback(FileInfo(metadata, size)) + // we can find .backup.metadata files, so no need for nginx workaround + tokenFolders.clear() + } + } + if (FileBackupFileType.Snapshot::class in fileTypes || + FileBackupFileType::class in fileTypes + ) { + val match = snapshotRegex.matchEntire(name) + if (match != null) { + val size = response.properties.contentLength() + val snapshot = FileBackupFileType.Snapshot( + androidId = parentName.substringBefore('.'), + time = match.groupValues[1].toLong(), + ) + callback(FileInfo(snapshot, size)) + } + } + if ((FileBackupFileType.Blob::class in fileTypes || + FileBackupFileType::class in fileTypes) && response.href.pathSize >= 3 + ) { + val androidIdSv = + response.href.pathSegments[response.href.pathSegments.size - 3] + if (folderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) { + if (chunkRegex.matches(name)) { + val blob = FileBackupFileType.Blob( + androidId = androidIdSv.substringBefore('.'), + name = name, + ) + val size = response.properties.contentLength() + callback(FileInfo(blob, size)) + } + } + } + } + } + // direct query for .backup.metadata as nginx doesn't support listing hidden files + tokenFolders.forEach { url -> + val metadataLocation = url.newBuilder().addPathSegment(FILE_BACKUP_METADATA).build() + try { + DavCollection(okHttpClient, metadataLocation).head { response -> + log.debugLog { "head($metadataLocation) = $response" } + val token = url.pathSegments.last { it.isNotBlank() }.toLong() + val metadata = LegacyAppBackupFile.Metadata(token) + val size = response.headers["content-length"]?.toLong() + ?: error("no content length") + callback(FileInfo(metadata, size)) + } + } catch (e: Exception) { + log.warn { "No $FILE_BACKUP_METADATA found in $url: $e" } + } + } + } + + override suspend fun remove(handle: FileHandle) { + val location = handle.toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + log.debugLog { "remove($handle)" } + + try { + val response = suspendCoroutine { cont -> + davCollection.delete { response -> + cont.resume(response) + } + } + log.debugLog { "remove($location) = $response" } + } catch (e: Exception) { + when (e) { + is NotFoundException -> log.info { "Not found: $location" } + is IOException -> throw e + else -> throw IOException(e) + } + } + } + + /** + * Renames [from] to [to]. + * + * @throws HttpException if [to] already exists + * * nginx code 412 + * * lighttp code 207 + * * dufs code 500 + */ + @Throws(HttpException::class) + override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) { + val location = "$url/${from.name}/".toHttpUrl() + val toUrl = "$url/${to.name}/".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + davCollection.move(toUrl, false) { response -> + log.debugLog { "rename(${from.name}, ${to.name}) = $response" } + } + } + + override suspend fun removeAll() { + val location = "$url/".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + try { + davCollection.delete { response -> + log.debugLog { "removeAll() = $response" } + } + } catch (e: NotFoundException) { + log.info { "Not found: $location" } + } + } + + private fun FileHandle.toHttpUrl(): HttpUrl = when (this) { + // careful with trailing slashes, use only for folders/collections + is TopLevelFolder -> "$url/$name/".toHttpUrl() + else -> "$url/$relativePath".toHttpUrl() + } + +} + +internal inline fun KLogger.debugLog(crossinline block: () -> String) { + if (DEBUG_LOG) debug { block() } +} diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavConfig.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavConfig.kt new file mode 100644 index 00000000..a2d2c180 --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavConfig.kt @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +public data class WebDavConfig( + val url: String, + val username: String, + val password: String, +) diff --git a/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavHelper.kt b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavHelper.kt new file mode 100644 index 00000000..8e594bce --- /dev/null +++ b/core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavHelper.kt @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +import at.bitfire.dav4jvm.DavCollection +import at.bitfire.dav4jvm.MultiResponseCallback +import at.bitfire.dav4jvm.Property +import at.bitfire.dav4jvm.Response +import at.bitfire.dav4jvm.Response.HrefRelation.SELF +import at.bitfire.dav4jvm.ResponseCallback +import at.bitfire.dav4jvm.exception.ConflictException +import at.bitfire.dav4jvm.exception.HttpException +import at.bitfire.dav4jvm.exception.NotFoundException +import at.bitfire.dav4jvm.property.webdav.DisplayName +import at.bitfire.dav4jvm.property.webdav.GetContentLength +import at.bitfire.dav4jvm.property.webdav.ResourceType +import io.github.oshai.kotlinlogging.KLogger +import okhttp3.HttpUrl + +/** + * Tries to do [DavCollection.propfind] with a depth of `-1`. + * Since `infinity` isn't supported by nginx either, + * we fallback to iterating over all folders found with depth `1` + * and do another PROPFIND on those, passing the given [callback]. + * + * @param maxDepth in case we need to fallback to recursive propfinds, we only go that far down. + */ +internal fun DavCollection.propfindDepthInfinity(maxDepth: Int, callback: MultiResponseCallback) { + try { + propfind( + depth = -1, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME, GetContentLength.NAME), + callback = callback, + ) + } catch (e: HttpException) { + if (e.isUnsupportedPropfind()) { + log.info { "Got ${e.response}, trying recursive depth=1 PROPFINDs..." } + propfindFakeInfinity(maxDepth, callback) + } else { + throw e + } + } +} + +internal fun DavCollection.propfindFakeInfinity(depth: Int, callback: MultiResponseCallback) { + if (depth <= 0) return + propfind( + depth = 1, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME, GetContentLength.NAME), + ) { response, relation -> + // This callback will be called for everything in the folder + callback.onResponse(response, relation) + if (relation != SELF && response.isFolder()) { + DavCollection(httpClient, response.href).propfindFakeInfinity(depth - 1, callback) + } + } +} + +internal fun DavCollection.mkColCreateMissing(callback: ResponseCallback) { + try { + mkCol(null) { response -> + callback.onResponse(response) + } + } catch (e: ConflictException) { + log.warning { "Error creating $location: $e" } + if (location.pathSize <= 1) throw e + val newLocation = location.newBuilder() + .removePathSegment(location.pathSize - 1) + .build() + DavCollection(httpClient, newLocation).mkColCreateMissing(callback) + // re-run original command to create parent collection + mkCol(null) { response -> + callback.onResponse(response) + } + } +} + +internal fun DavCollection.ensureFoldersExist(log: KLogger, folders: MutableSet) { + if (location.pathSize <= 2) return + val parent = location.newBuilder() + .removePathSegment(location.pathSize - 1) + .build() + if (parent in folders) return + val parentCollection = DavCollection(httpClient, parent) + try { + parentCollection.head { response -> + log.debugLog { "head($parent) = $response" } + folders.add(parent) + } + } catch (e: NotFoundException) { + log.debugLog { "$parent not found, creating..." } + parentCollection.mkColCreateMissing { response -> + log.debugLog { "mkColCreateMissing($parent) = $response" } + folders.add(parent) + } + } +} + +private fun HttpException.isUnsupportedPropfind(): Boolean { + // nginx is not including 'propfind-finite-depth' in body, so just relay on code + return code == 403 || code == 400 // dufs returns 400 +} + +internal fun List.contentLength(): Long { + // crash intentionally, if this isn't in the list + return filterIsInstance()[0].contentLength +} + +internal fun Response.isFolder(): Boolean { + return this[ResourceType::class.java]?.types?.contains(ResourceType.COLLECTION) == true +} diff --git a/core/src/test/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackendTest.kt b/core/src/test/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackendTest.kt new file mode 100644 index 00000000..0edf9f6a --- /dev/null +++ b/core/src/test/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackendTest.kt @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +import kotlinx.coroutines.runBlocking +import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.BackendTest +import kotlin.test.Test + +public class WebDavBackendTest : BackendTest() { + override val plugin: Backend = WebDavBackend(WebDavTestConfig.getConfig(), ".SeedvaultTest") + + @Test + public fun `test write, list, read, rename, delete`(): Unit = runBlocking { + testWriteListReadRenameDelete() + } +} diff --git a/core/src/test/java/org/calyxos/seedvault/core/backends/webdav/WebDavTestConfig.kt b/core/src/test/java/org/calyxos/seedvault/core/backends/webdav/WebDavTestConfig.kt new file mode 100644 index 00000000..b22558f0 --- /dev/null +++ b/core/src/test/java/org/calyxos/seedvault/core/backends/webdav/WebDavTestConfig.kt @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.calyxos.seedvault.core.backends.webdav + +import org.junit.Assume.assumeFalse +import kotlin.test.fail + +internal object WebDavTestConfig { + + fun getConfig(): WebDavConfig { + assumeFalse(System.getenv("NEXTCLOUD_URL").isNullOrEmpty()) + return WebDavConfig( + url = System.getenv("NEXTCLOUD_URL") ?: fail(), + username = System.getenv("NEXTCLOUD_USER") ?: fail(), + password = System.getenv("NEXTCLOUD_PASS") ?: fail(), + ) + } + +} diff --git a/core/src/test/resources/simplelogger.properties b/core/src/test/resources/simplelogger.properties new file mode 100644 index 00000000..e0f0d79d --- /dev/null +++ b/core/src/test/resources/simplelogger.properties @@ -0,0 +1 @@ +org.slf4j.simpleLogger.defaultLogLevel=trace diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5e1efe7a..c4299fc0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -115,5 +115,6 @@ android-library = { id = "com.android.library", version.ref = "androidGradlePlug google-ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } google-protobuf = { id = "com.google.protobuf", version.ref = "protobufPlugin" } jetbrains-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } +jetbrains-kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } jlleitschuh-ktlint = { id = "org.jlleitschuh.gradle.ktlint", version.ref = "ktlint" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 2350388d..812a36f2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -21,6 +21,7 @@ dependencyResolutionManagement { } rootProject.name = "Seedvault" +include(":core") include(":app") include(":contactsbackup") include(":storage:lib")