Add core gradle module with unified storage backends
This commit is contained in:
Torsten Grote
parent
fa56a74ad7
commit
9e56384cb2
28 changed files with 1474 additions and 13 deletions
|
|
@ -106,19 +106,7 @@ android {
|
|||
}
|
||||
|
||||
dependencies {
|
||||
|
||||
val aospLibs = fileTree("$projectDir/libs") {
|
||||
// For more information about this module:
|
||||
// https://android.googlesource.com/platform/frameworks/base/+/refs/tags/android-11.0.0_r3/Android.bp#507
|
||||
// framework_intermediates/classes-header.jar works for gradle build as well,
|
||||
// but not unit tests, so we use the actual classes (without updatable modules).
|
||||
//
|
||||
// out/target/common/obj/JAVA_LIBRARIES/framework-minus-apex_intermediates/classes.jar
|
||||
include("android.jar")
|
||||
// out/target/common/obj/JAVA_LIBRARIES/core-libart.com.android.art_intermediates/classes.jar
|
||||
include("libcore.jar")
|
||||
}
|
||||
|
||||
val aospLibs: FileTree by rootProject.extra
|
||||
compileOnly(aospLibs)
|
||||
|
||||
/**
|
||||
|
|
@ -149,6 +137,7 @@ dependencies {
|
|||
/**
|
||||
* Storage Dependencies
|
||||
*/
|
||||
implementation(project(":core"))
|
||||
implementation(project(":storage:lib"))
|
||||
|
||||
/**
|
||||
|
|
@ -188,6 +177,7 @@ dependencies {
|
|||
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}")
|
||||
|
||||
androidTestImplementation(aospLibs)
|
||||
androidTestImplementation(kotlin("test"))
|
||||
androidTestImplementation("androidx.test:runner:1.4.0")
|
||||
androidTestImplementation("androidx.test:rules:1.4.0")
|
||||
androidTestImplementation("androidx.test.ext:junit:1.1.3")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package com.stevesoltys.seedvault.plugins.saf
|
||||
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import androidx.test.filters.MediumTest
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import com.stevesoltys.seedvault.settings.SettingsManager
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.calyxos.seedvault.core.backends.Backend
|
||||
import org.calyxos.seedvault.core.backends.BackendTest
|
||||
import org.calyxos.seedvault.core.backends.saf.SafBackend
|
||||
import org.calyxos.seedvault.core.backends.saf.SafConfig
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
@MediumTest
|
||||
class SafBackendTest : BackendTest(), KoinComponent {
|
||||
|
||||
private val context = InstrumentationRegistry.getInstrumentation().targetContext
|
||||
private val settingsManager by inject<SettingsManager>()
|
||||
override val plugin: Backend
|
||||
get() {
|
||||
val safStorage = settingsManager.getSafStorage() ?: error("No SAF storage")
|
||||
val safConfig = SafConfig(
|
||||
config = safStorage.config,
|
||||
name = safStorage.name,
|
||||
isUsb = safStorage.isUsb,
|
||||
requiresNetwork = safStorage.requiresNetwork,
|
||||
rootId = safStorage.rootId,
|
||||
)
|
||||
return SafBackend(context, safConfig, ".SeedvaultTest")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun test(): Unit = runBlocking {
|
||||
testWriteListReadRenameDelete()
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,21 @@ plugins {
|
|||
alias(libs.plugins.jetbrains.kotlin.android) apply false
|
||||
alias(libs.plugins.jetbrains.dokka) apply false
|
||||
alias(libs.plugins.jlleitschuh.ktlint) apply false
|
||||
alias(libs.plugins.jetbrains.kotlin.jvm) apply false
|
||||
}
|
||||
|
||||
val aospLibs by extra {
|
||||
fileTree("$rootDir/app/libs") { // TODO move libs to root
|
||||
// For more information about this module:
|
||||
// https://android.googlesource.com/platform/frameworks/base/+/refs/tags/android-11.0.0_r3/Android.bp#507
|
||||
// framework_intermediates/classes-header.jar works for gradle build as well,
|
||||
// but not unit tests, so we use the actual classes (without updatable modules).
|
||||
//
|
||||
// out/target/common/obj/JAVA_LIBRARIES/framework-minus-apex_intermediates/classes.jar
|
||||
include("android.jar")
|
||||
// out/target/common/obj/JAVA_LIBRARIES/core-libart.com.android.art_intermediates/classes.jar
|
||||
include("libcore.jar")
|
||||
}
|
||||
}
|
||||
|
||||
subprojects {
|
||||
|
|
|
|||
1
core/.gitignore
vendored
Normal file
1
core/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
/build
|
||||
51
core/build.gradle.kts
Normal file
51
core/build.gradle.kts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
plugins {
|
||||
alias(libs.plugins.android.library)
|
||||
alias(libs.plugins.jetbrains.kotlin.android)
|
||||
}
|
||||
|
||||
java {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "org.calyxos.seedvault.core"
|
||||
compileSdk = libs.versions.compileSdk.get().toInt()
|
||||
|
||||
defaultConfig {
|
||||
minSdk = libs.versions.minSdk.get().toInt()
|
||||
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
|
||||
testInstrumentationRunnerArguments["disableAnalytics"] = "true"
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
|
||||
kotlinOptions {
|
||||
jvmTarget = JavaVersion.VERSION_17.toString()
|
||||
languageVersion = "1.8"
|
||||
freeCompilerArgs += listOf(
|
||||
"-opt-in=kotlin.RequiresOptIn",
|
||||
"-Xexplicit-api=strict"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
val aospLibs: FileTree by rootProject.extra
|
||||
compileOnly(aospLibs)
|
||||
compileOnly("org.ogce:xpp3:1.1.6")
|
||||
compileOnly(kotlin("test"))
|
||||
implementation(libs.bundles.kotlin)
|
||||
implementation(libs.bundles.coroutines)
|
||||
implementation(libs.androidx.documentfile)
|
||||
// implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("okio-jvm-3.7.0.jar"))
|
||||
implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("*.jar"))
|
||||
implementation("io.github.oshai:kotlin-logging-jvm:6.0.3")
|
||||
implementation("org.slf4j:slf4j-simple:2.0.3")
|
||||
|
||||
testImplementation(kotlin("test"))
|
||||
testImplementation("org.ogce:xpp3:1.1.6")
|
||||
}
|
||||
8
core/src/main/AndroidManifest.xml
Normal file
8
core/src/main/AndroidManifest.xml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?><!--
|
||||
SPDX-FileCopyrightText: 2021 The Calyx Institute
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
-->
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
</manifest>
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2021 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core
|
||||
|
||||
internal fun ByteArray.toHexString() = joinToString("") { "%02x".format(it) }
|
||||
|
||||
internal fun String.toByteArrayFromHex() = chunked(2).map { it.toInt(16).toByte() }.toByteArray()
|
||||
24
core/src/main/java/org/calyxos/seedvault/core/Utils.kt
Normal file
24
core/src/main/java/org/calyxos/seedvault/core/Utils.kt
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core
|
||||
|
||||
import android.Manifest.permission.INTERACT_ACROSS_USERS_FULL
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager.PERMISSION_GRANTED
|
||||
import android.os.UserManager
|
||||
|
||||
/**
|
||||
* Hack to allow other profiles access to USB backend.
|
||||
* @return the context of the device's main user, so use with great care!
|
||||
*/
|
||||
@Suppress("MissingPermission")
|
||||
public fun Context.getBackendContext(isUsbStorage: () -> Boolean): Context {
|
||||
if (checkSelfPermission(INTERACT_ACROSS_USERS_FULL) == PERMISSION_GRANTED && isUsbStorage()) {
|
||||
UserManager.get(this).getProfileParent(user)
|
||||
?.let { parent -> return createContextAsUser(parent, 0) }
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends
|
||||
|
||||
import okio.BufferedSink
|
||||
import okio.BufferedSource
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public interface Backend {
|
||||
|
||||
public suspend fun save(handle: FileHandle): BufferedSink
|
||||
|
||||
public suspend fun load(handle: FileHandle): BufferedSource
|
||||
|
||||
public suspend fun list(
|
||||
topLevelFolder: TopLevelFolder?,
|
||||
vararg fileTypes: KClass<out FileHandle>,
|
||||
callback: (FileInfo) -> Unit,
|
||||
)
|
||||
|
||||
public suspend fun remove(handle: FileHandle)
|
||||
|
||||
public suspend fun rename(from: TopLevelFolder, to: TopLevelFolder)
|
||||
|
||||
// TODO really all?
|
||||
public suspend fun removeAll()
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends
|
||||
|
||||
import android.annotation.WorkerThread
|
||||
import android.content.Context
|
||||
import android.net.ConnectivityManager
|
||||
import android.net.NetworkCapabilities.NET_CAPABILITY_INTERNET
|
||||
import at.bitfire.dav4jvm.exception.HttpException
|
||||
import java.io.IOException
|
||||
|
||||
public abstract class BackendProperties<T> {
|
||||
public abstract val config: T
|
||||
public abstract val name: String
|
||||
public abstract val isUsb: Boolean
|
||||
public abstract val requiresNetwork: Boolean
|
||||
|
||||
@WorkerThread
|
||||
public abstract fun isUnavailableUsb(context: Context): Boolean
|
||||
|
||||
/**
|
||||
* Returns true if this is storage that requires network access,
|
||||
* but it isn't available right now.
|
||||
*/
|
||||
public fun isUnavailableNetwork(context: Context, allowMetered: Boolean): Boolean {
|
||||
return requiresNetwork && !hasUnmeteredInternet(context, allowMetered)
|
||||
}
|
||||
|
||||
private fun hasUnmeteredInternet(context: Context, allowMetered: Boolean): Boolean {
|
||||
val cm = context.getSystemService(ConnectivityManager::class.java) ?: return false
|
||||
val isMetered = cm.isActiveNetworkMetered
|
||||
val capabilities = cm.getNetworkCapabilities(cm.activeNetwork) ?: return false
|
||||
return capabilities.hasCapability(NET_CAPABILITY_INTERNET) && (allowMetered || !isMetered)
|
||||
}
|
||||
}
|
||||
|
||||
public fun Exception.isOutOfSpace(): Boolean {
|
||||
return when (this) {
|
||||
is IOException -> message?.contains("No space left on device") == true ||
|
||||
(cause as? HttpException)?.code == 507
|
||||
|
||||
is HttpException -> code == 507
|
||||
|
||||
else -> false
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends
|
||||
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import at.bitfire.dav4jvm.exception.HttpException
|
||||
import org.calyxos.seedvault.core.toHexString
|
||||
import org.junit.Assert.assertArrayEquals
|
||||
import kotlin.random.Random
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFailsWith
|
||||
import kotlin.test.assertNotNull
|
||||
import kotlin.test.fail
|
||||
|
||||
@VisibleForTesting
|
||||
public abstract class BackendTest {
|
||||
|
||||
public abstract val plugin: Backend
|
||||
|
||||
protected suspend fun testWriteListReadRenameDelete() {
|
||||
try {
|
||||
plugin.removeAll()
|
||||
} catch (e: HttpException) {
|
||||
if (e.code != 404) fail(e.message, e)
|
||||
}
|
||||
|
||||
val now = System.currentTimeMillis()
|
||||
val bytes1 = Random.nextBytes(1337)
|
||||
val bytes2 = Random.nextBytes(1337 * 8)
|
||||
plugin.save(LegacyAppBackupFile.Metadata(now)).use {
|
||||
it.write(bytes1)
|
||||
}
|
||||
|
||||
plugin.save(FileBackupFileType.Snapshot("0123456789abcdef", now)).use {
|
||||
it.write(bytes2)
|
||||
}
|
||||
|
||||
var metadata: LegacyAppBackupFile.Metadata? = null
|
||||
var snapshot: FileBackupFileType.Snapshot? = null
|
||||
plugin.list(
|
||||
null,
|
||||
FileBackupFileType.Snapshot::class,
|
||||
FileBackupFileType.Blob::class,
|
||||
LegacyAppBackupFile.Metadata::class,
|
||||
) { fileInfo ->
|
||||
val handle = fileInfo.fileHandle
|
||||
if (handle is LegacyAppBackupFile.Metadata && handle.token == now) {
|
||||
metadata = handle
|
||||
} else if (handle is FileBackupFileType.Snapshot && handle.time == now) {
|
||||
snapshot = handle
|
||||
}
|
||||
}
|
||||
assertNotNull(metadata)
|
||||
assertNotNull(snapshot)
|
||||
|
||||
assertArrayEquals(bytes1, plugin.load(metadata as FileHandle).readByteArray())
|
||||
assertArrayEquals(bytes2, plugin.load(snapshot as FileHandle).readByteArray())
|
||||
|
||||
val blobName = Random.nextBytes(32).toHexString()
|
||||
var blob: FileBackupFileType.Blob? = null
|
||||
val bytes3 = Random.nextBytes(1337 * 16)
|
||||
plugin.save(FileBackupFileType.Blob("0123456789abcdef", blobName)).use {
|
||||
it.write(bytes3)
|
||||
}
|
||||
plugin.list(
|
||||
null,
|
||||
FileBackupFileType.Snapshot::class,
|
||||
FileBackupFileType.Blob::class,
|
||||
LegacyAppBackupFile.Metadata::class,
|
||||
) { fileInfo ->
|
||||
val handle = fileInfo.fileHandle
|
||||
if (handle is FileBackupFileType.Blob && handle.name == blobName) {
|
||||
blob = handle
|
||||
}
|
||||
}
|
||||
assertNotNull(blob)
|
||||
assertArrayEquals(bytes3, plugin.load(blob as FileHandle).readByteArray())
|
||||
|
||||
// try listing with top-level folder, should find two files of FileBackupFileType in there
|
||||
var numFiles = 0
|
||||
plugin.list(
|
||||
snapshot!!.topLevelFolder,
|
||||
FileBackupFileType.Snapshot::class,
|
||||
FileBackupFileType.Blob::class,
|
||||
LegacyAppBackupFile.Metadata::class,
|
||||
) { numFiles++ }
|
||||
assertEquals(2, numFiles)
|
||||
|
||||
plugin.remove(snapshot as FileHandle)
|
||||
|
||||
// rename snapshots
|
||||
val snapshotNewFolder = TopLevelFolder("0123456789abcdee.sv")
|
||||
plugin.rename(snapshot!!.topLevelFolder, snapshotNewFolder)
|
||||
|
||||
// rename to existing folder should fail
|
||||
val e = assertFailsWith<Exception> {
|
||||
plugin.rename(snapshotNewFolder, metadata!!.topLevelFolder)
|
||||
}
|
||||
println(e)
|
||||
|
||||
plugin.remove(metadata!!.topLevelFolder)
|
||||
plugin.remove(snapshotNewFolder)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends
|
||||
|
||||
public object Constants {
|
||||
|
||||
public const val DIRECTORY_ROOT: String = ".SeedVaultAndroidBackup"
|
||||
internal const val FILE_BACKUP_METADATA = ".backup.metadata"
|
||||
internal const val FILE_BACKUP_ICONS = ".backup.icons"
|
||||
public val tokenRegex: Regex = Regex("([0-9]{13})") // good until the year 2286
|
||||
public const val SNAPSHOT_EXT: String = ".SeedSnap"
|
||||
public val folderRegex: Regex = Regex("^[a-f0-9]{16}\\.sv$")
|
||||
public val chunkFolderRegex: Regex = Regex("[a-f0-9]{2}")
|
||||
public val chunkRegex: Regex = Regex("[a-f0-9]{64}")
|
||||
public val snapshotRegex: Regex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
|
||||
public const val MIME_TYPE: String = "application/octet-stream"
|
||||
public const val CHUNK_FOLDER_COUNT: Int = 256
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends
|
||||
|
||||
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_ICONS
|
||||
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA
|
||||
import org.calyxos.seedvault.core.backends.Constants.SNAPSHOT_EXT
|
||||
|
||||
public sealed class FileHandle {
|
||||
public abstract val name: String
|
||||
|
||||
/**
|
||||
* The relative path relative to the storage root without prepended or trailing slash (/).
|
||||
*/
|
||||
public abstract val relativePath: String
|
||||
}
|
||||
|
||||
public data class TopLevelFolder(override val name: String) : FileHandle() {
|
||||
override val relativePath: String = name
|
||||
}
|
||||
|
||||
public sealed class LegacyAppBackupFile : FileHandle() {
|
||||
public abstract val token: Long
|
||||
public val topLevelFolder: TopLevelFolder get() = TopLevelFolder(token.toString())
|
||||
override val relativePath: String get() = "$token/$name"
|
||||
|
||||
public data class Metadata(override val token: Long) : LegacyAppBackupFile() {
|
||||
override val name: String = FILE_BACKUP_METADATA
|
||||
}
|
||||
|
||||
public data class IconsFile(override val token: Long) : LegacyAppBackupFile() {
|
||||
override val name: String = FILE_BACKUP_ICONS
|
||||
}
|
||||
|
||||
public data class Blob(
|
||||
override val token: Long,
|
||||
override val name: String,
|
||||
) : LegacyAppBackupFile()
|
||||
}
|
||||
|
||||
public sealed class FileBackupFileType : FileHandle() {
|
||||
public abstract val androidId: String
|
||||
public val topLevelFolder: TopLevelFolder get() = TopLevelFolder("$androidId.sv")
|
||||
|
||||
public data class Blob(
|
||||
override val androidId: String,
|
||||
override val name: String,
|
||||
) : FileBackupFileType() {
|
||||
override val relativePath: String get() = "$androidId.sv/${name.substring(0, 2)}/$name"
|
||||
}
|
||||
|
||||
public data class Snapshot(
|
||||
override val androidId: String,
|
||||
val time: Long,
|
||||
) : FileBackupFileType() {
|
||||
override val name: String = "$time$SNAPSHOT_EXT"
|
||||
override val relativePath: String get() = "$androidId.sv/$name"
|
||||
}
|
||||
}
|
||||
|
||||
public data class FileInfo(
|
||||
val fileHandle: FileHandle,
|
||||
val size: Long,
|
||||
)
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.saf
|
||||
|
||||
import android.content.Context
|
||||
import androidx.documentfile.provider.DocumentFile
|
||||
import org.calyxos.seedvault.core.backends.FileBackupFileType
|
||||
import org.calyxos.seedvault.core.backends.FileHandle
|
||||
import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
|
||||
import org.calyxos.seedvault.core.backends.TopLevelFolder
|
||||
|
||||
internal class DocumentFileCache(
|
||||
private val context: Context,
|
||||
private val baseFile: DocumentFile,
|
||||
private val root: String,
|
||||
) {
|
||||
|
||||
private val cache = mutableMapOf<String, DocumentFile>()
|
||||
|
||||
internal suspend fun getRootFile(): DocumentFile {
|
||||
return cache.getOrPut(root) {
|
||||
baseFile.getOrCreateDirectory(context, root)
|
||||
}
|
||||
}
|
||||
|
||||
internal suspend fun getFile(fh: FileHandle): DocumentFile = when (fh) {
|
||||
is TopLevelFolder -> cache.getOrPut("$root/${fh.relativePath}") {
|
||||
getRootFile().getOrCreateDirectory(context, fh.name)
|
||||
}
|
||||
|
||||
is LegacyAppBackupFile -> cache.getOrPut("$root/${fh.relativePath}") {
|
||||
getFile(fh.topLevelFolder).getOrCreateFile(context, fh.name)
|
||||
}
|
||||
|
||||
is FileBackupFileType.Blob -> {
|
||||
val subFolderName = fh.name.substring(0, 2)
|
||||
cache.getOrPut("$root/${fh.topLevelFolder.name}/$subFolderName") {
|
||||
getFile(fh.topLevelFolder).getOrCreateDirectory(context, subFolderName)
|
||||
}.getOrCreateFile(context, fh.name)
|
||||
}
|
||||
|
||||
is FileBackupFileType.Snapshot -> {
|
||||
getFile(fh.topLevelFolder).getOrCreateFile(context, fh.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.saf
|
||||
|
||||
import android.content.Context
|
||||
import android.provider.DocumentsContract.renameDocument
|
||||
import androidx.documentfile.provider.DocumentFile
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import okio.BufferedSink
|
||||
import okio.BufferedSource
|
||||
import okio.buffer
|
||||
import okio.sink
|
||||
import okio.source
|
||||
import org.calyxos.seedvault.core.backends.Backend
|
||||
import org.calyxos.seedvault.core.backends.Constants.DIRECTORY_ROOT
|
||||
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA
|
||||
import org.calyxos.seedvault.core.backends.Constants.chunkFolderRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.chunkRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.folderRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.snapshotRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.tokenRegex
|
||||
import org.calyxos.seedvault.core.backends.FileBackupFileType
|
||||
import org.calyxos.seedvault.core.backends.FileHandle
|
||||
import org.calyxos.seedvault.core.backends.FileInfo
|
||||
import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
|
||||
import org.calyxos.seedvault.core.backends.TopLevelFolder
|
||||
import org.calyxos.seedvault.core.getBackendContext
|
||||
import java.io.IOException
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public class SafBackend(
|
||||
private val appContext: Context,
|
||||
private val safConfig: SafConfig,
|
||||
root: String = DIRECTORY_ROOT,
|
||||
) : Backend {
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
/**
|
||||
* Attention: This context might be from a different user. Use with care.
|
||||
*/
|
||||
private val context: Context get() = appContext.getBackendContext { safConfig.isUsb }
|
||||
private val cache = DocumentFileCache(context, safConfig.getDocumentFile(context), root)
|
||||
|
||||
override suspend fun save(handle: FileHandle): BufferedSink {
|
||||
val file = cache.getFile(handle)
|
||||
return file.getOutputStream(context.contentResolver).sink().buffer()
|
||||
}
|
||||
|
||||
override suspend fun load(handle: FileHandle): BufferedSource {
|
||||
val file = cache.getFile(handle)
|
||||
return file.getInputStream(context.contentResolver).source().buffer()
|
||||
}
|
||||
|
||||
override suspend fun list(
|
||||
topLevelFolder: TopLevelFolder?,
|
||||
vararg fileTypes: KClass<out FileHandle>,
|
||||
callback: (FileInfo) -> Unit,
|
||||
) {
|
||||
if (TopLevelFolder::class in fileTypes) throw UnsupportedOperationException()
|
||||
if (LegacyAppBackupFile::class in fileTypes) throw UnsupportedOperationException()
|
||||
if (LegacyAppBackupFile.IconsFile::class in fileTypes) throw UnsupportedOperationException()
|
||||
if (LegacyAppBackupFile.Blob::class in fileTypes) throw UnsupportedOperationException()
|
||||
|
||||
val folder = if (topLevelFolder == null) {
|
||||
cache.getRootFile()
|
||||
} else {
|
||||
cache.getFile(topLevelFolder)
|
||||
}
|
||||
// limit depth based on wanted types and if top-level folder is given
|
||||
var depth = if (FileBackupFileType.Blob::class in fileTypes) 3 else 2
|
||||
if (topLevelFolder != null) depth -= 1
|
||||
|
||||
folder.listFilesRecursive(depth) { file ->
|
||||
if (!file.isFile) return@listFilesRecursive
|
||||
val parentName = file.parentFile?.name ?: return@listFilesRecursive
|
||||
val name = file.name ?: return@listFilesRecursive
|
||||
if (LegacyAppBackupFile.Metadata::class in fileTypes && name == FILE_BACKUP_METADATA &&
|
||||
parentName.matches(tokenRegex)
|
||||
) {
|
||||
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
|
||||
callback(FileInfo(metadata, file.length()))
|
||||
}
|
||||
if (FileBackupFileType.Snapshot::class in fileTypes ||
|
||||
FileBackupFileType::class in fileTypes
|
||||
) {
|
||||
val match = snapshotRegex.matchEntire(name)
|
||||
if (match != null) {
|
||||
val snapshot = FileBackupFileType.Snapshot(
|
||||
androidId = parentName.substringBefore('.'),
|
||||
time = match.groupValues[1].toLong(),
|
||||
)
|
||||
callback(FileInfo(snapshot, file.length()))
|
||||
}
|
||||
}
|
||||
if ((FileBackupFileType.Blob::class in fileTypes ||
|
||||
FileBackupFileType::class in fileTypes)
|
||||
) {
|
||||
val androidIdSv = file.parentFile?.parentFile?.name ?: ""
|
||||
if (folderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) {
|
||||
if (chunkRegex.matches(name)) {
|
||||
val blob = FileBackupFileType.Blob(
|
||||
androidId = androidIdSv.substringBefore('.'),
|
||||
name = name,
|
||||
)
|
||||
callback(FileInfo(blob, file.length()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun DocumentFile.listFilesRecursive(
|
||||
depth: Int,
|
||||
callback: (DocumentFile) -> Unit,
|
||||
) {
|
||||
if (depth <= 0) return
|
||||
listFilesBlocking(context).forEach { file ->
|
||||
callback(file)
|
||||
if (file.isDirectory) file.listFilesRecursive(depth - 1, callback)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun remove(handle: FileHandle) {
|
||||
val file = cache.getFile(handle)
|
||||
if (!file.delete()) throw IOException("could not delete ${handle.relativePath}")
|
||||
}
|
||||
|
||||
override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) {
|
||||
val fromFile = cache.getFile(from)
|
||||
// don't use fromFile.renameTo(to.name) as that creates "${to.name} (1)"
|
||||
val newUri = renameDocument(context.contentResolver, fromFile.uri, to.name)
|
||||
?: throw IOException("could not rename ${from.relativePath}")
|
||||
val toFile = DocumentFile.fromTreeUri(context, newUri)
|
||||
?: throw IOException("renamed URI invalid: $newUri")
|
||||
if (toFile.name != to.name) {
|
||||
toFile.delete()
|
||||
throw IOException("renamed to ${toFile.name}, but expected ${to.name}")
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun removeAll() {
|
||||
cache.getRootFile().listFilesBlocking(context).forEach {
|
||||
it.delete()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.saf
|
||||
|
||||
import android.content.Context
|
||||
import android.net.Uri
|
||||
import android.provider.DocumentsContract.Root.COLUMN_ROOT_ID
|
||||
import androidx.annotation.WorkerThread
|
||||
import androidx.documentfile.provider.DocumentFile
|
||||
import org.calyxos.seedvault.core.backends.BackendProperties
|
||||
|
||||
public data class SafConfig(
|
||||
override val config: Uri,
|
||||
override val name: String,
|
||||
override val isUsb: Boolean,
|
||||
override val requiresNetwork: Boolean,
|
||||
/**
|
||||
* The [COLUMN_ROOT_ID] for the [uri].
|
||||
* This is only nullable for historic reasons, because we didn't always store it.
|
||||
*/
|
||||
val rootId: String?,
|
||||
) : BackendProperties<Uri>() {
|
||||
|
||||
internal val uri: Uri = config
|
||||
|
||||
public fun getDocumentFile(context: Context): DocumentFile =
|
||||
DocumentFile.fromTreeUri(context, config)
|
||||
?: throw AssertionError("Should only happen on API < 21.")
|
||||
|
||||
/**
|
||||
* Returns true if this is USB storage that is not available, false otherwise.
|
||||
*
|
||||
* Must be run off UI thread (ideally I/O).
|
||||
*/
|
||||
@WorkerThread
|
||||
override fun isUnavailableUsb(context: Context): Boolean {
|
||||
return isUsb && !getDocumentFile(context).isDirectory
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,217 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2021 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.saf
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.ContentResolver
|
||||
import android.content.Context
|
||||
import android.database.ContentObserver
|
||||
import android.database.Cursor
|
||||
import android.net.Uri
|
||||
import android.provider.DocumentsContract.Document.COLUMN_DOCUMENT_ID
|
||||
import android.provider.DocumentsContract.EXTRA_LOADING
|
||||
import android.provider.DocumentsContract.buildChildDocumentsUriUsingTree
|
||||
import android.provider.DocumentsContract.buildDocumentUriUsingTree
|
||||
import android.provider.DocumentsContract.getDocumentId
|
||||
import android.util.Log
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import androidx.documentfile.provider.DocumentFile
|
||||
import kotlinx.coroutines.TimeoutCancellationException
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import org.calyxos.seedvault.core.backends.Constants.MIME_TYPE
|
||||
import java.io.IOException
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
private const val TAG = "SafHelper"
|
||||
|
||||
@Throws(IOException::class)
|
||||
public fun DocumentFile.getInputStream(contentResolver: ContentResolver): InputStream {
|
||||
return uri.openInputStream(contentResolver)
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
public fun DocumentFile.getOutputStream(contentResolver: ContentResolver): OutputStream {
|
||||
return uri.openOutputStream(contentResolver)
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a file exists and if not, creates it.
|
||||
*
|
||||
* If we were trying to create it right away, some providers create "filename (1)".
|
||||
*/
|
||||
@Throws(IOException::class)
|
||||
internal suspend fun DocumentFile.getOrCreateFile(context: Context, name: String): DocumentFile {
|
||||
return try {
|
||||
findFileBlocking(context, name) ?: createFileOrThrow(name, MIME_TYPE)
|
||||
} catch (e: Exception) {
|
||||
// SAF can throw all sorts of exceptions, so wrap it in IOException.
|
||||
// E.g. IllegalArgumentException can be thrown by FileSystemProvider#isChildDocument()
|
||||
// when flash drive is not plugged-in:
|
||||
// http://aosp.opersys.com/xref/android-11.0.0_r8/xref/frameworks/base/core/java/com/android/internal/content/FileSystemProvider.java#135
|
||||
if (e is IOException) throw e
|
||||
else throw IOException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
internal fun DocumentFile.createFileOrThrow(
|
||||
name: String,
|
||||
mimeType: String = MIME_TYPE,
|
||||
): DocumentFile {
|
||||
val file = createFile(mimeType, name) ?: throw IOException("Unable to create file: $name")
|
||||
if (file.name != name) {
|
||||
file.delete()
|
||||
if (file.name == null) { // this happens when file existed already
|
||||
// try to find the original file we were looking for
|
||||
val foundFile = findFile(name)
|
||||
if (foundFile?.name == name) return foundFile
|
||||
}
|
||||
throw IOException("Wanted to create $name, but got ${file.name}")
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a directory already exists and if not, creates it.
|
||||
*/
|
||||
@Throws(IOException::class)
|
||||
public suspend fun DocumentFile.getOrCreateDirectory(context: Context, name: String): DocumentFile {
|
||||
return findFileBlocking(context, name) ?: createDirectoryOrThrow(name)
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
public fun DocumentFile.createDirectoryOrThrow(name: String): DocumentFile {
|
||||
val directory = createDirectory(name)
|
||||
?: throw IOException("Unable to create directory: $name")
|
||||
if (directory.name != name) {
|
||||
directory.delete()
|
||||
throw IOException("Wanted to directory $name, but got ${directory.name}")
|
||||
}
|
||||
return directory
|
||||
}
|
||||
|
||||
/**
|
||||
* Works like [DocumentFile.listFiles] except
|
||||
* that it waits until the DocumentProvider has a result.
|
||||
* This prevents getting an empty list even though there are children to be listed.
|
||||
*/
|
||||
@Throws(IOException::class)
|
||||
public suspend fun DocumentFile.listFilesBlocking(context: Context): List<DocumentFile> {
|
||||
val resolver = context.contentResolver
|
||||
val childrenUri = buildChildDocumentsUriUsingTree(uri, getDocumentId(uri))
|
||||
val projection = arrayOf(COLUMN_DOCUMENT_ID)
|
||||
val result = ArrayList<DocumentFile>()
|
||||
|
||||
try {
|
||||
getLoadedCursor {
|
||||
resolver.query(childrenUri, projection, null, null, null)
|
||||
}
|
||||
} catch (e: TimeoutCancellationException) {
|
||||
throw IOException(e)
|
||||
}.use { cursor ->
|
||||
while (cursor.moveToNext()) {
|
||||
val documentId = cursor.getString(0)
|
||||
val documentUri = buildDocumentUriUsingTree(uri, documentId)
|
||||
result.add(getTreeDocumentFile(this, context, documentUri))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* An extremely dirty reflection hack to instantiate a TreeDocumentFile with a parent.
|
||||
*
|
||||
* All other public ways to get a TreeDocumentFile only work from [Uri]s
|
||||
* (e.g. [DocumentFile.fromTreeUri]) and always set parent to null.
|
||||
*
|
||||
* We have a test for this method to ensure CI will alert us when this reflection breaks.
|
||||
* Also, [DocumentFile] is part of AndroidX, so we control the dependency and notice when it fails.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
@SuppressLint("CheckedExceptions")
|
||||
public fun getTreeDocumentFile(
|
||||
parent: DocumentFile,
|
||||
context: Context,
|
||||
uri: Uri,
|
||||
): DocumentFile {
|
||||
@SuppressWarnings("MagicNumber")
|
||||
val constructor = parent.javaClass.declaredConstructors.find {
|
||||
it.name == "androidx.documentfile.provider.TreeDocumentFile" && it.parameterCount == 3
|
||||
}
|
||||
check(constructor != null) { "Could not find constructor for TreeDocumentFile" }
|
||||
constructor.isAccessible = true
|
||||
return constructor.newInstance(parent, context, uri) as DocumentFile
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as [DocumentFile.findFile] only that it re-queries when the first result was stale.
|
||||
*
|
||||
* Most documents providers including Nextcloud are listing the full directory content
|
||||
* when querying for a specific file in a directory,
|
||||
* so there is no point in trying to optimize the query by not listing all children.
|
||||
*/
|
||||
public suspend fun DocumentFile.findFileBlocking(
|
||||
context: Context,
|
||||
displayName: String,
|
||||
): DocumentFile? {
|
||||
val files = try {
|
||||
listFilesBlocking(context)
|
||||
} catch (e: IOException) {
|
||||
Log.e(TAG, "Error finding file blocking", e)
|
||||
return null
|
||||
}
|
||||
for (doc in files) {
|
||||
if (displayName == doc.name) return doc
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a cursor for the given query while ensuring that the cursor was loaded.
|
||||
*
|
||||
* When the SAF backend is a cloud storage provider (e.g. Nextcloud),
|
||||
* it can happen that the query returns an outdated (e.g. empty) cursor
|
||||
* which will only be updated in response to this query.
|
||||
*
|
||||
* See: https://commonsware.com/blog/2019/12/14/scoped-storage-stories-listfiles-woe.html
|
||||
*
|
||||
* This method uses a [suspendCancellableCoroutine] to wait for the result of a [ContentObserver]
|
||||
* registered on the cursor in case the cursor is still loading ([EXTRA_LOADING]).
|
||||
* If the cursor is not loading, it will be returned right away.
|
||||
*
|
||||
* @param timeout an optional time-out in milliseconds
|
||||
* @throws TimeoutCancellationException if there was no result before the time-out
|
||||
* @throws IOException if the query returns null
|
||||
*/
|
||||
@VisibleForTesting
|
||||
@Throws(IOException::class, TimeoutCancellationException::class)
|
||||
public suspend fun getLoadedCursor(timeout: Long = 15_000, query: () -> Cursor?): Cursor =
|
||||
withTimeout(timeout) {
|
||||
suspendCancellableCoroutine { cont ->
|
||||
val cursor = query() ?: throw IOException()
|
||||
cont.invokeOnCancellation { cursor.close() }
|
||||
val loading = cursor.extras.getBoolean(EXTRA_LOADING, false)
|
||||
if (loading) {
|
||||
Log.d(TAG, "Wait for children to get loaded...")
|
||||
cursor.registerContentObserver(object : ContentObserver(null) {
|
||||
override fun onChange(selfChange: Boolean, uri: Uri?) {
|
||||
Log.d(TAG, "Children loaded. Continue...")
|
||||
cursor.close()
|
||||
val newCursor = query()
|
||||
if (newCursor == null) {
|
||||
cont.cancel(IOException("query returned no results"))
|
||||
} else cont.resume(newCursor)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
// not loading, return cursor right away
|
||||
cont.resume(cursor)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2021 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.saf
|
||||
|
||||
import android.content.ContentResolver
|
||||
import android.net.Uri
|
||||
import android.provider.MediaStore
|
||||
import java.io.IOException
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
|
||||
public fun Uri.getDocumentPath(): String? {
|
||||
return lastPathSegment?.split(':')?.getOrNull(1)
|
||||
}
|
||||
|
||||
public fun Uri.getVolume(): String? {
|
||||
val volume = lastPathSegment?.split(':')?.getOrNull(0)
|
||||
return if (volume == "primary") MediaStore.VOLUME_EXTERNAL_PRIMARY else volume
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
public fun Uri.openInputStream(contentResolver: ContentResolver): InputStream {
|
||||
return try {
|
||||
contentResolver.openInputStream(this)
|
||||
} catch (e: IllegalArgumentException) {
|
||||
// This is necessary, because contrary to the documentation, files that have been deleted
|
||||
// after we retrieved their Uri, will throw an IllegalArgumentException
|
||||
throw IOException(e)
|
||||
} ?: throw IOException("Stream for $this returned null")
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
public fun Uri.openOutputStream(contentResolver: ContentResolver): OutputStream {
|
||||
return try {
|
||||
contentResolver.openOutputStream(this, "wt")
|
||||
} catch (e: IllegalArgumentException) {
|
||||
// This is necessary, because contrary to the documentation, files that have been deleted
|
||||
// after we retrieved their Uri, will throw an IllegalArgumentException
|
||||
throw IOException(e)
|
||||
} ?: throw IOException("Stream for $this returned null")
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
import at.bitfire.dav4jvm.Property
|
||||
import at.bitfire.dav4jvm.PropertyFactory
|
||||
import at.bitfire.dav4jvm.property.webdav.NS_WEBDAV
|
||||
import org.xmlpull.v1.XmlPullParser
|
||||
|
||||
/**
|
||||
* A fake version of [at.bitfire.dav4jvm.property.webdav.GetLastModified] which we register
|
||||
* so we don't need to depend on `org.apache.commons.lang3` which is used for date parsing.
|
||||
*/
|
||||
internal class GetLastModified : Property {
|
||||
companion object {
|
||||
@JvmField
|
||||
val NAME = Property.Name(NS_WEBDAV, "getlastmodified")
|
||||
}
|
||||
|
||||
object Factory : PropertyFactory {
|
||||
override fun getName() = NAME
|
||||
override fun create(parser: XmlPullParser): GetLastModified? = null
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
import java.io.IOException
|
||||
import java.io.PipedInputStream
|
||||
import java.io.PipedOutputStream
|
||||
|
||||
internal class PipedCloseActionOutputStream(
|
||||
inputStream: PipedInputStream,
|
||||
) : PipedOutputStream(inputStream) {
|
||||
|
||||
private var onClose: (() -> Unit)? = null
|
||||
|
||||
override fun write(b: Int) {
|
||||
try {
|
||||
super.write(b)
|
||||
} catch (e: Exception) {
|
||||
try {
|
||||
onClose?.invoke()
|
||||
} catch (closeException: Exception) {
|
||||
e.addSuppressed(closeException)
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
override fun write(b: ByteArray, off: Int, len: Int) {
|
||||
try {
|
||||
super.write(b, off, len)
|
||||
} catch (e: Exception) {
|
||||
try {
|
||||
onClose?.invoke()
|
||||
} catch (closeException: Exception) {
|
||||
e.addSuppressed(closeException)
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
override fun close() {
|
||||
super.close()
|
||||
try {
|
||||
onClose?.invoke()
|
||||
} catch (e: Exception) {
|
||||
if (e is IOException) throw e
|
||||
else throw IOException(e)
|
||||
}
|
||||
}
|
||||
|
||||
fun doOnClose(function: () -> Unit) {
|
||||
this.onClose = function
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,281 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
import at.bitfire.dav4jvm.BasicDigestAuthHandler
|
||||
import at.bitfire.dav4jvm.DavCollection
|
||||
import at.bitfire.dav4jvm.PropertyRegistry
|
||||
import at.bitfire.dav4jvm.Response.HrefRelation.SELF
|
||||
import at.bitfire.dav4jvm.exception.HttpException
|
||||
import at.bitfire.dav4jvm.exception.NotFoundException
|
||||
import io.github.oshai.kotlinlogging.KLogger
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import kotlinx.coroutines.DelicateCoroutinesApi
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.GlobalScope
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import okhttp3.ConnectionSpec
|
||||
import okhttp3.HttpUrl
|
||||
import okhttp3.HttpUrl.Companion.toHttpUrl
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.RequestBody
|
||||
import okio.BufferedSink
|
||||
import okio.BufferedSource
|
||||
import okio.buffer
|
||||
import okio.sink
|
||||
import org.calyxos.seedvault.core.backends.Backend
|
||||
import org.calyxos.seedvault.core.backends.Constants.DIRECTORY_ROOT
|
||||
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA
|
||||
import org.calyxos.seedvault.core.backends.Constants.chunkFolderRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.chunkRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.folderRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.snapshotRegex
|
||||
import org.calyxos.seedvault.core.backends.Constants.tokenRegex
|
||||
import org.calyxos.seedvault.core.backends.FileBackupFileType
|
||||
import org.calyxos.seedvault.core.backends.FileHandle
|
||||
import org.calyxos.seedvault.core.backends.FileInfo
|
||||
import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
|
||||
import org.calyxos.seedvault.core.backends.TopLevelFolder
|
||||
import java.io.IOException
|
||||
import java.io.PipedInputStream
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.suspendCoroutine
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
private const val DEBUG_LOG = true
|
||||
|
||||
@OptIn(DelicateCoroutinesApi::class)
|
||||
internal class WebDavBackend(
|
||||
webDavConfig: WebDavConfig,
|
||||
root: String = DIRECTORY_ROOT,
|
||||
) : Backend {
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
private val authHandler = BasicDigestAuthHandler(
|
||||
domain = null, // Optional, to only authenticate against hosts with this domain.
|
||||
username = webDavConfig.username,
|
||||
password = webDavConfig.password,
|
||||
)
|
||||
private val okHttpClient = OkHttpClient.Builder()
|
||||
.followRedirects(false)
|
||||
.authenticator(authHandler)
|
||||
.addNetworkInterceptor(authHandler)
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.writeTimeout(60, TimeUnit.SECONDS)
|
||||
.readTimeout(240, TimeUnit.SECONDS)
|
||||
.pingInterval(45, TimeUnit.SECONDS)
|
||||
.connectionSpecs(listOf(ConnectionSpec.MODERN_TLS))
|
||||
.retryOnConnectionFailure(true)
|
||||
.build()
|
||||
|
||||
private val url = "${webDavConfig.url}/$root"
|
||||
private val folders = mutableSetOf<HttpUrl>() // cache for existing/created folders
|
||||
|
||||
init {
|
||||
PropertyRegistry.register(GetLastModified.Factory)
|
||||
}
|
||||
|
||||
override suspend fun save(handle: FileHandle): BufferedSink {
|
||||
val location = handle.toHttpUrl()
|
||||
val davCollection = DavCollection(okHttpClient, location)
|
||||
davCollection.ensureFoldersExist(log, folders)
|
||||
|
||||
val pipedInputStream = PipedInputStream()
|
||||
val pipedOutputStream = PipedCloseActionOutputStream(pipedInputStream)
|
||||
|
||||
val body = object : RequestBody() {
|
||||
override fun isOneShot(): Boolean = true
|
||||
override fun contentType() = "application/octet-stream".toMediaType()
|
||||
override fun writeTo(sink: BufferedSink) {
|
||||
pipedInputStream.use { inputStream ->
|
||||
sink.outputStream().use { outputStream ->
|
||||
inputStream.copyTo(outputStream)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val deferred = GlobalScope.async(Dispatchers.IO) {
|
||||
davCollection.put(body) { response ->
|
||||
log.debugLog { "save($location) = $response" }
|
||||
}
|
||||
}
|
||||
pipedOutputStream.doOnClose {
|
||||
runBlocking { // blocking i/o wait
|
||||
deferred.await()
|
||||
}
|
||||
}
|
||||
return pipedOutputStream.sink().buffer()
|
||||
}
|
||||
|
||||
override suspend fun load(handle: FileHandle): BufferedSource {
|
||||
val location = handle.toHttpUrl()
|
||||
val davCollection = DavCollection(okHttpClient, location)
|
||||
|
||||
val response = davCollection.get(accept = "", headers = null)
|
||||
log.debugLog { "load($location) = $response" }
|
||||
if (response.code / 100 != 2) throw IOException("HTTP error ${response.code}")
|
||||
return response.body?.source() ?: throw IOException()
|
||||
}
|
||||
|
||||
override suspend fun list(
|
||||
topLevelFolder: TopLevelFolder?,
|
||||
vararg fileTypes: KClass<out FileHandle>,
|
||||
callback: (FileInfo) -> Unit,
|
||||
) {
|
||||
if (TopLevelFolder::class in fileTypes) throw UnsupportedOperationException()
|
||||
if (LegacyAppBackupFile::class in fileTypes) throw UnsupportedOperationException()
|
||||
if (LegacyAppBackupFile.IconsFile::class in fileTypes) throw UnsupportedOperationException()
|
||||
if (LegacyAppBackupFile.Blob::class in fileTypes) throw UnsupportedOperationException()
|
||||
|
||||
// limit depth based on wanted types and if top-level folder is given
|
||||
var depth = if (FileBackupFileType.Blob::class in fileTypes) 3 else 2
|
||||
if (topLevelFolder != null) depth -= 1
|
||||
|
||||
val location = if (topLevelFolder == null) {
|
||||
"$url/".toHttpUrl()
|
||||
} else {
|
||||
"$url/${topLevelFolder.name}/".toHttpUrl()
|
||||
}
|
||||
val davCollection = DavCollection(okHttpClient, location)
|
||||
val tokenFolders = mutableSetOf<HttpUrl>()
|
||||
davCollection.propfindDepthInfinity(depth) { response, relation ->
|
||||
log.debugLog { "list() = $response" }
|
||||
|
||||
// work around nginx's inability to find files starting with .
|
||||
if (relation != SELF && LegacyAppBackupFile.Metadata::class in fileTypes &&
|
||||
response.isFolder() && response.hrefName().matches(tokenRegex)
|
||||
) {
|
||||
tokenFolders.add(response.href)
|
||||
}
|
||||
if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2) {
|
||||
val name = response.hrefName()
|
||||
val parentName = response.href.pathSegments[response.href.pathSegments.size - 2]
|
||||
|
||||
if (LegacyAppBackupFile.Metadata::class in fileTypes) {
|
||||
if (name == FILE_BACKUP_METADATA && parentName.matches(tokenRegex)) {
|
||||
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
|
||||
val size = response.properties.contentLength()
|
||||
callback(FileInfo(metadata, size))
|
||||
// we can find .backup.metadata files, so no need for nginx workaround
|
||||
tokenFolders.clear()
|
||||
}
|
||||
}
|
||||
if (FileBackupFileType.Snapshot::class in fileTypes ||
|
||||
FileBackupFileType::class in fileTypes
|
||||
) {
|
||||
val match = snapshotRegex.matchEntire(name)
|
||||
if (match != null) {
|
||||
val size = response.properties.contentLength()
|
||||
val snapshot = FileBackupFileType.Snapshot(
|
||||
androidId = parentName.substringBefore('.'),
|
||||
time = match.groupValues[1].toLong(),
|
||||
)
|
||||
callback(FileInfo(snapshot, size))
|
||||
}
|
||||
}
|
||||
if ((FileBackupFileType.Blob::class in fileTypes ||
|
||||
FileBackupFileType::class in fileTypes) && response.href.pathSize >= 3
|
||||
) {
|
||||
val androidIdSv =
|
||||
response.href.pathSegments[response.href.pathSegments.size - 3]
|
||||
if (folderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) {
|
||||
if (chunkRegex.matches(name)) {
|
||||
val blob = FileBackupFileType.Blob(
|
||||
androidId = androidIdSv.substringBefore('.'),
|
||||
name = name,
|
||||
)
|
||||
val size = response.properties.contentLength()
|
||||
callback(FileInfo(blob, size))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// direct query for .backup.metadata as nginx doesn't support listing hidden files
|
||||
tokenFolders.forEach { url ->
|
||||
val metadataLocation = url.newBuilder().addPathSegment(FILE_BACKUP_METADATA).build()
|
||||
try {
|
||||
DavCollection(okHttpClient, metadataLocation).head { response ->
|
||||
log.debugLog { "head($metadataLocation) = $response" }
|
||||
val token = url.pathSegments.last { it.isNotBlank() }.toLong()
|
||||
val metadata = LegacyAppBackupFile.Metadata(token)
|
||||
val size = response.headers["content-length"]?.toLong()
|
||||
?: error("no content length")
|
||||
callback(FileInfo(metadata, size))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
log.warn { "No $FILE_BACKUP_METADATA found in $url: $e" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun remove(handle: FileHandle) {
|
||||
val location = handle.toHttpUrl()
|
||||
val davCollection = DavCollection(okHttpClient, location)
|
||||
|
||||
log.debugLog { "remove($handle)" }
|
||||
|
||||
try {
|
||||
val response = suspendCoroutine { cont ->
|
||||
davCollection.delete { response ->
|
||||
cont.resume(response)
|
||||
}
|
||||
}
|
||||
log.debugLog { "remove($location) = $response" }
|
||||
} catch (e: Exception) {
|
||||
when (e) {
|
||||
is NotFoundException -> log.info { "Not found: $location" }
|
||||
is IOException -> throw e
|
||||
else -> throw IOException(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renames [from] to [to].
|
||||
*
|
||||
* @throws HttpException if [to] already exists
|
||||
* * nginx code 412
|
||||
* * lighttp code 207
|
||||
* * dufs code 500
|
||||
*/
|
||||
@Throws(HttpException::class)
|
||||
override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) {
|
||||
val location = "$url/${from.name}/".toHttpUrl()
|
||||
val toUrl = "$url/${to.name}/".toHttpUrl()
|
||||
val davCollection = DavCollection(okHttpClient, location)
|
||||
davCollection.move(toUrl, false) { response ->
|
||||
log.debugLog { "rename(${from.name}, ${to.name}) = $response" }
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun removeAll() {
|
||||
val location = "$url/".toHttpUrl()
|
||||
val davCollection = DavCollection(okHttpClient, location)
|
||||
try {
|
||||
davCollection.delete { response ->
|
||||
log.debugLog { "removeAll() = $response" }
|
||||
}
|
||||
} catch (e: NotFoundException) {
|
||||
log.info { "Not found: $location" }
|
||||
}
|
||||
}
|
||||
|
||||
private fun FileHandle.toHttpUrl(): HttpUrl = when (this) {
|
||||
// careful with trailing slashes, use only for folders/collections
|
||||
is TopLevelFolder -> "$url/$name/".toHttpUrl()
|
||||
else -> "$url/$relativePath".toHttpUrl()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal inline fun KLogger.debugLog(crossinline block: () -> String) {
|
||||
if (DEBUG_LOG) debug { block() }
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
public data class WebDavConfig(
|
||||
val url: String,
|
||||
val username: String,
|
||||
val password: String,
|
||||
)
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
import at.bitfire.dav4jvm.DavCollection
|
||||
import at.bitfire.dav4jvm.MultiResponseCallback
|
||||
import at.bitfire.dav4jvm.Property
|
||||
import at.bitfire.dav4jvm.Response
|
||||
import at.bitfire.dav4jvm.Response.HrefRelation.SELF
|
||||
import at.bitfire.dav4jvm.ResponseCallback
|
||||
import at.bitfire.dav4jvm.exception.ConflictException
|
||||
import at.bitfire.dav4jvm.exception.HttpException
|
||||
import at.bitfire.dav4jvm.exception.NotFoundException
|
||||
import at.bitfire.dav4jvm.property.webdav.DisplayName
|
||||
import at.bitfire.dav4jvm.property.webdav.GetContentLength
|
||||
import at.bitfire.dav4jvm.property.webdav.ResourceType
|
||||
import io.github.oshai.kotlinlogging.KLogger
|
||||
import okhttp3.HttpUrl
|
||||
|
||||
/**
|
||||
* Tries to do [DavCollection.propfind] with a depth of `-1`.
|
||||
* Since `infinity` isn't supported by nginx either,
|
||||
* we fallback to iterating over all folders found with depth `1`
|
||||
* and do another PROPFIND on those, passing the given [callback].
|
||||
*
|
||||
* @param maxDepth in case we need to fallback to recursive propfinds, we only go that far down.
|
||||
*/
|
||||
internal fun DavCollection.propfindDepthInfinity(maxDepth: Int, callback: MultiResponseCallback) {
|
||||
try {
|
||||
propfind(
|
||||
depth = -1,
|
||||
reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME, GetContentLength.NAME),
|
||||
callback = callback,
|
||||
)
|
||||
} catch (e: HttpException) {
|
||||
if (e.isUnsupportedPropfind()) {
|
||||
log.info { "Got ${e.response}, trying recursive depth=1 PROPFINDs..." }
|
||||
propfindFakeInfinity(maxDepth, callback)
|
||||
} else {
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun DavCollection.propfindFakeInfinity(depth: Int, callback: MultiResponseCallback) {
|
||||
if (depth <= 0) return
|
||||
propfind(
|
||||
depth = 1,
|
||||
reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME, GetContentLength.NAME),
|
||||
) { response, relation ->
|
||||
// This callback will be called for everything in the folder
|
||||
callback.onResponse(response, relation)
|
||||
if (relation != SELF && response.isFolder()) {
|
||||
DavCollection(httpClient, response.href).propfindFakeInfinity(depth - 1, callback)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun DavCollection.mkColCreateMissing(callback: ResponseCallback) {
|
||||
try {
|
||||
mkCol(null) { response ->
|
||||
callback.onResponse(response)
|
||||
}
|
||||
} catch (e: ConflictException) {
|
||||
log.warning { "Error creating $location: $e" }
|
||||
if (location.pathSize <= 1) throw e
|
||||
val newLocation = location.newBuilder()
|
||||
.removePathSegment(location.pathSize - 1)
|
||||
.build()
|
||||
DavCollection(httpClient, newLocation).mkColCreateMissing(callback)
|
||||
// re-run original command to create parent collection
|
||||
mkCol(null) { response ->
|
||||
callback.onResponse(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun DavCollection.ensureFoldersExist(log: KLogger, folders: MutableSet<HttpUrl>) {
|
||||
if (location.pathSize <= 2) return
|
||||
val parent = location.newBuilder()
|
||||
.removePathSegment(location.pathSize - 1)
|
||||
.build()
|
||||
if (parent in folders) return
|
||||
val parentCollection = DavCollection(httpClient, parent)
|
||||
try {
|
||||
parentCollection.head { response ->
|
||||
log.debugLog { "head($parent) = $response" }
|
||||
folders.add(parent)
|
||||
}
|
||||
} catch (e: NotFoundException) {
|
||||
log.debugLog { "$parent not found, creating..." }
|
||||
parentCollection.mkColCreateMissing { response ->
|
||||
log.debugLog { "mkColCreateMissing($parent) = $response" }
|
||||
folders.add(parent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun HttpException.isUnsupportedPropfind(): Boolean {
|
||||
// nginx is not including 'propfind-finite-depth' in body, so just relay on code
|
||||
return code == 403 || code == 400 // dufs returns 400
|
||||
}
|
||||
|
||||
internal fun List<Property>.contentLength(): Long {
|
||||
// crash intentionally, if this isn't in the list
|
||||
return filterIsInstance<GetContentLength>()[0].contentLength
|
||||
}
|
||||
|
||||
internal fun Response.isFolder(): Boolean {
|
||||
return this[ResourceType::class.java]?.types?.contains(ResourceType.COLLECTION) == true
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.calyxos.seedvault.core.backends.Backend
|
||||
import org.calyxos.seedvault.core.backends.BackendTest
|
||||
import kotlin.test.Test
|
||||
|
||||
public class WebDavBackendTest : BackendTest() {
|
||||
override val plugin: Backend = WebDavBackend(WebDavTestConfig.getConfig(), ".SeedvaultTest")
|
||||
|
||||
@Test
|
||||
public fun `test write, list, read, rename, delete`(): Unit = runBlocking {
|
||||
testWriteListReadRenameDelete()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package org.calyxos.seedvault.core.backends.webdav
|
||||
|
||||
import org.junit.Assume.assumeFalse
|
||||
import kotlin.test.fail
|
||||
|
||||
internal object WebDavTestConfig {
|
||||
|
||||
fun getConfig(): WebDavConfig {
|
||||
assumeFalse(System.getenv("NEXTCLOUD_URL").isNullOrEmpty())
|
||||
return WebDavConfig(
|
||||
url = System.getenv("NEXTCLOUD_URL") ?: fail(),
|
||||
username = System.getenv("NEXTCLOUD_USER") ?: fail(),
|
||||
password = System.getenv("NEXTCLOUD_PASS") ?: fail(),
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
1
core/src/test/resources/simplelogger.properties
Normal file
1
core/src/test/resources/simplelogger.properties
Normal file
|
|
@ -0,0 +1 @@
|
|||
org.slf4j.simpleLogger.defaultLogLevel=trace
|
||||
|
|
@ -115,5 +115,6 @@ android-library = { id = "com.android.library", version.ref = "androidGradlePlug
|
|||
google-ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" }
|
||||
google-protobuf = { id = "com.google.protobuf", version.ref = "protobufPlugin" }
|
||||
jetbrains-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" }
|
||||
jetbrains-kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" }
|
||||
jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
|
||||
jlleitschuh-ktlint = { id = "org.jlleitschuh.gradle.ktlint", version.ref = "ktlint" }
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ dependencyResolutionManagement {
|
|||
}
|
||||
|
||||
rootProject.name = "Seedvault"
|
||||
include(":core")
|
||||
include(":app")
|
||||
include(":contactsbackup")
|
||||
include(":storage:lib")
|
||||
|
|
|
|||
Loading…
Reference in a new issue