diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f2e5486..347c612c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,10 @@ jobs: cache: 'gradle' - name: Build + env: + NEXTCLOUD_URL: ${{ vars.NEXTCLOUD_URL }} + NEXTCLOUD_USER: ${{ secrets.NEXTCLOUD_USER }} + NEXTCLOUD_PASS: ${{ secrets.NEXTCLOUD_PASS }} run: ./gradlew compileDebugAndroidTestSources check assemble ktlintCheck - name: Upload APKs diff --git a/Android.bp b/Android.bp index d61e977c..cdf2e5ac 100644 --- a/Android.bp +++ b/Android.bp @@ -44,6 +44,10 @@ android_app { "seedvault-lib-koin-android", // bip39 "seedvault-lib-kotlin-bip39", + // WebDAV + "seedvault-lib-dav4jvm", + "seedvault-lib-okhttp", + "seedvault-lib-okio", ], manifest: "app/src/main/AndroidManifest.xml", diff --git a/app/build.gradle.kts b/app/build.gradle.kts index ec43fa21..215f0a68 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -159,6 +159,9 @@ dependencies { implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar")) + // dav4jvm - later versions of okhttp need kotlin > 1.9.0 + implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("*.jar")) + /** * Test Dependencies (do not concern the AOSP build) */ diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f2278814..da3bafa2 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -16,6 +16,9 @@ + + + diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePlugin.kt index 53becfac..e052bf7a 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePlugin.kt @@ -75,4 +75,7 @@ interface StoragePlugin { } -class EncryptedMetadata(val token: Long, val inputStreamRetriever: () -> InputStream) +class EncryptedMetadata(val token: Long, val inputStreamRetriever: suspend () -> InputStream) + +internal val tokenRegex = Regex("([0-9]{13})") // good until the year 2286 +internal val chunkFolderRegex = Regex("[a-f0-9]{2}") diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/saf/DocumentsProviderStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/saf/DocumentsProviderStoragePlugin.kt index e8e02baa..5f2f3510 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/saf/DocumentsProviderStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/saf/DocumentsProviderStoragePlugin.kt @@ -7,6 +7,8 @@ import androidx.documentfile.provider.DocumentFile import com.stevesoltys.seedvault.getStorageContext import com.stevesoltys.seedvault.plugins.EncryptedMetadata import com.stevesoltys.seedvault.plugins.StoragePlugin +import com.stevesoltys.seedvault.plugins.chunkFolderRegex +import com.stevesoltys.seedvault.plugins.tokenRegex import com.stevesoltys.seedvault.settings.Storage import java.io.FileNotFoundException import java.io.IOException @@ -137,9 +139,6 @@ internal suspend fun getBackups(context: Context, rootDir: DocumentFile): List { WebDavStoragePlugin(androidContext(), WebDavConfig("", "", "")) } + + single { DocumentsStorage(androidContext(), get()) } + @Suppress("Deprecation") + single { DocumentsProviderLegacyPlugin(androidContext(), get()) } +} diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt new file mode 100644 index 00000000..7a1391fc --- /dev/null +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt @@ -0,0 +1,297 @@ +package com.stevesoltys.seedvault.plugins.webdav + +import android.content.Context +import android.util.Log +import at.bitfire.dav4jvm.BasicDigestAuthHandler +import at.bitfire.dav4jvm.DavCollection +import at.bitfire.dav4jvm.Response.HrefRelation.SELF +import at.bitfire.dav4jvm.exception.NotFoundException +import at.bitfire.dav4jvm.property.DisplayName +import at.bitfire.dav4jvm.property.ResourceType +import at.bitfire.dav4jvm.property.ResourceType.Companion.COLLECTION +import com.stevesoltys.seedvault.plugins.EncryptedMetadata +import com.stevesoltys.seedvault.plugins.StoragePlugin +import com.stevesoltys.seedvault.plugins.chunkFolderRegex +import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA +import com.stevesoltys.seedvault.plugins.saf.FILE_NO_MEDIA +import com.stevesoltys.seedvault.plugins.tokenRegex +import com.stevesoltys.seedvault.settings.Storage +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.RequestBody +import okio.BufferedSink +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.io.PipedInputStream +import java.io.PipedOutputStream +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine + +private val TAG = WebDavStoragePlugin::class.java.simpleName +const val DEBUG_LOG = true +const val DIRECTORY_ROOT = ".SeedVaultAndroidBackup" + +@OptIn(DelicateCoroutinesApi::class) +@Suppress("BlockingMethodInNonBlockingContext") +internal class WebDavStoragePlugin( + context: Context, + webDavConfig: WebDavConfig, +) : StoragePlugin { + + private val authHandler = BasicDigestAuthHandler( + domain = null, // Optional, to only authenticate against hosts with this domain. + username = webDavConfig.username, + password = webDavConfig.password, + ) + private val okHttpClient = OkHttpClient.Builder() + .followRedirects(false) + .authenticator(authHandler) + .addNetworkInterceptor(authHandler) + .build() + + private val url = "${webDavConfig.url}/$DIRECTORY_ROOT" + + @Throws(IOException::class) + override suspend fun startNewRestoreSet(token: Long) { + try { + val location = "$url/$token".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + val response = suspendCoroutine { cont -> + davCollection.mkCol(null) { response -> + cont.resume(response) + } + } + debugLog { "startNewRestoreSet($token) = $response" } + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } + } + + @Throws(IOException::class) + override suspend fun initializeDevice() { + // TODO does it make sense to delete anything + // when [startNewRestoreSet] is always called first? Maybe unify both calls? + } + + @Throws(IOException::class) + override suspend fun hasData(token: Long, name: String): Boolean { + val location = "$url/$token/$name".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + return try { + val response = suspendCoroutine { cont -> + davCollection.head { response -> + cont.resume(response) + } + } + debugLog { "hasData($token, $name) = $response" } + response.isSuccessful + } catch (e: NotFoundException) { + debugLog { "hasData($token, $name) = $e" } + false + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } + } + + @Throws(IOException::class) + override suspend fun getOutputStream(token: Long, name: String): OutputStream { + return try { + doGetOutputStream(token, name) + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException("Error getting OutputStream for $token and $name: ", e) + } + } + + @Throws(IOException::class) + private suspend fun doGetOutputStream(token: Long, name: String): OutputStream { + val location = "$url/$token/$name".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + val pipedInputStream = PipedInputStream() + val pipedOutputStream = PipedCloseActionOutputStream(pipedInputStream) + + val body = object : RequestBody() { + override fun contentType() = "application/octet-stream".toMediaType() + override fun writeTo(sink: BufferedSink) { + pipedInputStream.use { inputStream -> + sink.outputStream().use { outputStream -> + inputStream.copyTo(outputStream) + } + } + } + } + val deferred = GlobalScope.async(Dispatchers.IO) { + davCollection.put(body) { response -> + debugLog { "getOutputStream($token, $name) = $response" } + } + } + pipedOutputStream.doOnClose { + runBlocking { // blocking i/o wait + deferred.await() + } + } + return pipedOutputStream + } + + @Throws(IOException::class) + override suspend fun getInputStream(token: Long, name: String): InputStream { + return try { + doGetInputStream(token, name) + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException("Error getting InputStream for $token and $name: ", e) + } + } + + @Throws(IOException::class) + private fun doGetInputStream(token: Long, name: String): InputStream { + val location = "$url/$token/$name".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + val pipedInputStream = PipedInputStream() + val pipedOutputStream = PipedOutputStream(pipedInputStream) + + GlobalScope.launch(Dispatchers.IO) { + davCollection.get(accept = "", headers = null) { response -> + val inputStream = response.body?.byteStream() + ?: throw IOException("No response body") + debugLog { "getInputStream($token, $name) = $response" } + pipedOutputStream.use { outputStream -> + inputStream.copyTo(outputStream) + } + } + } + return pipedInputStream + } + + @Throws(IOException::class) + override suspend fun removeData(token: Long, name: String) { + val location = "$url/$token/$name".toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + try { + val response = suspendCoroutine { cont -> + davCollection.delete { response -> + cont.resume(response) + } + } + debugLog { "removeData($token, $name) = $response" } + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } + } + + @Throws(IOException::class) + override suspend fun hasBackup(storage: Storage): Boolean { + // TODO this requires refactoring + return true + } + + override suspend fun getAvailableBackups(): Sequence? { + return try { + doGetAvailableBackups() + } catch (e: Exception) { + Log.e(TAG, "Error getting available backups: ", e) + null + } + } + + private suspend fun doGetAvailableBackups(): Sequence { + val location = url.toHttpUrl() + val davCollection = DavCollection(okHttpClient, location) + + // get all restore set tokens in root folder + val tokens = ArrayList() + davCollection.propfind( + depth = 2, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), + ) { response, relation -> + debugLog { "getAvailableBackups() = $response" } + // This callback will be called for every file in the folder + if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2 && + response.hrefName() == FILE_BACKUP_METADATA + ) { + val tokenName = response.href.pathSegments[response.href.pathSegments.size - 2] + getTokenOrNull(tokenName)?.let { token -> + tokens.add(token) + } + } + } + val tokenIterator = tokens.iterator() + return generateSequence { + if (!tokenIterator.hasNext()) return@generateSequence null // end sequence + val token = tokenIterator.next() + EncryptedMetadata(token) { + getInputStream(token, FILE_BACKUP_METADATA) + } + } + } + + private fun getTokenOrNull(name: String): Long? { + val looksLikeToken = name.isNotEmpty() && tokenRegex.matches(name) + if (looksLikeToken) { + return try { + name.toLong() + } catch (e: NumberFormatException) { + throw AssertionError(e) // regex must be wrong + } + } + if (isUnexpectedFile(name)) { + Log.w(TAG, "Found invalid backup set folder: $name") + } + return null + } + + private fun isUnexpectedFile(name: String): Boolean { + return name != FILE_NO_MEDIA && + !chunkFolderRegex.matches(name) && + !name.endsWith(".SeedSnap") + } + + private fun Response.isFolder(): Boolean { + return this[ResourceType::class.java]?.types?.contains(COLLECTION) == true + } + + override val providerPackageName: String = context.packageName // 100% built-in plugin + + private class PipedCloseActionOutputStream( + inputStream: PipedInputStream, + ) : PipedOutputStream(inputStream) { + + private var onClose: (() -> Unit)? = null + + @Throws(IOException::class) + override fun close() { + super.close() + try { + onClose?.invoke() + } catch (e: Exception) { + if (e is IOException) throw e + else throw IOException(e) + } + } + + fun doOnClose(function: () -> Unit) { + this.onClose = function + } + } + + private fun debugLog(block: () -> String) { + if (DEBUG_LOG) Log.d(TAG, block()) + } + +} diff --git a/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt new file mode 100644 index 00000000..47578ea2 --- /dev/null +++ b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt @@ -0,0 +1,88 @@ +package com.stevesoltys.seedvault.plugins.webdav + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.stevesoltys.seedvault.TestApp +import com.stevesoltys.seedvault.getRandomByteArray +import com.stevesoltys.seedvault.getRandomString +import com.stevesoltys.seedvault.plugins.EncryptedMetadata +import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA +import com.stevesoltys.seedvault.transport.TransportTest +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.assertThrows +import org.junit.runner.RunWith +import org.robolectric.annotation.Config +import java.io.IOException +import kotlin.random.Random + +@RunWith(AndroidJUnit4::class) +@Config( + sdk = [33], // robolectric does not support 34, yet + application = TestApp::class +) +internal class WebDavStoragePluginTest : TransportTest() { + + private val plugin = WebDavStoragePlugin(context, WebDavTestConfig.getConfig()) + + @Test + fun `test restore sets and reading+writing`() = runBlocking { + val token = System.currentTimeMillis() + val metadata = getRandomByteArray() + + // initially, we don't have any backups + assertEquals(emptySet(), plugin.getAvailableBackups()?.toSet()) + + // and no data + assertFalse(plugin.hasData(token, FILE_BACKUP_METADATA)) + + // start a new restore set, initialize it and write out the metadata file + plugin.startNewRestoreSet(token) + plugin.initializeDevice() + plugin.getOutputStream(token, FILE_BACKUP_METADATA).use { + it.write(metadata) + } + try { + // now we have one backup matching our token + val backups = plugin.getAvailableBackups()?.toSet() ?: fail() + assertEquals(1, backups.size) + assertEquals(token, backups.first().token) + + // read back written data + assertArrayEquals( + metadata, + plugin.getInputStream(token, FILE_BACKUP_METADATA).use { it.readAllBytes() }, + ) + + // it has data now + assertTrue(plugin.hasData(token, FILE_BACKUP_METADATA)) + } finally { + // remove data at the end, so consecutive test runs pass + plugin.removeData(token, FILE_BACKUP_METADATA) + } + } + + @Test + fun `test streams for non-existent data`() = runBlocking { + val token = Random.nextLong(System.currentTimeMillis(), 9999999999999) + val file = getRandomString() + + assertFalse(plugin.hasData(token, file)) + + assertThrows { + plugin.getOutputStream(token, file).use { it.write(getRandomByteArray()) } + } + + assertThrows { + plugin.getInputStream(token, file).use { + it.readAllBytes() + } + } + Unit + } + +} diff --git a/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavTestConfig.kt b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavTestConfig.kt new file mode 100644 index 00000000..e7f5c674 --- /dev/null +++ b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavTestConfig.kt @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2024 The Calyx Institute + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.stevesoltys.seedvault.plugins.webdav + +import org.junit.Assume.assumeFalse +import org.junit.jupiter.api.Assertions.fail + +object WebDavTestConfig { + + fun getConfig(): WebDavConfig { + assumeFalse(System.getenv("NEXTCLOUD_URL").isNullOrEmpty()) + return WebDavConfig( + url = System.getenv("NEXTCLOUD_URL") ?: fail(), + username = System.getenv("NEXTCLOUD_USER") ?: fail(), + password = System.getenv("NEXTCLOUD_PASS") ?: fail(), + ) + } + +} diff --git a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt index 0af1caa2..b5e58569 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt @@ -22,6 +22,7 @@ import com.stevesoltys.seedvault.settings.SettingsManager import io.mockk.every import io.mockk.mockk import io.mockk.mockkStatic +import io.mockk.slot import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD import kotlin.random.Random @@ -70,8 +71,14 @@ internal abstract class TransportTest { init { mockkStatic(Log::class) + val logTagSlot = slot() + val logMsgSlot = slot() every { Log.v(any(), any()) } returns 0 - every { Log.d(any(), any()) } returns 0 + every { Log.d(capture(logTagSlot), capture(logMsgSlot)) } answers { + println("${logTagSlot.captured} - ${logMsgSlot.captured}") + 0 + } + every { Log.d(any(), any(), any()) } returns 0 every { Log.i(any(), any()) } returns 0 every { Log.w(any(), ofType(String::class)) } returns 0 every { Log.w(any(), ofType(String::class), any()) } returns 0 diff --git a/libs/dav4jvm/Android.bp b/libs/dav4jvm/Android.bp new file mode 100644 index 00000000..7a4b82d7 --- /dev/null +++ b/libs/dav4jvm/Android.bp @@ -0,0 +1,17 @@ +java_import { + name: "seedvault-lib-dav4jvm", + jars: ["dav4jvm-2.2.1.jar"], + sdk_version: "current", +} + +java_import { + name: "seedvault-lib-okhttp", + jars: ["okhttp-4.11.0.jar"], + sdk_version: "current", +} + +java_import { + name: "seedvault-lib-okio", + jars: ["okio-jvm-3.7.0.jar"], + sdk_version: "current", +} diff --git a/libs/dav4jvm/dav4jvm-2.2.1.jar b/libs/dav4jvm/dav4jvm-2.2.1.jar new file mode 100644 index 00000000..582827d4 Binary files /dev/null and b/libs/dav4jvm/dav4jvm-2.2.1.jar differ diff --git a/libs/dav4jvm/okhttp-4.11.0.jar b/libs/dav4jvm/okhttp-4.11.0.jar new file mode 100644 index 00000000..2df9400d Binary files /dev/null and b/libs/dav4jvm/okhttp-4.11.0.jar differ diff --git a/libs/dav4jvm/okio-jvm-3.7.0.jar b/libs/dav4jvm/okio-jvm-3.7.0.jar new file mode 100644 index 00000000..8da081a9 Binary files /dev/null and b/libs/dav4jvm/okio-jvm-3.7.0.jar differ