Initial implementation of WebDavStoragePlugin

.github/workflows/build.yml

@@ -35,6 +35,10 @@ jobs:
           cache: 'gradle'
 
       - name: Build
+        env:
+          NEXTCLOUD_URL: ${{ vars.NEXTCLOUD_URL }}
+          NEXTCLOUD_USER: ${{ secrets.NEXTCLOUD_USER }}
+          NEXTCLOUD_PASS: ${{ secrets.NEXTCLOUD_PASS }}
         run: ./gradlew compileDebugAndroidTestSources check assemble ktlintCheck
 
       - name: Upload APKs

Android.bp

@@ -44,6 +44,10 @@ android_app {
         "seedvault-lib-koin-android",
         // bip39
         "seedvault-lib-kotlin-bip39",
+        // WebDAV
+        "seedvault-lib-dav4jvm",
+        "seedvault-lib-okhttp",
+        "seedvault-lib-okio",
     ],
     manifest: "app/src/main/AndroidManifest.xml",
 

app/build.gradle.kts

@@ -159,6 +159,9 @@ dependencies {
 
     implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar"))
 
+    // dav4jvm - later versions of okhttp need kotlin > 1.9.0
+    implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("*.jar"))
+
     /**
      * Test Dependencies (do not concern the AOSP build)
      */

app/src/main/AndroidManifest.xml

@@ -16,6 +16,9 @@
     <!-- This is needed to check for internet access when backup is stored on network storage -->
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 
+    <!-- Used for internal WebDAV plugin -->
+    <uses-permission android:name="android.permission.INTERNET" />
+
     <!-- This is needed to inform users about backup status and errors -->
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
 

app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePlugin.kt

@@ -75,4 +75,7 @@ interface StoragePlugin {
 
 }
 
-class EncryptedMetadata(val token: Long, val inputStreamRetriever: () -> InputStream)
+class EncryptedMetadata(val token: Long, val inputStreamRetriever: suspend () -> InputStream)
+
+internal val tokenRegex = Regex("([0-9]{13})") // good until the year 2286
+internal val chunkFolderRegex = Regex("[a-f0-9]{2}")

app/src/main/java/com/stevesoltys/seedvault/plugins/saf/DocumentsProviderStoragePlugin.kt

@@ -7,6 +7,8 @@ import androidx.documentfile.provider.DocumentFile
 import com.stevesoltys.seedvault.getStorageContext
 import com.stevesoltys.seedvault.plugins.EncryptedMetadata
 import com.stevesoltys.seedvault.plugins.StoragePlugin
+import com.stevesoltys.seedvault.plugins.chunkFolderRegex
+import com.stevesoltys.seedvault.plugins.tokenRegex
 import com.stevesoltys.seedvault.settings.Storage
 import java.io.FileNotFoundException
 import java.io.IOException
@@ -137,9 +139,6 @@ internal suspend fun getBackups(context: Context, rootDir: DocumentFile): List<B
     return backupSets
 }
 
-private val tokenRegex = Regex("([0-9]{13})") // good until the year 2286
-private val chunkFolderRegex = Regex("[a-f0-9]{2}")
-
 private fun DocumentFile.getTokenOrNull(name: String?): Long? {
     val looksLikeToken = name != null && tokenRegex.matches(name)
     // check for isDirectory only if we already have a valid token (causes DB query)

app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavConfig.kt

@@ -0,0 +1,12 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.plugins.webdav
+
+data class WebDavConfig(
+    val url: String,
+    val username: String,
+    val password: String,
+)

app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavModule.kt

@@ -0,0 +1,17 @@
+package com.stevesoltys.seedvault.plugins.webdav
+
+import com.stevesoltys.seedvault.plugins.LegacyStoragePlugin
+import com.stevesoltys.seedvault.plugins.StoragePlugin
+import com.stevesoltys.seedvault.plugins.saf.DocumentsProviderLegacyPlugin
+import com.stevesoltys.seedvault.plugins.saf.DocumentsStorage
+import org.koin.android.ext.koin.androidContext
+import org.koin.dsl.module
+
+val webDavModule = module {
+    // TODO PluginManager should create the plugin on demand
+    single<StoragePlugin> { WebDavStoragePlugin(androidContext(), WebDavConfig("", "", "")) }
+
+    single { DocumentsStorage(androidContext(), get()) }
+    @Suppress("Deprecation")
+    single<LegacyStoragePlugin> { DocumentsProviderLegacyPlugin(androidContext(), get()) }
+}

app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt

@@ -0,0 +1,297 @@
+package com.stevesoltys.seedvault.plugins.webdav
+
+import android.content.Context
+import android.util.Log
+import at.bitfire.dav4jvm.BasicDigestAuthHandler
+import at.bitfire.dav4jvm.DavCollection
+import at.bitfire.dav4jvm.Response.HrefRelation.SELF
+import at.bitfire.dav4jvm.exception.NotFoundException
+import at.bitfire.dav4jvm.property.DisplayName
+import at.bitfire.dav4jvm.property.ResourceType
+import at.bitfire.dav4jvm.property.ResourceType.Companion.COLLECTION
+import com.stevesoltys.seedvault.plugins.EncryptedMetadata
+import com.stevesoltys.seedvault.plugins.StoragePlugin
+import com.stevesoltys.seedvault.plugins.chunkFolderRegex
+import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA
+import com.stevesoltys.seedvault.plugins.saf.FILE_NO_MEDIA
+import com.stevesoltys.seedvault.plugins.tokenRegex
+import com.stevesoltys.seedvault.settings.Storage
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.RequestBody
+import okio.BufferedSink
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine
+
+private val TAG = WebDavStoragePlugin::class.java.simpleName
+const val DEBUG_LOG = true
+const val DIRECTORY_ROOT = ".SeedVaultAndroidBackup"
+
+@OptIn(DelicateCoroutinesApi::class)
+@Suppress("BlockingMethodInNonBlockingContext")
+internal class WebDavStoragePlugin(
+    context: Context,
+    webDavConfig: WebDavConfig,
+) : StoragePlugin {
+
+    private val authHandler = BasicDigestAuthHandler(
+        domain = null, // Optional, to only authenticate against hosts with this domain.
+        username = webDavConfig.username,
+        password = webDavConfig.password,
+    )
+    private val okHttpClient = OkHttpClient.Builder()
+        .followRedirects(false)
+        .authenticator(authHandler)
+        .addNetworkInterceptor(authHandler)
+        .build()
+
+    private val url = "${webDavConfig.url}/$DIRECTORY_ROOT"
+
+    @Throws(IOException::class)
+    override suspend fun startNewRestoreSet(token: Long) {
+        try {
+            val location = "$url/$token".toHttpUrl()
+            val davCollection = DavCollection(okHttpClient, location)
+
+            val response = suspendCoroutine { cont ->
+                davCollection.mkCol(null) { response ->
+                    cont.resume(response)
+                }
+            }
+            debugLog { "startNewRestoreSet($token) = $response" }
+        } catch (e: Exception) {
+            if (e is IOException) throw e
+            else throw IOException(e)
+        }
+    }
+
+    @Throws(IOException::class)
+    override suspend fun initializeDevice() {
+        // TODO does it make sense to delete anything
+        //  when [startNewRestoreSet] is always called first? Maybe unify both calls?
+    }
+
+    @Throws(IOException::class)
+    override suspend fun hasData(token: Long, name: String): Boolean {
+        val location = "$url/$token/$name".toHttpUrl()
+        val davCollection = DavCollection(okHttpClient, location)
+
+        return try {
+            val response = suspendCoroutine { cont ->
+                davCollection.head { response ->
+                    cont.resume(response)
+                }
+            }
+            debugLog { "hasData($token, $name) = $response" }
+            response.isSuccessful
+        } catch (e: NotFoundException) {
+            debugLog { "hasData($token, $name) = $e" }
+            false
+        } catch (e: Exception) {
+            if (e is IOException) throw e
+            else throw IOException(e)
+        }
+    }
+
+    @Throws(IOException::class)
+    override suspend fun getOutputStream(token: Long, name: String): OutputStream {
+        return try {
+            doGetOutputStream(token, name)
+        } catch (e: Exception) {
+            if (e is IOException) throw e
+            else throw IOException("Error getting OutputStream for $token and $name: ", e)
+        }
+    }
+
+    @Throws(IOException::class)
+    private suspend fun doGetOutputStream(token: Long, name: String): OutputStream {
+        val location = "$url/$token/$name".toHttpUrl()
+        val davCollection = DavCollection(okHttpClient, location)
+
+        val pipedInputStream = PipedInputStream()
+        val pipedOutputStream = PipedCloseActionOutputStream(pipedInputStream)
+
+        val body = object : RequestBody() {
+            override fun contentType() = "application/octet-stream".toMediaType()
+            override fun writeTo(sink: BufferedSink) {
+                pipedInputStream.use { inputStream ->
+                    sink.outputStream().use { outputStream ->
+                        inputStream.copyTo(outputStream)
+                    }
+                }
+            }
+        }
+        val deferred = GlobalScope.async(Dispatchers.IO) {
+            davCollection.put(body) { response ->
+                debugLog { "getOutputStream($token, $name) = $response" }
+            }
+        }
+        pipedOutputStream.doOnClose {
+            runBlocking { // blocking i/o wait
+                deferred.await()
+            }
+        }
+        return pipedOutputStream
+    }
+
+    @Throws(IOException::class)
+    override suspend fun getInputStream(token: Long, name: String): InputStream {
+        return try {
+            doGetInputStream(token, name)
+        } catch (e: Exception) {
+            if (e is IOException) throw e
+            else throw IOException("Error getting InputStream for $token and $name: ", e)
+        }
+    }
+
+    @Throws(IOException::class)
+    private fun doGetInputStream(token: Long, name: String): InputStream {
+        val location = "$url/$token/$name".toHttpUrl()
+        val davCollection = DavCollection(okHttpClient, location)
+
+        val pipedInputStream = PipedInputStream()
+        val pipedOutputStream = PipedOutputStream(pipedInputStream)
+
+        GlobalScope.launch(Dispatchers.IO) {
+            davCollection.get(accept = "", headers = null) { response ->
+                val inputStream = response.body?.byteStream()
+                    ?: throw IOException("No response body")
+                debugLog { "getInputStream($token, $name) = $response" }
+                pipedOutputStream.use { outputStream ->
+                    inputStream.copyTo(outputStream)
+                }
+            }
+        }
+        return pipedInputStream
+    }
+
+    @Throws(IOException::class)
+    override suspend fun removeData(token: Long, name: String) {
+        val location = "$url/$token/$name".toHttpUrl()
+        val davCollection = DavCollection(okHttpClient, location)
+
+        try {
+            val response = suspendCoroutine { cont ->
+                davCollection.delete { response ->
+                    cont.resume(response)
+                }
+            }
+            debugLog { "removeData($token, $name) = $response" }
+        } catch (e: Exception) {
+            if (e is IOException) throw e
+            else throw IOException(e)
+        }
+    }
+
+    @Throws(IOException::class)
+    override suspend fun hasBackup(storage: Storage): Boolean {
+        // TODO this requires refactoring
+        return true
+    }
+
+    override suspend fun getAvailableBackups(): Sequence<EncryptedMetadata>? {
+        return try {
+            doGetAvailableBackups()
+        } catch (e: Exception) {
+            Log.e(TAG, "Error getting available backups: ", e)
+            null
+        }
+    }
+
+    private suspend fun doGetAvailableBackups(): Sequence<EncryptedMetadata> {
+        val location = url.toHttpUrl()
+        val davCollection = DavCollection(okHttpClient, location)
+
+        // get all restore set tokens in root folder
+        val tokens = ArrayList<Long>()
+        davCollection.propfind(
+            depth = 2,
+            reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME),
+        ) { response, relation ->
+            debugLog { "getAvailableBackups() = $response" }
+            // This callback will be called for every file in the folder
+            if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2 &&
+                response.hrefName() == FILE_BACKUP_METADATA
+            ) {
+                val tokenName = response.href.pathSegments[response.href.pathSegments.size - 2]
+                getTokenOrNull(tokenName)?.let { token ->
+                    tokens.add(token)
+                }
+            }
+        }
+        val tokenIterator = tokens.iterator()
+        return generateSequence {
+            if (!tokenIterator.hasNext()) return@generateSequence null // end sequence
+            val token = tokenIterator.next()
+            EncryptedMetadata(token) {
+                getInputStream(token, FILE_BACKUP_METADATA)
+            }
+        }
+    }
+
+    private fun getTokenOrNull(name: String): Long? {
+        val looksLikeToken = name.isNotEmpty() && tokenRegex.matches(name)
+        if (looksLikeToken) {
+            return try {
+                name.toLong()
+            } catch (e: NumberFormatException) {
+                throw AssertionError(e) // regex must be wrong
+            }
+        }
+        if (isUnexpectedFile(name)) {
+            Log.w(TAG, "Found invalid backup set folder: $name")
+        }
+        return null
+    }
+
+    private fun isUnexpectedFile(name: String): Boolean {
+        return name != FILE_NO_MEDIA &&
+            !chunkFolderRegex.matches(name) &&
+            !name.endsWith(".SeedSnap")
+    }
+
+    private fun Response.isFolder(): Boolean {
+        return this[ResourceType::class.java]?.types?.contains(COLLECTION) == true
+    }
+
+    override val providerPackageName: String = context.packageName // 100% built-in plugin
+
+    private class PipedCloseActionOutputStream(
+        inputStream: PipedInputStream,
+    ) : PipedOutputStream(inputStream) {
+
+        private var onClose: (() -> Unit)? = null
+
+        @Throws(IOException::class)
+        override fun close() {
+            super.close()
+            try {
+                onClose?.invoke()
+            } catch (e: Exception) {
+                if (e is IOException) throw e
+                else throw IOException(e)
+            }
+        }
+
+        fun doOnClose(function: () -> Unit) {
+            this.onClose = function
+        }
+    }
+
+    private fun debugLog(block: () -> String) {
+        if (DEBUG_LOG) Log.d(TAG, block())
+    }
+
+}

app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt

@@ -0,0 +1,88 @@
+package com.stevesoltys.seedvault.plugins.webdav
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.stevesoltys.seedvault.TestApp
+import com.stevesoltys.seedvault.getRandomByteArray
+import com.stevesoltys.seedvault.getRandomString
+import com.stevesoltys.seedvault.plugins.EncryptedMetadata
+import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA
+import com.stevesoltys.seedvault.transport.TransportTest
+import kotlinx.coroutines.runBlocking
+import org.junit.Test
+import org.junit.jupiter.api.Assertions.assertArrayEquals
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertFalse
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Assertions.fail
+import org.junit.jupiter.api.assertThrows
+import org.junit.runner.RunWith
+import org.robolectric.annotation.Config
+import java.io.IOException
+import kotlin.random.Random
+
+@RunWith(AndroidJUnit4::class)
+@Config(
+    sdk = [33], // robolectric does not support 34, yet
+    application = TestApp::class
+)
+internal class WebDavStoragePluginTest : TransportTest() {
+
+    private val plugin = WebDavStoragePlugin(context, WebDavTestConfig.getConfig())
+
+    @Test
+    fun `test restore sets and reading+writing`() = runBlocking {
+        val token = System.currentTimeMillis()
+        val metadata = getRandomByteArray()
+
+        // initially, we don't have any backups
+        assertEquals(emptySet<EncryptedMetadata>(), plugin.getAvailableBackups()?.toSet())
+
+        // and no data
+        assertFalse(plugin.hasData(token, FILE_BACKUP_METADATA))
+
+        // start a new restore set, initialize it and write out the metadata file
+        plugin.startNewRestoreSet(token)
+        plugin.initializeDevice()
+        plugin.getOutputStream(token, FILE_BACKUP_METADATA).use {
+            it.write(metadata)
+        }
+        try {
+            // now we have one backup matching our token
+            val backups = plugin.getAvailableBackups()?.toSet() ?: fail()
+            assertEquals(1, backups.size)
+            assertEquals(token, backups.first().token)
+
+            // read back written data
+            assertArrayEquals(
+                metadata,
+                plugin.getInputStream(token, FILE_BACKUP_METADATA).use { it.readAllBytes() },
+            )
+
+            // it has data now
+            assertTrue(plugin.hasData(token, FILE_BACKUP_METADATA))
+        } finally {
+            // remove data at the end, so consecutive test runs pass
+            plugin.removeData(token, FILE_BACKUP_METADATA)
+        }
+    }
+
+    @Test
+    fun `test streams for non-existent data`() = runBlocking {
+        val token = Random.nextLong(System.currentTimeMillis(), 9999999999999)
+        val file = getRandomString()
+
+        assertFalse(plugin.hasData(token, file))
+
+        assertThrows<IOException> {
+            plugin.getOutputStream(token, file).use { it.write(getRandomByteArray()) }
+        }
+
+        assertThrows<IOException> {
+            plugin.getInputStream(token, file).use {
+                it.readAllBytes()
+            }
+        }
+        Unit
+    }
+
+}

app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavTestConfig.kt

@@ -0,0 +1,22 @@
+/*
+ * SPDX-FileCopyrightText: 2024 The Calyx Institute
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.stevesoltys.seedvault.plugins.webdav
+
+import org.junit.Assume.assumeFalse
+import org.junit.jupiter.api.Assertions.fail
+
+object WebDavTestConfig {
+
+    fun getConfig(): WebDavConfig {
+        assumeFalse(System.getenv("NEXTCLOUD_URL").isNullOrEmpty())
+        return WebDavConfig(
+            url = System.getenv("NEXTCLOUD_URL") ?: fail(),
+            username = System.getenv("NEXTCLOUD_USER") ?: fail(),
+            password = System.getenv("NEXTCLOUD_PASS") ?: fail(),
+        )
+    }
+
+}

app/src/test/java/com/stevesoltys/seedvault/transport/TransportTest.kt

@@ -22,6 +22,7 @@ import com.stevesoltys.seedvault.settings.SettingsManager
 import io.mockk.every
 import io.mockk.mockk
 import io.mockk.mockkStatic
+import io.mockk.slot
 import org.junit.jupiter.api.TestInstance
 import org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD
 import kotlin.random.Random
@@ -70,8 +71,14 @@ internal abstract class TransportTest {
 
     init {
         mockkStatic(Log::class)
+        val logTagSlot = slot<String>()
+        val logMsgSlot = slot<String>()
         every { Log.v(any(), any()) } returns 0
-        every { Log.d(any(), any()) } returns 0
+        every { Log.d(capture(logTagSlot), capture(logMsgSlot)) } answers {
+            println("${logTagSlot.captured} - ${logMsgSlot.captured}")
+            0
+        }
+        every { Log.d(any(), any(), any()) } returns 0
         every { Log.i(any(), any()) } returns 0
         every { Log.w(any(), ofType(String::class)) } returns 0
         every { Log.w(any(), ofType(String::class), any()) } returns 0

libs/dav4jvm/Android.bp

@@ -0,0 +1,17 @@
+java_import {
+    name: "seedvault-lib-dav4jvm",
+    jars: ["dav4jvm-2.2.1.jar"],
+    sdk_version: "current",
+}
+
+java_import {
+    name: "seedvault-lib-okhttp",
+    jars: ["okhttp-4.11.0.jar"],
+    sdk_version: "current",
+}
+
+java_import {
+    name: "seedvault-lib-okio",
+    jars: ["okio-jvm-3.7.0.jar"],
+    sdk_version: "current",
+}