From c522c460fd7c44db7a5841ccb90ca9016300ab53 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 13 Jun 2024 09:32:00 -0300 Subject: [PATCH 1/5] Add trailing slash to WebDAV collection requests --- .../seedvault/plugins/webdav/WebDavStoragePlugin.kt | 10 +++++----- .../seedvault/storage/WebDavStoragePlugin.kt | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt index a2392ba2..b940e393 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt @@ -34,7 +34,7 @@ internal class WebDavStoragePlugin( ) : WebDavStorage(webDavConfig, root), StoragePlugin { override suspend fun test(): Boolean { - val location = baseUrl.toHttpUrl() + val location = (if (baseUrl.endsWith('/')) baseUrl else "$baseUrl/").toHttpUrl() val davCollection = DavCollection(okHttpClient, location) val webDavSupported = suspendCoroutine { cont -> @@ -50,7 +50,7 @@ internal class WebDavStoragePlugin( } override suspend fun getFreeSpace(): Long? { - val location = url.toHttpUrl() + val location = "$url/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) val availableBytes = suspendCoroutine { cont -> @@ -70,7 +70,7 @@ internal class WebDavStoragePlugin( @Throws(IOException::class) override suspend fun startNewRestoreSet(token: Long) { - val location = "$url/$token".toHttpUrl() + val location = "$url/$token/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) val response = davCollection.createFolder() @@ -81,7 +81,7 @@ internal class WebDavStoragePlugin( override suspend fun initializeDevice() { // TODO does it make sense to delete anything // when [startNewRestoreSet] is always called first? Maybe unify both calls? - val location = url.toHttpUrl() + val location = "$url/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) try { @@ -169,7 +169,7 @@ internal class WebDavStoragePlugin( } private suspend fun doGetAvailableBackups(): Sequence { - val location = url.toHttpUrl() + val location = "$url/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) // get all restore set tokens in root folder diff --git a/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt index a8b8d726..e3019868 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt @@ -49,7 +49,7 @@ internal class WebDavStoragePlugin( @Throws(IOException::class) override suspend fun init() { - val location = url.toHttpUrl() + val location = "$url/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) try { @@ -67,7 +67,7 @@ internal class WebDavStoragePlugin( @Throws(IOException::class) override suspend fun getAvailableChunkIds(): List { - val location = "$url/$folder".toHttpUrl() + val location = "$url/$folder/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) debugLog { "getAvailableChunkIds($location)" } @@ -117,7 +117,7 @@ internal class WebDavStoragePlugin( ) { val s = missingChunkFolders.size for ((i, chunkFolderName) in missingChunkFolders.withIndex()) { - val location = "$url/$folder/$chunkFolderName".toHttpUrl() + val location = "$url/$folder/$chunkFolderName/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) val response = davCollection.createFolder() debugLog { "Created missing folder $chunkFolderName (${i + 1}/$s) $response" } @@ -156,7 +156,7 @@ internal class WebDavStoragePlugin( @Throws(IOException::class) override suspend fun getBackupSnapshotsForRestore(): List { - val location = url.toHttpUrl() + val location = "$url/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) debugLog { "getBackupSnapshotsForRestore($location)" } @@ -220,7 +220,7 @@ internal class WebDavStoragePlugin( @Throws(IOException::class) override suspend fun getCurrentBackupSnapshots(): List { - val location = "$url/$folder".toHttpUrl() + val location = "$url/$folder/".toHttpUrl() val davCollection = DavCollection(okHttpClient, location) debugLog { "getCurrentBackupSnapshots($location)" } From 1dd898b0681ea6a64b06e031586a1ee86a981a0d Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 13 Jun 2024 14:58:37 -0300 Subject: [PATCH 2/5] Add fallback when WebDAV PROPFIND with depth=2 isn't supported --- .../seedvault/plugins/webdav/WebDavStorage.kt | 45 ++++++++++++++ .../plugins/webdav/WebDavStoragePlugin.kt | 58 +++++++++++++++---- .../seedvault/storage/WebDavStoragePlugin.kt | 12 +--- .../plugins/webdav/WebDavStoragePluginTest.kt | 4 ++ 4 files changed, 98 insertions(+), 21 deletions(-) diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt index 605eca72..314a7d25 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt @@ -8,7 +8,11 @@ package com.stevesoltys.seedvault.plugins.webdav import android.util.Log import at.bitfire.dav4jvm.BasicDigestAuthHandler import at.bitfire.dav4jvm.DavCollection +import at.bitfire.dav4jvm.MultiResponseCallback import at.bitfire.dav4jvm.Response +import at.bitfire.dav4jvm.Response.HrefRelation.SELF +import at.bitfire.dav4jvm.exception.HttpException +import at.bitfire.dav4jvm.property.DisplayName import at.bitfire.dav4jvm.property.ResourceType import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers @@ -124,6 +128,47 @@ internal abstract class WebDavStorage( return pipedInputStream } + /** + * Tries to do [DavCollection.propfind] with a depth of `2` which is not in RFC4918. + * Since `infinity` isn't supported by nginx either, + * we fallback to iterating over all folders found with depth `1` + * and do another PROPFIND on those, passing the given [callback]. + */ + protected fun DavCollection.propfindDepthTwo(callback: MultiResponseCallback) { + try { + propfind( + depth = 2, // this isn't defined in RFC4918 + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), + callback = callback, + ) + } catch (e: HttpException) { + if (e.code == 400) { + Log.i(TAG, "Got ${e.response}, trying two depth=1 PROPFINDs...") + propfindFakeTwo(callback) + } else { + throw e + } + } + } + + private fun DavCollection.propfindFakeTwo(callback: MultiResponseCallback) { + propfind( + depth = 1, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), + ) { response, relation -> + debugLog { "propFindFakeTwo() = $response" } + // This callback will be called for everything in the folder + callback.onResponse(response, relation) + if (relation != SELF && response.isFolder()) { + DavCollection(okHttpClient, response.href).propfind( + depth = 1, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), + callback = callback, + ) + } + } + } + protected suspend fun DavCollection.createFolder(xmlBody: String? = null): okhttp3.Response { return try { suspendCoroutine { cont -> diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt index b940e393..e2de1e63 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt @@ -9,6 +9,7 @@ import android.content.Context import android.util.Log import at.bitfire.dav4jvm.DavCollection import at.bitfire.dav4jvm.Response.HrefRelation.SELF +import at.bitfire.dav4jvm.exception.HttpException import at.bitfire.dav4jvm.exception.NotFoundException import at.bitfire.dav4jvm.property.DisplayName import at.bitfire.dav4jvm.property.QuotaAvailableBytes @@ -174,20 +175,25 @@ internal class WebDavStoragePlugin( // get all restore set tokens in root folder val tokens = ArrayList() - davCollection.propfind( - depth = 2, - reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), - ) { response, relation -> - debugLog { "getAvailableBackups() = $response" } - // This callback will be called for every file in the folder - if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2 && - response.hrefName() == FILE_BACKUP_METADATA - ) { - val tokenName = response.href.pathSegments[response.href.pathSegments.size - 2] - getTokenOrNull(tokenName)?.let { token -> - tokens.add(token) + try { + davCollection.propfind( + depth = 2, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), + ) { response, relation -> + debugLog { "getAvailableBackups() = $response" } + // This callback will be called for every file in the folder + if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2 && + response.hrefName() == FILE_BACKUP_METADATA + ) { + val tokenName = response.href.pathSegments[response.href.pathSegments.size - 2] + getTokenOrNull(tokenName)?.let { token -> + tokens.add(token) + } } } + } catch (e: HttpException) { + if (e.code == 400) getBackupTokenWithDepthOne(davCollection, tokens) + else throw e } val tokenIterator = tokens.iterator() return generateSequence { @@ -199,6 +205,34 @@ internal class WebDavStoragePlugin( } } + private fun getBackupTokenWithDepthOne(davCollection: DavCollection, tokens: ArrayList) { + davCollection.propfind( + depth = 1, + reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), + ) { response, relation -> + debugLog { "getBackupTokenWithDepthOne() = $response" } + + // we are only interested in sub-folders, skip rest + if (relation == SELF || !response.isFolder()) return@propfind + + val token = getTokenOrNull(response.hrefName()) ?: return@propfind + val tokenUrl = response.href.newBuilder() + .addPathSegment(FILE_BACKUP_METADATA) + .build() + // check if .backup.metadata file exists using HEAD request, + // because some servers (e.g. nginx don't list hidden files with PROPFIND) + try { + DavCollection(okHttpClient, tokenUrl).head { + debugLog { "getBackupTokenWithDepthOne() = $response" } + tokens.add(token) + } + } catch (e: Exception) { + // just log exception and continue, we want to find all files that are there + Log.e(TAG, "Error retrieving $tokenUrl: ", e) + } + } + } + private fun getTokenOrNull(name: String): Long? { val looksLikeToken = name.isNotEmpty() && tokenRegex.matches(name) if (looksLikeToken) { diff --git a/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt index e3019868..f8592af6 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt @@ -77,10 +77,7 @@ internal class WebDavStoragePlugin( val chunkIds = ArrayList() try { val duration = measureDuration { - davCollection.propfind( - depth = 2, - reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), - ) { response, relation -> + davCollection.propfindDepthTwo { response, relation -> debugLog { "getAvailableChunkIds() = $response" } // This callback will be called for every file in the folder if (relation != SELF && response.isFolder()) { @@ -162,13 +159,10 @@ internal class WebDavStoragePlugin( val snapshots = ArrayList() try { - davCollection.propfind( - depth = 2, - reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME), - ) { response, relation -> + davCollection.propfindDepthTwo { response, relation -> debugLog { "getBackupSnapshotsForRestore() = $response" } // This callback will be called for every file in the folder - if (relation != SELF && !response.isFolder()) { + if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2) { val name = response.hrefName() val match = snapshotRegex.matchEntire(name) if (match != null) { diff --git a/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt index c549ba6f..dbbc7cc4 100644 --- a/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt +++ b/app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt @@ -71,6 +71,10 @@ internal class WebDavStoragePluginTest : TransportTest() { plugin.getOutputStream(token, FILE_BACKUP_METADATA).use { it.write(metadata) } + + // now we have data + assertTrue(plugin.hasData(token, FILE_BACKUP_METADATA)) + try { // now we have one backup matching our token val backups = plugin.getAvailableBackups()?.toSet() ?: fail() From 56e26083fca5e943fe17a0c139adac1c966ef322 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 13 Jun 2024 15:40:28 -0300 Subject: [PATCH 3/5] Fix PROPFIND fallback for lighttpd as well --- .../seedvault/plugins/webdav/WebDavStorage.kt | 14 +++++++++++++- .../plugins/webdav/WebDavStoragePlugin.kt | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt index 314a7d25..1793320c 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt @@ -142,7 +142,7 @@ internal abstract class WebDavStorage( callback = callback, ) } catch (e: HttpException) { - if (e.code == 400) { + if (e.isUnsupportedPropfind()) { Log.i(TAG, "Got ${e.response}, trying two depth=1 PROPFINDs...") propfindFakeTwo(callback) } else { @@ -169,6 +169,18 @@ internal abstract class WebDavStorage( } } + protected fun HttpException.isUnsupportedPropfind(): Boolean { + // nginx returns 400 for depth=2 + if (code == 400) { + return true + } + // lighttpd returns 403 with error as if we used infinity + if (code == 403 && responseBody?.contains("propfind-finite-depth") == true) { + return true + } + return false + } + protected suspend fun DavCollection.createFolder(xmlBody: String? = null): okhttp3.Response { return try { suspendCoroutine { cont -> diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt index e2de1e63..8bd9c972 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt @@ -192,7 +192,7 @@ internal class WebDavStoragePlugin( } } } catch (e: HttpException) { - if (e.code == 400) getBackupTokenWithDepthOne(davCollection, tokens) + if (e.isUnsupportedPropfind()) getBackupTokenWithDepthOne(davCollection, tokens) else throw e } val tokenIterator = tokens.iterator() From 4ac02018fa1bfc7ac9637c147cf16672e5edc280 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Thu, 13 Jun 2024 17:44:18 -0300 Subject: [PATCH 4/5] Fix uncaught throwable with dufs WebDAV server The underlying issue will affect other functionality though: https://github.com/sigoden/dufs/issues/400 --- .../com/stevesoltys/seedvault/plugins/StoragePluginManager.kt | 2 +- .../stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePluginManager.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePluginManager.kt index d4614e39..aa8ec43e 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePluginManager.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/StoragePluginManager.kt @@ -139,7 +139,7 @@ class StoragePluginManager( suspend fun getFreeSpace(): Long? { return try { appPlugin.getFreeSpace() - } catch (e: Exception) { + } catch (e: Throwable) { // NoClassDefFound isn't an [Exception], can get thrown by dav4jvm Log.e("StoragePluginManager", "Error getting free space: ", e) null } diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt index 8bd9c972..952a42cd 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt @@ -163,7 +163,7 @@ internal class WebDavStoragePlugin( override suspend fun getAvailableBackups(): Sequence? { return try { doGetAvailableBackups() - } catch (e: Exception) { + } catch (e: Throwable) { // NoClassDefFound isn't an [Exception], can get thrown by dav4jvm Log.e(TAG, "Error getting available backups: ", e) null } From f15018253c0fc652d01bc3c9edb363c1ee7e19e4 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Fri, 14 Jun 2024 09:16:42 -0300 Subject: [PATCH 5/5] Prevent NoClassDefFound error when a WebDAV server returns GetLastModified which happens in the case of dufs (see last commit) --- .../seedvault/plugins/webdav/WebDavStorage.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt index 1793320c..12893a3d 100644 --- a/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt +++ b/app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt @@ -9,8 +9,12 @@ import android.util.Log import at.bitfire.dav4jvm.BasicDigestAuthHandler import at.bitfire.dav4jvm.DavCollection import at.bitfire.dav4jvm.MultiResponseCallback +import at.bitfire.dav4jvm.Property +import at.bitfire.dav4jvm.PropertyFactory +import at.bitfire.dav4jvm.PropertyRegistry import at.bitfire.dav4jvm.Response import at.bitfire.dav4jvm.Response.HrefRelation.SELF +import at.bitfire.dav4jvm.XmlUtils.NS_WEBDAV import at.bitfire.dav4jvm.exception.HttpException import at.bitfire.dav4jvm.property.DisplayName import at.bitfire.dav4jvm.property.ResourceType @@ -27,6 +31,7 @@ import okhttp3.OkHttpClient import okhttp3.RequestBody import okhttp3.internal.closeQuietly import okio.BufferedSink +import org.xmlpull.v1.XmlPullParser import java.io.IOException import java.io.InputStream import java.io.OutputStream @@ -69,6 +74,10 @@ internal abstract class WebDavStorage( protected val baseUrl = webDavConfig.url protected val url = "${webDavConfig.url}/$root" + init { + PropertyRegistry.register(GetLastModified.Factory) + } + @Throws(IOException::class) protected suspend fun getOutputStream(location: HttpUrl): OutputStream { val davCollection = DavCollection(okHttpClient, location) @@ -237,3 +246,19 @@ internal abstract class WebDavStorage( } } + +/** + * A fake version of [at.bitfire.dav4jvm.property.GetLastModified] which we register + * so we don't need to depend on `org.apache.commons.lang3` which is used for date parsing. + */ +class GetLastModified : Property { + companion object { + @JvmField + val NAME = Property.Name(NS_WEBDAV, "getlastmodified") + } + + object Factory : PropertyFactory { + override fun getName() = NAME + override fun create(parser: XmlPullParser): GetLastModified? = null + } +}