mobi/seedvault
1
0
Fork 0
Code Pull requests Activity

Use new WebDavBackend in WebDavStoragePlugin

Browse source
This commit is contained in:
Torsten Grote 2024-08-26 11:00:20 -03:00
parent 099e0ba6d5
commit 8c05ccc39d
No known key found for this signature in database
GPG key ID: 3E5F77D92CF891FF
18 changed files with 255 additions and 534 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavConfig.kt
View file

@ -1,12 +0,0 @@
/*
* SPDX-FileCopyrightText: 2024 The Calyx Institute
* SPDX-License-Identifier: Apache-2.0
*/
package com.stevesoltys.seedvault.plugins.webdav
data class WebDavConfig(
val url: String,
val username: String,
val password: String,
)

3
app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavFactory.kt
View file

@ -9,13 +9,14 @@ import android.annotation.SuppressLint
import android.content.Context
import android.provider.Settings
import com.stevesoltys.seedvault.plugins.StoragePlugin
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
class WebDavFactory(
private val context: Context,
) {
fun createAppStoragePlugin(config: WebDavConfig): StoragePlugin<WebDavConfig> {
return WebDavStoragePlugin(context, config)
return WebDavStoragePlugin(config)
}
fun createFilesStoragePlugin(

1
app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavHandler.kt
View file

@ -14,6 +14,7 @@ import com.stevesoltys.seedvault.settings.SettingsManager
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asStateFlow
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import java.io.IOException
internal sealed interface WebDavConfigState {

1
app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavProperties.kt
View file

@ -7,6 +7,7 @@ package com.stevesoltys.seedvault.plugins.webdav
import android.content.Context
import com.stevesoltys.seedvault.plugins.StorageProperties
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
data class WebDavProperties(
override val config: WebDavConfig,

1
app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStorage.kt
View file

@ -29,6 +29,7 @@ import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.RequestBody
import okio.BufferedSink
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import org.xmlpull.v1.XmlPullParser
import java.io.IOException
import java.io.InputStream

218
app/src/main/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePlugin.kt
View file

@ -5,233 +5,95 @@
package com.stevesoltys.seedvault.plugins.webdav
import android.content.Context
import android.util.Log
import at.bitfire.dav4jvm.DavCollection
import at.bitfire.dav4jvm.Response.HrefRelation.SELF
import at.bitfire.dav4jvm.exception.HttpException
import at.bitfire.dav4jvm.exception.NotFoundException
import at.bitfire.dav4jvm.property.webdav.DisplayName
import at.bitfire.dav4jvm.property.webdav.QuotaAvailableBytes
import at.bitfire.dav4jvm.property.webdav.ResourceType
import com.stevesoltys.seedvault.plugins.EncryptedMetadata
import com.stevesoltys.seedvault.plugins.StoragePlugin
import com.stevesoltys.seedvault.plugins.chunkFolderRegex
import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA
import com.stevesoltys.seedvault.plugins.saf.FILE_NO_MEDIA
import com.stevesoltys.seedvault.plugins.tokenRegex
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.calyxos.backup.storage.plugin.PluginConstants.SNAPSHOT_EXT
import com.stevesoltys.seedvault.worker.FILE_BACKUP_ICONS
import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
import org.calyxos.seedvault.core.backends.webdav.WebDavBackend
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import kotlin.coroutines.resume
import kotlin.coroutines.suspendCoroutine
internal class WebDavStoragePlugin(
context: Context,
webDavConfig: WebDavConfig,
root: String = DIRECTORY_ROOT,
) : WebDavStorage(webDavConfig, root), StoragePlugin<WebDavConfig> {
override suspend fun test(): Boolean {
val location = (if (baseUrl.endsWith('/')) baseUrl else "$baseUrl/").toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
private val delegate = WebDavBackend(webDavConfig, root)
val webDavSupported = suspendCoroutine { cont ->
davCollection.options { davCapabilities, response ->
debugLog { "test() = $davCapabilities $response" }
if (davCapabilities.contains("1")) cont.resume(true)
else if (davCapabilities.contains("2")) cont.resume(true)
else if (davCapabilities.contains("3")) cont.resume(true)
else cont.resume(false)
}
}
return webDavSupported
override suspend fun test(): Boolean {
return delegate.test()
}
override suspend fun getFreeSpace(): Long? {
val location = "$url/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
val availableBytes = suspendCoroutine { cont ->
davCollection.propfind(depth = 0, QuotaAvailableBytes.NAME) { response, _ ->
debugLog { "getFreeSpace() = $response" }
val quota = response.properties.getOrNull(0) as? QuotaAvailableBytes
val availableBytes = quota?.quotaAvailableBytes ?: -1
if (availableBytes > 0) {
cont.resume(availableBytes)
} else {
cont.resume(null)
}
}
}
return availableBytes
return delegate.getFreeSpace()
}
@Throws(IOException::class)
override suspend fun startNewRestoreSet(token: Long) {
val location = "$url/$token/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
val response = davCollection.createFolder()
debugLog { "startNewRestoreSet($token) = $response" }
// no-op
}
@Throws(IOException::class)
override suspend fun initializeDevice() {
// TODO does it make sense to delete anything
// when [startNewRestoreSet] is always called first? Maybe unify both calls?
val location = "$url/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
try {
davCollection.head { response ->
debugLog { "Root exists: $response" }
}
} catch (e: NotFoundException) {
val response = davCollection.createFolder()
debugLog { "initializeDevice() = $response" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException(e)
}
// no-op
}
@Throws(IOException::class)
override suspend fun getOutputStream(token: Long, name: String): OutputStream {
val location = "$url/$token/$name".toHttpUrl()
return try {
getOutputStream(location)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting OutputStream for $token and $name: ", e)
val handle = when (name) {
FILE_BACKUP_METADATA -> LegacyAppBackupFile.Metadata(token)
FILE_BACKUP_ICONS -> LegacyAppBackupFile.IconsFile(token)
else -> LegacyAppBackupFile.Blob(token, name)
}
return delegate.save(handle).outputStream()
}
@Throws(IOException::class)
override suspend fun getInputStream(token: Long, name: String): InputStream {
val location = "$url/$token/$name".toHttpUrl()
return try {
getInputStream(location)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting InputStream for $token and $name: ", e)
val handle = when (name) {
FILE_BACKUP_METADATA -> LegacyAppBackupFile.Metadata(token)
FILE_BACKUP_ICONS -> LegacyAppBackupFile.IconsFile(token)
else -> LegacyAppBackupFile.Blob(token, name)
}
return delegate.load(handle).inputStream()
}
@Throws(IOException::class)
override suspend fun removeData(token: Long, name: String) {
val location = "$url/$token/$name".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
try {
val response = suspendCoroutine { cont ->
davCollection.delete { response ->
cont.resume(response)
}
}
debugLog { "removeData($token, $name) = $response" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException(e)
val handle = when (name) {
FILE_BACKUP_METADATA -> LegacyAppBackupFile.Metadata(token)
FILE_BACKUP_ICONS -> LegacyAppBackupFile.IconsFile(token)
else -> LegacyAppBackupFile.Blob(token, name)
}
delegate.remove(handle)
}
override suspend fun getAvailableBackups(): Sequence<EncryptedMetadata>? {
return try {
doGetAvailableBackups()
// get all restore set tokens in root folder that have a metadata file
val tokens = ArrayList<Long>()
delegate.list(null, LegacyAppBackupFile.Metadata::class) { fileInfo ->
val handle = fileInfo.fileHandle as LegacyAppBackupFile.Metadata
tokens.add(handle.token)
}
val tokenIterator = tokens.iterator()
return generateSequence {
if (!tokenIterator.hasNext()) return@generateSequence null // end sequence
val token = tokenIterator.next()
EncryptedMetadata(token) {
getInputStream(token, FILE_BACKUP_METADATA)
}
}
} catch (e: Throwable) { // NoClassDefFound isn't an [Exception], can get thrown by dav4jvm
Log.e(TAG, "Error getting available backups: ", e)
null
}
}
private suspend fun doGetAvailableBackups(): Sequence<EncryptedMetadata> {
val location = "$url/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
// get all restore set tokens in root folder
val tokens = ArrayList<Long>()
try {
davCollection.propfind(
depth = 2,
reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME),
) { response, relation ->
debugLog { "getAvailableBackups() = $response" }
// This callback will be called for every file in the folder
if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2 &&
response.hrefName() == FILE_BACKUP_METADATA
) {
val tokenName = response.href.pathSegments[response.href.pathSegments.size - 2]
getTokenOrNull(tokenName)?.let { token ->
tokens.add(token)
}
}
}
} catch (e: HttpException) {
if (e.isUnsupportedPropfind()) getBackupTokenWithDepthOne(davCollection, tokens)
else throw e
}
val tokenIterator = tokens.iterator()
return generateSequence {
if (!tokenIterator.hasNext()) return@generateSequence null // end sequence
val token = tokenIterator.next()
EncryptedMetadata(token) {
getInputStream(token, FILE_BACKUP_METADATA)
}
}
}
private fun getBackupTokenWithDepthOne(davCollection: DavCollection, tokens: ArrayList<Long>) {
davCollection.propfind(
depth = 1,
reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME),
) { response, relation ->
debugLog { "getBackupTokenWithDepthOne() = $response" }
// we are only interested in sub-folders, skip rest
if (relation == SELF || !response.isFolder()) return@propfind
val token = getTokenOrNull(response.hrefName()) ?: return@propfind
val tokenUrl = response.href.newBuilder()
.addPathSegment(FILE_BACKUP_METADATA)
.build()
// check if .backup.metadata file exists using HEAD request,
// because some servers (e.g. nginx don't list hidden files with PROPFIND)
try {
DavCollection(okHttpClient, tokenUrl).head {
debugLog { "getBackupTokenWithDepthOne() = $response" }
tokens.add(token)
}
} catch (e: Exception) {
// just log exception and continue, we want to find all files that are there
Log.e(TAG, "Error retrieving $tokenUrl: ", e)
}
}
}
private fun getTokenOrNull(name: String): Long? {
val looksLikeToken = name.isNotEmpty() && tokenRegex.matches(name)
if (looksLikeToken) {
return try {
name.toLong()
} catch (e: NumberFormatException) {
throw AssertionError(e) // regex must be wrong
}
}
if (isUnexpectedFile(name)) {
Log.w(TAG, "Found invalid backup set folder: $name")
}
return null
}
private fun isUnexpectedFile(name: String): Boolean {
return name != FILE_NO_MEDIA &&
!chunkFolderRegex.matches(name) &&
!name.endsWith(SNAPSHOT_EXT)
}
override val providerPackageName: String = context.packageName // 100% built-in plugin
override val providerPackageName: String? = null // 100% built-in plugin
}

2
app/src/main/java/com/stevesoltys/seedvault/settings/SettingsManager.kt
View file

@ -15,11 +15,11 @@ import com.stevesoltys.seedvault.permitDiskReads
import com.stevesoltys.seedvault.plugins.StoragePlugin
import com.stevesoltys.seedvault.plugins.saf.DocumentsProviderStoragePlugin
import com.stevesoltys.seedvault.plugins.saf.SafStorage
import com.stevesoltys.seedvault.plugins.webdav.WebDavConfig
import com.stevesoltys.seedvault.plugins.webdav.WebDavHandler.Companion.createWebDavProperties
import com.stevesoltys.seedvault.plugins.webdav.WebDavProperties
import com.stevesoltys.seedvault.plugins.webdav.WebDavStoragePlugin
import com.stevesoltys.seedvault.transport.backup.BackupCoordinator
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import java.util.concurrent.ConcurrentSkipListSet
internal const val PREF_KEY_TOKEN = "token"

242
app/src/main/java/com/stevesoltys/seedvault/storage/WebDavStoragePlugin.kt
View file

@ -5,187 +5,75 @@
package com.stevesoltys.seedvault.storage
import android.util.Log
import at.bitfire.dav4jvm.DavCollection
import at.bitfire.dav4jvm.Response.HrefRelation.SELF
import at.bitfire.dav4jvm.exception.NotFoundException
import at.bitfire.dav4jvm.property.webdav.DisplayName
import at.bitfire.dav4jvm.property.webdav.ResourceType
import com.stevesoltys.seedvault.plugins.chunkFolderRegex
import com.stevesoltys.seedvault.plugins.webdav.DIRECTORY_ROOT
import com.stevesoltys.seedvault.plugins.webdav.WebDavConfig
import com.stevesoltys.seedvault.plugins.webdav.WebDavStorage
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.calyxos.backup.storage.api.StoragePlugin
import org.calyxos.backup.storage.api.StoredSnapshot
import org.calyxos.backup.storage.plugin.PluginConstants.SNAPSHOT_EXT
import org.calyxos.backup.storage.plugin.PluginConstants.chunkRegex
import org.calyxos.backup.storage.plugin.PluginConstants.snapshotRegex
import org.koin.core.time.measureDuration
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.backends.webdav.WebDavBackend
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import kotlin.coroutines.resume
import kotlin.coroutines.suspendCoroutine
internal class WebDavStoragePlugin(
/**
* The result of Settings.Secure.getString(context.contentResolver, Settings.Secure.ANDROID_ID)
*/
androidId: String,
private val androidId: String,
webDavConfig: WebDavConfig,
root: String = DIRECTORY_ROOT,
) : WebDavStorage(webDavConfig, root), StoragePlugin {
) : StoragePlugin {
/**
* The folder name is our user ID plus .sv extension (for SeedVault).
* The user or `androidId` is unique to each combination of app-signing key, user, and device
* so we don't leak anything by not hashing this and can use it as is.
*/
private val folder: String = "$androidId.sv"
private val topLevelFolder = TopLevelFolder("$androidId.sv")
private val delegate = WebDavBackend(webDavConfig, root)
@Throws(IOException::class)
override suspend fun init() {
val location = "$url/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
try {
davCollection.head { response ->
debugLog { "Root exists: $response" }
}
} catch (e: NotFoundException) {
val response = davCollection.createFolder()
debugLog { "init() = $response" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException(e)
}
// no-op
}
@Throws(IOException::class)
override suspend fun getAvailableChunkIds(): List<String> {
val location = "$url/$folder/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
debugLog { "getAvailableChunkIds($location)" }
val expectedChunkFolders = (0x00..0xff).map {
Integer.toHexString(it).padStart(2, '0')
}.toHashSet()
val chunkIds = ArrayList<String>()
try {
val duration = measureDuration {
davCollection.propfindDepthTwo { response, relation ->
debugLog { "getAvailableChunkIds() = $response" }
// This callback will be called for every file in the folder
if (relation != SELF && response.isFolder()) {
val name = response.hrefName()
if (chunkFolderRegex.matches(name)) {
expectedChunkFolders.remove(name)
}
} else if (relation != SELF && response.href.pathSize >= 2) {
val folderName =
response.href.pathSegments[response.href.pathSegments.size - 2]
if (folderName != folder && chunkFolderRegex.matches(folderName)) {
val name = response.hrefName()
if (chunkRegex.matches(name)) chunkIds.add(name)
}
}
}
}
Log.i(TAG, "Retrieving chunks took $duration")
} catch (e: NotFoundException) {
debugLog { "Folder not found: $location" }
davCollection.createFolder()
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error populating chunk folders: ", e)
delegate.list(topLevelFolder, FileBackupFileType.Blob::class) { fileInfo ->
chunkIds.add(fileInfo.fileHandle.name)
}
Log.i(TAG, "Got ${chunkIds.size} available chunks")
createMissingChunkFolders(expectedChunkFolders)
return chunkIds
}
@Throws(IOException::class)
private suspend fun createMissingChunkFolders(
missingChunkFolders: Set<String>,
) {
val s = missingChunkFolders.size
for ((i, chunkFolderName) in missingChunkFolders.withIndex()) {
val location = "$url/$folder/$chunkFolderName/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
val response = davCollection.createFolder()
debugLog { "Created missing folder $chunkFolderName (${i + 1}/$s) $response" }
}
}
@Throws(IOException::class)
override suspend fun getChunkOutputStream(chunkId: String): OutputStream {
val chunkFolderName = chunkId.substring(0, 2)
val location = "$url/$folder/$chunkFolderName/$chunkId".toHttpUrl()
debugLog { "getChunkOutputStream($location) for $chunkId" }
return try {
getOutputStream(location)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting OutputStream for $chunkId: ", e)
}
val fileHandle = FileBackupFileType.Blob(androidId, chunkId)
return delegate.save(fileHandle).outputStream()
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotOutputStream(timestamp: Long): OutputStream {
val location = "$url/$folder/$timestamp$SNAPSHOT_EXT".toHttpUrl()
debugLog { "getBackupSnapshotOutputStream($location)" }
return try {
getOutputStream(location)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting OutputStream for $timestamp$SNAPSHOT_EXT: ", e)
}
val fileHandle = FileBackupFileType.Snapshot(androidId, timestamp)
return delegate.save(fileHandle).outputStream()
}
/************************* Restore *******************************/
@Throws(IOException::class)
override suspend fun getBackupSnapshotsForRestore(): List<StoredSnapshot> {
val location = "$url/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
debugLog { "getBackupSnapshotsForRestore($location)" }
val snapshots = ArrayList<StoredSnapshot>()
try {
davCollection.propfindDepthTwo { response, relation ->
debugLog { "getBackupSnapshotsForRestore() = $response" }
// This callback will be called for every file in the folder
if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2) {
val name = response.hrefName()
val match = snapshotRegex.matchEntire(name)
if (match != null) {
val timestamp = match.groupValues[1].toLong()
val folderName =
response.href.pathSegments[response.href.pathSegments.size - 2]
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
}
}
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting snapshots for restore: ", e)
delegate.list(null, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
return snapshots
}
@Throws(IOException::class)
override suspend fun getBackupSnapshotInputStream(storedSnapshot: StoredSnapshot): InputStream {
val timestamp = storedSnapshot.timestamp
val location = "$url/${storedSnapshot.userId}/$timestamp$SNAPSHOT_EXT".toHttpUrl()
debugLog { "getBackupSnapshotInputStream($location)" }
return try {
getInputStream(location)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting InputStream for $storedSnapshot: ", e)
}
val androidId = storedSnapshot.androidId
val handle = FileBackupFileType.Snapshot(androidId, storedSnapshot.timestamp)
return delegate.load(handle).inputStream()
}
@Throws(IOException::class)
@ -193,92 +81,38 @@ internal class WebDavStoragePlugin(
snapshot: StoredSnapshot,
chunkId: String,
): InputStream {
val chunkFolderName = chunkId.substring(0, 2)
val location = "$url/${snapshot.userId}/$chunkFolderName/$chunkId".toHttpUrl()
debugLog { "getChunkInputStream($location) for $chunkId" }
return try {
getInputStream(location)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting InputStream for $chunkFolderName/$chunkId: ", e)
}
val handle = FileBackupFileType.Blob(snapshot.androidId, chunkId)
return delegate.load(handle).inputStream()
}
/************************* Pruning *******************************/
@Throws(IOException::class)
override suspend fun getCurrentBackupSnapshots(): List<StoredSnapshot> {
val location = "$url/$folder/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
debugLog { "getCurrentBackupSnapshots($location)" }
val snapshots = ArrayList<StoredSnapshot>()
try {
val duration = measureDuration {
davCollection.propfind(
depth = 1,
reqProp = arrayOf(DisplayName.NAME, ResourceType.NAME),
) { response, relation ->
debugLog { "getCurrentBackupSnapshots() = $response" }
// This callback will be called for every file in the folder
if (relation != SELF && !response.isFolder()) {
val match = snapshotRegex.matchEntire(response.hrefName())
if (match != null) {
val timestamp = match.groupValues[1].toLong()
val storedSnapshot = StoredSnapshot(folder, timestamp)
snapshots.add(storedSnapshot)
}
}
}
}
Log.i(TAG, "getCurrentBackupSnapshots took $duration")
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error getting current snapshots: ", e)
delegate.list(topLevelFolder, FileBackupFileType.Snapshot::class) { fileInfo ->
val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot
val folderName = handle.topLevelFolder.name
val timestamp = handle.time
val storedSnapshot = StoredSnapshot(folderName, timestamp)
snapshots.add(storedSnapshot)
}
Log.i(TAG, "Got ${snapshots.size} snapshots.")
return snapshots
}
@Throws(IOException::class)
override suspend fun deleteBackupSnapshot(storedSnapshot: StoredSnapshot) {
val timestamp = storedSnapshot.timestamp
Log.d(TAG, "Deleting snapshot $timestamp")
val location = "$url/${storedSnapshot.userId}/$timestamp$SNAPSHOT_EXT".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
try {
val response = suspendCoroutine { cont ->
davCollection.delete { response ->
cont.resume(response)
}
}
debugLog { "deleteBackupSnapshot() = $response" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException(e)
}
val androidId = storedSnapshot.androidId
val handle = FileBackupFileType.Snapshot(androidId, storedSnapshot.timestamp)
delegate.remove(handle)
}
@Throws(IOException::class)
override suspend fun deleteChunks(chunkIds: List<String>) {
chunkIds.forEach { chunkId ->
val chunkFolderName = chunkId.substring(0, 2)
val location = "$url/$folder/$chunkFolderName/$chunkId".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
try {
val response = suspendCoroutine { cont ->
davCollection.delete { response ->
cont.resume(response)
}
}
debugLog { "deleteChunks($chunkId) = $response" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException(e)
}
val androidId = topLevelFolder.name.substringBefore(".sv")
val handle = FileBackupFileType.Blob(androidId, chunkId)
delegate.remove(handle)
}
}
}

2
app/src/main/java/com/stevesoltys/seedvault/ui/storage/StorageViewModel.kt
View file

@ -16,7 +16,6 @@ import com.stevesoltys.seedvault.R
import com.stevesoltys.seedvault.plugins.StoragePluginManager
import com.stevesoltys.seedvault.plugins.saf.SafHandler
import com.stevesoltys.seedvault.plugins.saf.SafStorage
import com.stevesoltys.seedvault.plugins.webdav.WebDavConfig
import com.stevesoltys.seedvault.plugins.webdav.WebDavHandler
import com.stevesoltys.seedvault.plugins.webdav.WebDavProperties
import com.stevesoltys.seedvault.plugins.webdav.WebDavStoragePlugin
@ -26,6 +25,7 @@ import com.stevesoltys.seedvault.ui.MutableLiveEvent
import com.stevesoltys.seedvault.ui.storage.StorageOption.SafOption
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
internal abstract class StorageViewModel(
private val app: Application,

42
app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavStoragePluginTest.kt
View file

@ -8,23 +8,20 @@ package com.stevesoltys.seedvault.plugins.webdav
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.stevesoltys.seedvault.TestApp
import com.stevesoltys.seedvault.getRandomByteArray
import com.stevesoltys.seedvault.getRandomString
import com.stevesoltys.seedvault.plugins.EncryptedMetadata
import com.stevesoltys.seedvault.plugins.saf.FILE_BACKUP_METADATA
import com.stevesoltys.seedvault.transport.TransportTest
import kotlinx.coroutines.runBlocking
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import org.junit.Test
import org.junit.jupiter.api.Assertions.assertArrayEquals
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Assertions.fail
import org.junit.jupiter.api.assertThrows
import org.junit.runner.RunWith
import org.robolectric.annotation.Config
import java.io.IOException
import kotlin.random.Random
@RunWith(AndroidJUnit4::class)
@Config(
@ -33,13 +30,13 @@ import kotlin.random.Random
)
internal class WebDavStoragePluginTest : TransportTest() {
private val plugin = WebDavStoragePlugin(context, WebDavTestConfig.getConfig())
private val plugin = WebDavStoragePlugin(WebDavTestConfig.getConfig())
@Test
fun `test self-test`() = runBlocking {
assertTrue(plugin.test())
val plugin2 = WebDavStoragePlugin(context, WebDavConfig("https://github.com/", "", ""))
val plugin2 = WebDavStoragePlugin(WebDavConfig("https://github.com/", "", ""))
val e = assertThrows<Exception> {
assertFalse(plugin2.test())
}
@ -86,37 +83,4 @@ internal class WebDavStoragePluginTest : TransportTest() {
}
}
@Test
fun `test streams for non-existent data`() = runBlocking {
val token = Random.nextLong(System.currentTimeMillis(), 9999999999999)
val file = getRandomString()
assertFalse(plugin.hasData(token, file))
assertThrows<IOException> {
plugin.getOutputStream(token, file).use { it.write(getRandomByteArray()) }
}
assertThrows<IOException> {
plugin.getInputStream(token, file).use {
it.readAllBytes()
}
}
Unit
}
@Test
fun `test missing root dir`() = runBlocking {
val plugin = WebDavStoragePlugin(context, WebDavTestConfig.getConfig(), getRandomString())
assertNull(plugin.getAvailableBackups())
assertFalse(plugin.hasData(42L, "foo"))
assertThrows<IOException> {
plugin.removeData(42L, "foo")
}
Unit
}
}

1
app/src/test/java/com/stevesoltys/seedvault/plugins/webdav/WebDavTestConfig.kt
View file

@ -5,6 +5,7 @@
package com.stevesoltys.seedvault.plugins.webdav
import org.calyxos.seedvault.core.backends.webdav.WebDavConfig
import org.junit.Assume.assumeFalse
import org.junit.jupiter.api.Assertions.fail

51
app/src/test/java/com/stevesoltys/seedvault/storage/WebDavStoragePluginTest.kt
View file

@ -6,7 +6,6 @@
package com.stevesoltys.seedvault.storage
import com.stevesoltys.seedvault.getRandomByteArray
import com.stevesoltys.seedvault.getRandomString
import com.stevesoltys.seedvault.plugins.webdav.WebDavTestConfig
import com.stevesoltys.seedvault.transport.backup.BackupTest
import kotlinx.coroutines.runBlocking
@ -14,14 +13,13 @@ import org.calyxos.backup.storage.api.StoredSnapshot
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Test
import org.junit.jupiter.api.assertThrows
import java.io.IOException
internal class WebDavStoragePluginTest : BackupTest() {
private val plugin = WebDavStoragePlugin("foo", WebDavTestConfig.getConfig())
private val androidId = "abcdef0123456789"
private val plugin = WebDavStoragePlugin(androidId, WebDavTestConfig.getConfig())
private val snapshot = StoredSnapshot("foo.sv", System.currentTimeMillis())
private val snapshot = StoredSnapshot("$androidId.sv", System.currentTimeMillis())
@Test
fun `test chunks`() = runBlocking {
@ -82,8 +80,9 @@ internal class WebDavStoragePluginTest : BackupTest() {
)
// other device writes another snapshot
val otherPlugin = WebDavStoragePlugin("bar", WebDavTestConfig.getConfig())
val otherSnapshot = StoredSnapshot("bar.sv", System.currentTimeMillis())
val androidId2 = "0123456789abcdef"
val otherPlugin = WebDavStoragePlugin(androidId2, WebDavTestConfig.getConfig())
val otherSnapshot = StoredSnapshot("$androidId2.sv", System.currentTimeMillis())
val otherSnapshotBytes = getRandomByteArray()
assertEquals(emptyList<String>(), otherPlugin.getAvailableChunkIds())
otherPlugin.getBackupSnapshotOutputStream(otherSnapshot.timestamp).use {
@ -104,44 +103,6 @@ internal class WebDavStoragePluginTest : BackupTest() {
}
}
@Test
fun `test missing root dir`() = runBlocking {
val plugin = WebDavStoragePlugin(
androidId = "foo",
webDavConfig = WebDavTestConfig.getConfig(),
root = getRandomString(),
)
assertThrows<IOException> {
plugin.getCurrentBackupSnapshots()
}
assertThrows<IOException> {
plugin.getBackupSnapshotsForRestore()
}
assertThrows<IOException> {
plugin.getAvailableChunkIds()
}
assertThrows<IOException> {
plugin.deleteChunks(listOf("foo"))
}
assertThrows<IOException> {
plugin.deleteBackupSnapshot(snapshot)
}
assertThrows<IOException> {
plugin.getBackupSnapshotOutputStream(snapshot.timestamp).close()
}
assertThrows<IOException> {
plugin.getBackupSnapshotInputStream(snapshot).use { it.readAllBytes() }
}
assertThrows<IOException> {
plugin.getChunkOutputStream("foo").close()
}
assertThrows<IOException> {
plugin.getChunkInputStream(snapshot, "foo").use { it.readAllBytes() }
}
Unit
}
}
private fun ByteArray.toHexString() = joinToString("") { "%02x".format(it) }

1
app/src/test/resources/simplelogger.properties Normal file
View file

@ -0,0 +1 @@
org.slf4j.simpleLogger.defaultLogLevel=trace

25
core/src/main/java/org/calyxos/seedvault/core/backends/Backend.kt
View file

@ -5,12 +5,26 @@
package org.calyxos.seedvault.core.backends
import androidx.annotation.VisibleForTesting
import okio.BufferedSink
import okio.BufferedSource
import kotlin.reflect.KClass
public interface Backend {
/**
* Returns true if the plugin is working, or false if it isn't.
* @throws Exception any kind of exception to provide more info on the error
*/
public suspend fun test(): Boolean
/**
* Retrieves the available storage space in bytes.
* @return the number of bytes available or null if the number is unknown.
* Returning a negative number or zero to indicate unknown is discouraged.
*/
public suspend fun getFreeSpace(): Long?
public suspend fun save(handle: FileHandle): BufferedSink
public suspend fun load(handle: FileHandle): BufferedSource
@ -25,7 +39,16 @@ public interface Backend {
public suspend fun rename(from: TopLevelFolder, to: TopLevelFolder)
// TODO really all?
@VisibleForTesting
public suspend fun removeAll()
/**
* Returns the package name of the app that provides the storage backend
* which is used for the current backup location.
*
* Backends are advised to cache this as it will be requested frequently.
*
* @return null if no package name could be found
*/
public val providerPackageName: String?
}

6
core/src/main/java/org/calyxos/seedvault/core/backends/FileHandle.kt
View file

@ -43,6 +43,12 @@ public sealed class LegacyAppBackupFile : FileHandle() {
public sealed class FileBackupFileType : FileHandle() {
public abstract val androidId: String
/**
* The folder name is our user ID plus .sv extension (for SeedVault).
* The user or `androidId` is unique to each combination of app-signing key, user, and device
* so we don't leak anything by not hashing this and can use it as is.
*/
public val topLevelFolder: TopLevelFolder get() = TopLevelFolder("$androidId.sv")
public data class Blob(

10
core/src/main/java/org/calyxos/seedvault/core/backends/saf/SafBackend.kt
View file

@ -45,6 +45,14 @@ public class SafBackend(
private val context: Context get() = appContext.getBackendContext { safConfig.isUsb }
private val cache = DocumentFileCache(context, safConfig.getDocumentFile(context), root)
override suspend fun test(): Boolean {
TODO("Not yet implemented")
}
override suspend fun getFreeSpace(): Long? {
TODO("Not yet implemented")
}
override suspend fun save(handle: FileHandle): BufferedSink {
val file = cache.getFile(handle)
return file.getOutputStream(context.contentResolver).sink().buffer()
@ -148,4 +156,6 @@ public class SafBackend(
}
}
override val providerPackageName: String? get() = TODO("Not yet implemented")
}

167
core/src/main/java/org/calyxos/seedvault/core/backends/webdav/WebDavBackend.kt
View file

@ -11,6 +11,7 @@ import at.bitfire.dav4jvm.PropertyRegistry
import at.bitfire.dav4jvm.Response.HrefRelation.SELF
import at.bitfire.dav4jvm.exception.HttpException
import at.bitfire.dav4jvm.exception.NotFoundException
import at.bitfire.dav4jvm.property.webdav.QuotaAvailableBytes
import io.github.oshai.kotlinlogging.KLogger
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.DelicateCoroutinesApi
@ -51,7 +52,7 @@ import kotlin.reflect.KClass
private const val DEBUG_LOG = true
@OptIn(DelicateCoroutinesApi::class)
internal class WebDavBackend(
public class WebDavBackend(
webDavConfig: WebDavConfig,
root: String = DIRECTORY_ROOT,
) : Backend {
@ -75,13 +76,53 @@ internal class WebDavBackend(
.retryOnConnectionFailure(true)
.build()
private val url = "${webDavConfig.url}/$root"
private val baseUrl = webDavConfig.url.trimEnd('/')
private val url = "$baseUrl/$root"
private val folders = mutableSetOf<HttpUrl>() // cache for existing/created folders
init {
PropertyRegistry.register(GetLastModified.Factory)
}
override suspend fun test(): Boolean {
val location = "$baseUrl/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
val hasCaps = suspendCoroutine { cont ->
davCollection.options { davCapabilities, response ->
log.debugLog { "test() = $davCapabilities $response" }
if (davCapabilities.contains("1")) cont.resume(true)
else if (davCapabilities.contains("2")) cont.resume(true)
else if (davCapabilities.contains("3")) cont.resume(true)
else cont.resume(false)
}
}
if (!hasCaps) return false
val rootCollection = DavCollection(okHttpClient, "$url/foo".toHttpUrl())
rootCollection.ensureFoldersExist(log, folders) // only considers parents, so foo isn't used
return true
}
override suspend fun getFreeSpace(): Long? {
val location = "$url/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
val availableBytes = suspendCoroutine { cont ->
davCollection.propfind(depth = 0, QuotaAvailableBytes.NAME) { response, _ ->
log.debugLog { "getFreeSpace() = $response" }
val quota = response.properties.getOrNull(0) as? QuotaAvailableBytes
val availableBytes = quota?.quotaAvailableBytes ?: -1
if (availableBytes > 0) {
cont.resume(availableBytes)
} else {
cont.resume(null)
}
}
}
return availableBytes
}
override suspend fun save(handle: FileHandle): BufferedSink {
val location = handle.toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
@ -118,10 +159,15 @@ internal class WebDavBackend(
val location = handle.toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
val response = davCollection.get(accept = "", headers = null)
val response = try {
davCollection.get(accept = "", headers = null)
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error loading $location", e)
}
log.debugLog { "load($location) = $response" }
if (response.code / 100 != 2) throw IOException("HTTP error ${response.code}")
return response.body?.source() ?: throw IOException()
return response.body?.source() ?: throw IOException("Body was null for $location")
}
override suspend fun list(
@ -145,58 +191,67 @@ internal class WebDavBackend(
}
val davCollection = DavCollection(okHttpClient, location)
val tokenFolders = mutableSetOf<HttpUrl>()
davCollection.propfindDepthInfinity(depth) { response, relation ->
log.debugLog { "list() = $response" }
try {
davCollection.propfindDepthInfinity(depth) { response, relation ->
log.debugLog { "list() = $response" }
// work around nginx's inability to find files starting with .
if (relation != SELF && LegacyAppBackupFile.Metadata::class in fileTypes &&
response.isFolder() && response.hrefName().matches(tokenRegex)
) {
tokenFolders.add(response.href)
}
if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2) {
val name = response.hrefName()
val parentName = response.href.pathSegments[response.href.pathSegments.size - 2]
// work around nginx's inability to find files starting with .
if (relation != SELF && LegacyAppBackupFile.Metadata::class in fileTypes &&
response.isFolder() && response.hrefName().matches(tokenRegex)
) {
tokenFolders.add(response.href)
}
if (relation != SELF && !response.isFolder() && response.href.pathSize >= 2) {
val name = response.hrefName()
val parentName = response.href.pathSegments[response.href.pathSegments.size - 2]
if (LegacyAppBackupFile.Metadata::class in fileTypes) {
if (name == FILE_BACKUP_METADATA && parentName.matches(tokenRegex)) {
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
val size = response.properties.contentLength()
callback(FileInfo(metadata, size))
// we can find .backup.metadata files, so no need for nginx workaround
tokenFolders.clear()
}
}
if (FileBackupFileType.Snapshot::class in fileTypes ||
FileBackupFileType::class in fileTypes
) {
val match = snapshotRegex.matchEntire(name)
if (match != null) {
val size = response.properties.contentLength()
val snapshot = FileBackupFileType.Snapshot(
androidId = parentName.substringBefore('.'),
time = match.groupValues[1].toLong(),
)
callback(FileInfo(snapshot, size))
}
}
if ((FileBackupFileType.Blob::class in fileTypes ||
FileBackupFileType::class in fileTypes) && response.href.pathSize >= 3
) {
val androidIdSv =
response.href.pathSegments[response.href.pathSegments.size - 3]
if (folderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) {
if (chunkRegex.matches(name)) {
val blob = FileBackupFileType.Blob(
androidId = androidIdSv.substringBefore('.'),
name = name,
)
if (LegacyAppBackupFile.Metadata::class in fileTypes) {
if (name == FILE_BACKUP_METADATA && parentName.matches(tokenRegex)) {
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
val size = response.properties.contentLength()
callback(FileInfo(blob, size))
callback(FileInfo(metadata, size))
// we can find .backup.metadata files, so no need for nginx workaround
tokenFolders.clear()
}
}
if (FileBackupFileType.Snapshot::class in fileTypes ||
FileBackupFileType::class in fileTypes
) {
val match = snapshotRegex.matchEntire(name)
if (match != null) {
val size = response.properties.contentLength()
val snapshot = FileBackupFileType.Snapshot(
androidId = parentName.substringBefore('.'),
time = match.groupValues[1].toLong(),
)
callback(FileInfo(snapshot, size))
}
}
if ((FileBackupFileType.Blob::class in fileTypes ||
FileBackupFileType::class in fileTypes) && response.href.pathSize >= 3
) {
val androidIdSv =
response.href.pathSegments[response.href.pathSegments.size - 3]
if (folderRegex.matches(androidIdSv) &&
chunkFolderRegex.matches(parentName)
) {
if (chunkRegex.matches(name)) {
val blob = FileBackupFileType.Blob(
androidId = androidIdSv.substringBefore('.'),
name = name,
)
val size = response.properties.contentLength()
callback(FileInfo(blob, size))
}
}
}
}
}
} catch (e: NotFoundException) {
log.warn(e) { "$location not found" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error listing $location", e)
}
// direct query for .backup.metadata as nginx doesn't support listing hidden files
tokenFolders.forEach { url ->
@ -251,8 +306,13 @@ internal class WebDavBackend(
val location = "$url/${from.name}/".toHttpUrl()
val toUrl = "$url/${to.name}/".toHttpUrl()
val davCollection = DavCollection(okHttpClient, location)
davCollection.move(toUrl, false) { response ->
log.debugLog { "rename(${from.name}, ${to.name}) = $response" }
try {
davCollection.move(toUrl, false) { response ->
log.debugLog { "rename(${from.name}, ${to.name}) = $response" }
}
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error renaming $location to ${to.name}", e)
}
}
@ -265,9 +325,14 @@ internal class WebDavBackend(
}
} catch (e: NotFoundException) {
log.info { "Not found: $location" }
} catch (e: Exception) {
if (e is IOException) throw e
else throw IOException("Error removing all at $location", e)
}
}
override val providerPackageName: String? = null // 100% built-in plugin
private fun FileHandle.toHttpUrl(): HttpUrl = when (this) {
// careful with trailing slashes, use only for folders/collections
is TopLevelFolder -> "$url/$name/".toHttpUrl()

4
storage/lib/src/main/java/org/calyxos/backup/storage/api/Snapshot.kt
View file

@ -29,7 +29,9 @@ public data class StoredSnapshot(
* The timestamp identifying a snapshot of the [userId].
*/
public val timestamp: Long,
)
) {
public val androidId: String = userId.substringBefore(".sv")
}
/**
* Defines which backup snapshots should be retained when pruning backups.