Prepare backends for new app backup repository

This commit is contained in:
Torsten Grote 2024-09-03 13:51:13 -03:00
parent d2df088f2c
commit c19787a7fa
No known key found for this signature in database
GPG key ID: 3E5F77D92CF891FF
8 changed files with 228 additions and 63 deletions

View file

@ -33,7 +33,7 @@ class SafBackendTest : BackendTest(), KoinComponent {
requiresNetwork = safStorage.requiresNetwork,
rootId = safStorage.rootId,
)
override val plugin: Backend = SafBackend(context, safProperties, ".SeedvaultTest")
override val backend: Backend = SafBackend(context, safProperties, ".SeedvaultTest")
@Test
fun `test write list read rename delete`(): Unit = runBlocking {

View file

@ -16,26 +16,26 @@ import kotlin.test.assertNotNull
@VisibleForTesting
public abstract class BackendTest {
public abstract val plugin: Backend
public abstract val backend: Backend
protected suspend fun testWriteListReadRenameDelete() {
plugin.removeAll()
backend.removeAll()
val androidId = "0123456789abcdef"
val now = System.currentTimeMillis()
val bytes1 = Random.nextBytes(1337)
val bytes2 = Random.nextBytes(1337 * 8)
plugin.save(LegacyAppBackupFile.Metadata(now)).use {
backend.save(LegacyAppBackupFile.Metadata(now)).use {
it.write(bytes1)
}
plugin.save(FileBackupFileType.Snapshot(androidId, now)).use {
backend.save(FileBackupFileType.Snapshot(androidId, now)).use {
it.write(bytes2)
}
var metadata: LegacyAppBackupFile.Metadata? = null
var snapshot: FileBackupFileType.Snapshot? = null
plugin.list(
var fileSnapshot: FileBackupFileType.Snapshot? = null
backend.list(
null,
FileBackupFileType.Snapshot::class,
FileBackupFileType.Blob::class,
@ -45,22 +45,22 @@ public abstract class BackendTest {
if (handle is LegacyAppBackupFile.Metadata && handle.token == now) {
metadata = handle
} else if (handle is FileBackupFileType.Snapshot && handle.time == now) {
snapshot = handle
fileSnapshot = handle
}
}
assertNotNull(metadata)
assertNotNull(snapshot)
assertNotNull(fileSnapshot)
assertContentEquals(bytes1, plugin.load(metadata as FileHandle).readAllBytes())
assertContentEquals(bytes2, plugin.load(snapshot as FileHandle).readAllBytes())
assertContentEquals(bytes1, backend.load(metadata as FileHandle).readAllBytes())
assertContentEquals(bytes2, backend.load(fileSnapshot as FileHandle).readAllBytes())
val blobName = Random.nextBytes(32).toHexString()
var blob: FileBackupFileType.Blob? = null
val bytes3 = Random.nextBytes(1337 * 16)
plugin.save(FileBackupFileType.Blob(androidId, blobName)).use {
backend.save(FileBackupFileType.Blob(androidId, blobName)).use {
it.write(bytes3)
}
plugin.list(
backend.list(
null,
FileBackupFileType.Snapshot::class,
FileBackupFileType.Blob::class,
@ -72,32 +72,75 @@ public abstract class BackendTest {
}
}
assertNotNull(blob)
assertContentEquals(bytes3, plugin.load(blob as FileHandle).readAllBytes())
assertContentEquals(bytes3, backend.load(blob as FileHandle).readAllBytes())
// try listing with top-level folder, should find two files of FileBackupFileType in there
var numFiles = 0
plugin.list(
snapshot!!.topLevelFolder,
backend.list(
fileSnapshot!!.topLevelFolder,
FileBackupFileType.Snapshot::class,
FileBackupFileType.Blob::class,
LegacyAppBackupFile.Metadata::class,
) { numFiles++ }
assertEquals(2, numFiles)
plugin.remove(snapshot as FileHandle)
val repoId = Random.nextBytes(32).toHexString()
val snapshotId = Random.nextBytes(32).toHexString()
val blobId = Random.nextBytes(32).toHexString()
val bytes4 = Random.nextBytes(1337)
val bytes5 = Random.nextBytes(1337 * 8)
backend.save(AppBackupFileType.Snapshot(repoId, snapshotId)).use {
it.write(bytes4)
}
var appSnapshot: AppBackupFileType.Snapshot? = null
backend.list(
null,
AppBackupFileType.Snapshot::class,
) { fileInfo ->
val handle = fileInfo.fileHandle
if (handle is AppBackupFileType.Snapshot) {
appSnapshot = handle
}
}
assertNotNull(appSnapshot)
assertContentEquals(bytes4, backend.load(appSnapshot as FileHandle).readAllBytes())
backend.save(AppBackupFileType.Blob(repoId, blobId)).use {
it.write(bytes5)
}
var blobHandle: AppBackupFileType.Blob? = null
backend.list(
TopLevelFolder(repoId),
AppBackupFileType.Blob::class,
LegacyAppBackupFile.Metadata::class,
) { fileInfo ->
val handle = fileInfo.fileHandle
if (handle is AppBackupFileType.Blob) {
blobHandle = handle
}
}
assertNotNull(blobHandle)
assertContentEquals(bytes5, backend.load(blobHandle as FileHandle).readAllBytes())
backend.remove(fileSnapshot as FileHandle)
backend.remove(appSnapshot as FileHandle)
backend.remove(blobHandle as FileHandle)
// rename snapshots
val snapshotNewFolder = TopLevelFolder("a123456789abcdef.sv")
plugin.rename(snapshot!!.topLevelFolder, snapshotNewFolder)
backend.rename(fileSnapshot!!.topLevelFolder, snapshotNewFolder)
// rename to existing folder should fail
val e = assertFailsWith<Exception> {
plugin.rename(snapshotNewFolder, metadata!!.topLevelFolder)
backend.rename(snapshotNewFolder, metadata!!.topLevelFolder)
}
println(e)
plugin.remove(metadata!!.topLevelFolder)
plugin.remove(snapshotNewFolder)
backend.remove(metadata!!.topLevelFolder)
backend.remove(snapshotNewFolder)
}
protected suspend fun testRemoveCreateWriteFile() {
@ -105,14 +148,14 @@ public abstract class BackendTest {
val blob = LegacyAppBackupFile.Blob(now, Random.nextBytes(32).toHexString())
val bytes = Random.nextBytes(2342)
plugin.remove(blob)
backend.remove(blob)
try {
plugin.save(blob).use {
backend.save(blob).use {
it.write(bytes)
}
assertContentEquals(bytes, plugin.load(blob as FileHandle).readAllBytes())
assertContentEquals(bytes, backend.load(blob as FileHandle).readAllBytes())
} finally {
plugin.remove(blob)
backend.remove(blob)
}
}

View file

@ -10,12 +10,18 @@ public object Constants {
public const val DIRECTORY_ROOT: String = ".SeedVaultAndroidBackup"
internal const val FILE_BACKUP_METADATA = ".backup.metadata"
internal const val FILE_BACKUP_ICONS = ".backup.icons"
public val tokenRegex: Regex = Regex("([0-9]{13})") // good until the year 2286
public const val SNAPSHOT_EXT: String = ".SeedSnap"
public val folderRegex: Regex = Regex("^[a-f0-9]{16}\\.sv$")
public val chunkFolderRegex: Regex = Regex("[a-f0-9]{2}")
public val chunkRegex: Regex = Regex("[a-f0-9]{64}")
public val snapshotRegex: Regex = Regex("([0-9]{13})\\.SeedSnap") // good until the year 2286
public val tokenRegex: Regex = Regex("^([0-9]{13})$") // good until the year 2286
public const val APP_SNAPSHOT_EXT: String = ".snapshot"
public const val FILE_SNAPSHOT_EXT: String = ".SeedSnap"
public val repoIdRegex: Regex = Regex("^[a-f0-9]{64}$")
public val fileFolderRegex: Regex = Regex("^[a-f0-9]{16}\\.sv$")
public val chunkFolderRegex: Regex = Regex("^[a-f0-9]{2}$")
public val blobFolderRegex: Regex = chunkFolderRegex
public val chunkRegex: Regex = repoIdRegex
public val blobRegex: Regex = repoIdRegex
// good until year 2286
public val appSnapshotRegex: Regex = Regex("(^[a-f0-9]{64})\\.snapshot$")
public val fileSnapshotRegex: Regex = Regex("(^[0-9]{13})\\.SeedSnap$") // good until year 2286
public const val MIME_TYPE: String = "application/octet-stream"
public const val CHUNK_FOLDER_COUNT: Int = 256

View file

@ -5,9 +5,10 @@
package org.calyxos.seedvault.core.backends
import org.calyxos.seedvault.core.backends.Constants.APP_SNAPSHOT_EXT
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_ICONS
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA
import org.calyxos.seedvault.core.backends.Constants.SNAPSHOT_EXT
import org.calyxos.seedvault.core.backends.Constants.FILE_SNAPSHOT_EXT
public sealed class FileHandle {
public abstract val name: String
@ -68,11 +69,32 @@ public sealed class FileBackupFileType : FileHandle() {
override val androidId: String,
val time: Long,
) : FileBackupFileType() {
override val name: String = "$time$SNAPSHOT_EXT"
override val name: String = "$time$FILE_SNAPSHOT_EXT"
override val relativePath: String get() = "$androidId.sv/$name"
}
}
public sealed class AppBackupFileType : FileHandle() {
public abstract val repoId: String
public val topLevelFolder: TopLevelFolder get() = TopLevelFolder(repoId)
public data class Blob(
override val repoId: String,
override val name: String,
) : AppBackupFileType() {
override val relativePath: String get() = "$repoId/${name.substring(0, 2)}/$name"
}
public data class Snapshot(
override val repoId: String,
val hash: String,
) : AppBackupFileType() {
override val name: String = "$hash$APP_SNAPSHOT_EXT"
override val relativePath: String get() = "$repoId/$name"
}
}
public data class FileInfo(
val fileHandle: FileHandle,
val size: Long,

View file

@ -7,6 +7,7 @@ package org.calyxos.seedvault.core.backends.saf
import android.content.Context
import androidx.documentfile.provider.DocumentFile
import org.calyxos.seedvault.core.backends.AppBackupFileType
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.FileHandle
import org.calyxos.seedvault.core.backends.LegacyAppBackupFile
@ -32,9 +33,16 @@ internal class DocumentFileCache(
getRootFile().getOrCreateDirectory(context, fh.name)
}
is LegacyAppBackupFile -> cache.getOrPut("$root/${fh.relativePath}") {
getOrCreateFile(fh.topLevelFolder).getOrCreateFile(context, fh.name)
}
is AppBackupFileType.Blob -> {
val subFolderName = fh.name.substring(0, 2)
cache.getOrPut("$root/${fh.topLevelFolder.name}/$subFolderName") {
getOrCreateFile(fh.topLevelFolder).getOrCreateDirectory(context, subFolderName)
}.getOrCreateFile(context, fh.name)
}
is AppBackupFileType.Snapshot -> {
getOrCreateFile(fh.topLevelFolder).getOrCreateFile(context, fh.name)
}
is FileBackupFileType.Blob -> {
val subFolderName = fh.name.substring(0, 2)
@ -46,6 +54,10 @@ internal class DocumentFileCache(
is FileBackupFileType.Snapshot -> {
getOrCreateFile(fh.topLevelFolder).getOrCreateFile(context, fh.name)
}
is LegacyAppBackupFile -> cache.getOrPut("$root/${fh.relativePath}") {
getOrCreateFile(fh.topLevelFolder).getOrCreateFile(context, fh.name)
}
}
internal suspend fun getFile(fh: FileHandle): DocumentFile? = when (fh) {
@ -53,7 +65,14 @@ internal class DocumentFileCache(
getRootFile().findFileBlocking(context, fh.name)
}
is LegacyAppBackupFile -> cache.getOrElse("$root/${fh.relativePath}") {
is AppBackupFileType.Blob -> {
val subFolderName = fh.name.substring(0, 2)
cache.getOrElse("$root/${fh.topLevelFolder.name}/$subFolderName") {
getFile(fh.topLevelFolder)?.findFileBlocking(context, subFolderName)
}?.findFileBlocking(context, fh.name)
}
is AppBackupFileType.Snapshot -> {
getFile(fh.topLevelFolder)?.findFileBlocking(context, fh.name)
}
@ -67,6 +86,10 @@ internal class DocumentFileCache(
is FileBackupFileType.Snapshot -> {
getFile(fh.topLevelFolder)?.findFileBlocking(context, fh.name)
}
is LegacyAppBackupFile -> cache.getOrElse("$root/${fh.relativePath}") {
getFile(fh.topLevelFolder)?.findFileBlocking(context, fh.name)
}
}
internal fun removeFromCache(fh: FileHandle) {

View file

@ -16,13 +16,18 @@ import androidx.core.database.getIntOrNull
import androidx.documentfile.provider.DocumentFile
import io.github.oshai.kotlinlogging.KLogger
import io.github.oshai.kotlinlogging.KotlinLogging
import org.calyxos.seedvault.core.backends.AppBackupFileType
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.Constants.DIRECTORY_ROOT
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA
import org.calyxos.seedvault.core.backends.Constants.appSnapshotRegex
import org.calyxos.seedvault.core.backends.Constants.blobFolderRegex
import org.calyxos.seedvault.core.backends.Constants.blobRegex
import org.calyxos.seedvault.core.backends.Constants.chunkFolderRegex
import org.calyxos.seedvault.core.backends.Constants.chunkRegex
import org.calyxos.seedvault.core.backends.Constants.folderRegex
import org.calyxos.seedvault.core.backends.Constants.snapshotRegex
import org.calyxos.seedvault.core.backends.Constants.fileFolderRegex
import org.calyxos.seedvault.core.backends.Constants.fileSnapshotRegex
import org.calyxos.seedvault.core.backends.Constants.repoIdRegex
import org.calyxos.seedvault.core.backends.Constants.tokenRegex
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.FileHandle
@ -103,7 +108,7 @@ public class SafBackend(
if (LegacyAppBackupFile.IconsFile::class in fileTypes) throw UnsupportedOperationException()
if (LegacyAppBackupFile.Blob::class in fileTypes) throw UnsupportedOperationException()
log.debugLog { "list($topLevelFolder, $fileTypes)" }
log.debugLog { "list($topLevelFolder, ${fileTypes.map { it.simpleName }})" }
val folder = if (topLevelFolder == null) {
cache.getRootFile()
@ -111,23 +116,45 @@ public class SafBackend(
cache.getOrCreateFile(topLevelFolder)
}
// limit depth based on wanted types and if top-level folder is given
var depth = if (FileBackupFileType.Blob::class in fileTypes) 3 else 2
var depth = if (FileBackupFileType.Blob::class in fileTypes ||
AppBackupFileType.Blob::class in fileTypes
) 3 else 2
if (topLevelFolder != null) depth -= 1
folder.listFilesRecursive(depth) { file ->
if (!file.isFile) return@listFilesRecursive
val parentName = file.parentFile?.name ?: return@listFilesRecursive
val name = file.name ?: return@listFilesRecursive
if (LegacyAppBackupFile.Metadata::class in fileTypes && name == FILE_BACKUP_METADATA &&
parentName.matches(tokenRegex)
if (AppBackupFileType.Snapshot::class in fileTypes ||
AppBackupFileType::class in fileTypes
) {
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
callback(FileInfo(metadata, file.length()))
val match = appSnapshotRegex.matchEntire(name)
if (match != null && repoIdRegex.matches(parentName)) {
val snapshot = AppBackupFileType.Snapshot(
repoId = parentName,
hash = match.groupValues[1],
)
callback(FileInfo(snapshot, file.length()))
}
}
if ((AppBackupFileType.Blob::class in fileTypes ||
AppBackupFileType::class in fileTypes)
) {
val repoId = file.parentFile?.parentFile?.name ?: ""
if (repoIdRegex.matches(repoId) && blobFolderRegex.matches(parentName)) {
if (blobRegex.matches(name)) {
val blob = AppBackupFileType.Blob(
repoId = repoId,
name = name,
)
callback(FileInfo(blob, file.length()))
}
}
}
if (FileBackupFileType.Snapshot::class in fileTypes ||
FileBackupFileType::class in fileTypes
) {
val match = snapshotRegex.matchEntire(name)
val match = fileSnapshotRegex.matchEntire(name)
if (match != null) {
val snapshot = FileBackupFileType.Snapshot(
androidId = parentName.substringBefore('.'),
@ -140,7 +167,7 @@ public class SafBackend(
FileBackupFileType::class in fileTypes)
) {
val androidIdSv = file.parentFile?.parentFile?.name ?: ""
if (folderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) {
if (fileFolderRegex.matches(androidIdSv) && chunkFolderRegex.matches(parentName)) {
if (chunkRegex.matches(name)) {
val blob = FileBackupFileType.Blob(
androidId = androidIdSv.substringBefore('.'),
@ -150,6 +177,12 @@ public class SafBackend(
}
}
}
if (LegacyAppBackupFile.Metadata::class in fileTypes && name == FILE_BACKUP_METADATA &&
parentName.matches(tokenRegex)
) {
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
callback(FileInfo(metadata, file.length()))
}
}
}
@ -173,16 +206,18 @@ public class SafBackend(
}
override suspend fun rename(from: TopLevelFolder, to: TopLevelFolder) {
log.debugLog { "rename($from, ${to.name})" }
val toName = to.name // querying name is expensive
log.debugLog { "rename($from, $toName)" }
val fromFile = cache.getOrCreateFile(from)
// don't use fromFile.renameTo(to.name) as that creates "${to.name} (1)"
val newUri = renameDocument(context.contentResolver, fromFile.uri, to.name)
val newUri = renameDocument(context.contentResolver, fromFile.uri, toName)
?: throw IOException("could not rename ${from.relativePath}")
cache.removeFromCache(from) // after renaming cached file isn't valid anymore
val toFile = DocumentFile.fromTreeUri(context, newUri)
?: throw IOException("renamed URI invalid: $newUri")
if (toFile.name != to.name) {
if (toFile.name != toName) {
toFile.delete()
throw IOException("renamed to ${toFile.name}, but expected ${to.name}")
throw IOException("renamed to ${toFile.name}, but expected $toName")
}
}

View file

@ -26,13 +26,17 @@ import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.RequestBody
import okio.BufferedSink
import org.calyxos.seedvault.core.backends.AppBackupFileType
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.Constants.DIRECTORY_ROOT
import org.calyxos.seedvault.core.backends.Constants.FILE_BACKUP_METADATA
import org.calyxos.seedvault.core.backends.Constants.appSnapshotRegex
import org.calyxos.seedvault.core.backends.Constants.blobFolderRegex
import org.calyxos.seedvault.core.backends.Constants.chunkFolderRegex
import org.calyxos.seedvault.core.backends.Constants.chunkRegex
import org.calyxos.seedvault.core.backends.Constants.folderRegex
import org.calyxos.seedvault.core.backends.Constants.snapshotRegex
import org.calyxos.seedvault.core.backends.Constants.fileFolderRegex
import org.calyxos.seedvault.core.backends.Constants.fileSnapshotRegex
import org.calyxos.seedvault.core.backends.Constants.repoIdRegex
import org.calyxos.seedvault.core.backends.Constants.tokenRegex
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.FileHandle
@ -180,7 +184,9 @@ public class WebDavBackend(
if (LegacyAppBackupFile.Blob::class in fileTypes) throw UnsupportedOperationException()
// limit depth based on wanted types and if top-level folder is given
var depth = if (FileBackupFileType.Blob::class in fileTypes) 3 else 2
var depth = if (FileBackupFileType.Blob::class in fileTypes ||
AppBackupFileType.Blob::class in fileTypes
) 3 else 2
if (topLevelFolder != null) depth -= 1
val location = if (topLevelFolder == null) {
@ -204,19 +210,40 @@ public class WebDavBackend(
val name = response.hrefName()
val parentName = response.href.pathSegments[response.href.pathSegments.size - 2]
if (LegacyAppBackupFile.Metadata::class in fileTypes) {
if (name == FILE_BACKUP_METADATA && parentName.matches(tokenRegex)) {
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
if (AppBackupFileType.Snapshot::class in fileTypes ||
AppBackupFileType::class in fileTypes
) {
val match = appSnapshotRegex.matchEntire(name)
if (match != null && repoIdRegex.matches(parentName)) {
val size = response.properties.contentLength()
callback(FileInfo(metadata, size))
// we can find .backup.metadata files, so no need for nginx workaround
tokenFolders.clear()
val snapshot = AppBackupFileType.Snapshot(
repoId = parentName,
hash = match.groupValues[1],
)
callback(FileInfo(snapshot, size))
}
}
if ((AppBackupFileType.Blob::class in fileTypes ||
AppBackupFileType::class in fileTypes) && response.href.pathSize >= 3
) {
val repoId = response.href.pathSegments[response.href.pathSegments.size - 3]
if (repoIdRegex.matches(repoId) &&
blobFolderRegex.matches(parentName)
) {
if (chunkRegex.matches(name)) {
val blob = AppBackupFileType.Blob(
repoId = repoId,
name = name,
)
val size = response.properties.contentLength()
callback(FileInfo(blob, size))
}
}
}
if (FileBackupFileType.Snapshot::class in fileTypes ||
FileBackupFileType::class in fileTypes
) {
val match = snapshotRegex.matchEntire(name)
val match = fileSnapshotRegex.matchEntire(name)
if (match != null) {
val size = response.properties.contentLength()
val snapshot = FileBackupFileType.Snapshot(
@ -231,7 +258,7 @@ public class WebDavBackend(
) {
val androidIdSv =
response.href.pathSegments[response.href.pathSegments.size - 3]
if (folderRegex.matches(androidIdSv) &&
if (fileFolderRegex.matches(androidIdSv) &&
chunkFolderRegex.matches(parentName)
) {
if (chunkRegex.matches(name)) {
@ -244,6 +271,15 @@ public class WebDavBackend(
}
}
}
if (LegacyAppBackupFile.Metadata::class in fileTypes) {
if (name == FILE_BACKUP_METADATA && parentName.matches(tokenRegex)) {
val metadata = LegacyAppBackupFile.Metadata(parentName.toLong())
val size = response.properties.contentLength()
callback(FileInfo(metadata, size))
// we can find .backup.metadata files, so no need for nginx workaround
tokenFolders.clear()
}
}
}
}
} catch (e: NotFoundException) {

View file

@ -11,7 +11,7 @@ import org.calyxos.seedvault.core.backends.BackendTest
import kotlin.test.Test
public class WebDavBackendTest : BackendTest() {
override val plugin: Backend = WebDavBackend(WebDavTestConfig.getConfig(), ".SeedvaultTest")
override val backend: Backend = WebDavBackend(WebDavTestConfig.getConfig(), ".SeedvaultTest")
@Test
public fun `test write, list, read, rename, delete`(): Unit = runBlocking {