From f51c758493bff392874540a3723ec25c82684b5d Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Fri, 16 Aug 2024 17:06:28 -0300 Subject: [PATCH] don't restore files that still exist unchanged (same size and lastModified) --- .../storage/restore/AbstractChunkRestore.kt | 2 - .../backup/storage/restore/FileRestore.kt | 70 +++++++++++-------- .../backup/storage/scanner/DocumentScanner.kt | 2 +- .../backup/storage/scanner/FileScanner.kt | 2 - .../backup/storage/scanner/MediaScanner.kt | 33 +++++++++ 5 files changed, 76 insertions(+), 33 deletions(-) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt index 9c207629..221aed56 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/AbstractChunkRestore.kt @@ -46,8 +46,6 @@ internal abstract class AbstractChunkRestore( tag: String, streamWriter: suspend (outputStream: OutputStream) -> Long, ) { - // TODO check if the file exists already (same name, size, chunk IDs) - // and skip it in this case fileRestore.restoreFile(file, observer, tag, streamWriter) } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/FileRestore.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/FileRestore.kt index b9519778..4084d73c 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/restore/FileRestore.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/restore/FileRestore.kt @@ -24,7 +24,6 @@ import kotlin.random.Random private const val TAG = "FileRestore" -@Suppress("BlockingMethodInNonBlockingContext") internal class FileRestore( private val context: Context, private val mediaScanner: MediaScanner, @@ -46,10 +45,12 @@ internal class FileRestore( bytes = restoreFile(file.mediaFile, streamWriter) finalTag = "M$tag" } + file.docFile != null -> { bytes = restoreFile(file, streamWriter) finalTag = "D$tag" } + else -> { error("unexpected file: $file") } @@ -63,39 +64,45 @@ internal class FileRestore( streamWriter: suspend (outputStream: OutputStream) -> Long, ): Long { // ensure directory exists - @Suppress("DEPRECATION") val dir = File("${getExternalStorageDirectory()}/${docFile.dir}") if (!dir.mkdirs() && !dir.isDirectory) { throw IOException("Could not create ${dir.absolutePath}") } - // find non-existing file-name var file = File(dir, docFile.name) - var i = 0 - // we don't support existing files, but at least don't overwrite them when they do exist - while (file.exists()) { - i++ - val lastDot = docFile.name.lastIndexOf('.') - val newName = if (lastDot == -1) "${docFile.name} ($i)" - else docFile.name.replaceRange(lastDot..lastDot, " ($i).") - file = File(dir, newName) - } - val bytesWritten = try { - // copy chunk(s) into file - file.outputStream().use { outputStream -> - streamWriter(outputStream) + // TODO should we also calculate and check the chunk IDs? + if (file.isFile && file.length() == docFile.size && + file.lastModified() == docFile.lastModified + ) { + Log.i(TAG, "Not restoring $file, already there unchanged.") + return file.length() // not restoring existing file with same length and date + } else { + var i = 0 + // don't overwrite existing files, if they exist + while (file.exists()) { // find non-existing file-name + i++ + val lastDot = docFile.name.lastIndexOf('.') + val newName = if (lastDot == -1) "${docFile.name} ($i)" + else docFile.name.replaceRange(lastDot..lastDot, " ($i).") + file = File(dir, newName) } - } catch (e: IOException) { - file.delete() - throw e + val bytesWritten = try { + // copy chunk(s) into file + file.outputStream().use { outputStream -> + streamWriter(outputStream) + } + } catch (e: IOException) { + file.delete() + throw e + } + // re-set lastModified timestamp + file.setLastModified(docFile.lastModified ?: 0) + + // This might be a media file, so do we need to index it. + // Otherwise things like a wrong size of 0 bytes in MediaStore can happen. + indexFile(file) + + return bytesWritten } - // re-set lastModified timestamp - file.setLastModified(docFile.lastModified ?: 0) - - // This might be a media file, so do we need to index it. - // Otherwise things like a wrong size of 0 bytes in MediaStore can happen. - indexFile(file) - - return bytesWritten } @Throws(IOException::class) @@ -103,6 +110,14 @@ internal class FileRestore( mediaFile: BackupMediaFile, streamWriter: suspend (outputStream: OutputStream) -> Long, ): Long { + // TODO should we also calculate and check the chunk IDs? + if (mediaScanner.existsMediaFileUnchanged(mediaFile)) { + Log.i( + TAG, + "Not restoring ${mediaFile.path}/${mediaFile.name}, already there unchanged." + ) + return mediaFile.size + } // Insert pending media item into MediaStore val contentValues = ContentValues().apply { put(MediaColumns.DISPLAY_NAME, mediaFile.name) @@ -143,7 +158,6 @@ internal class FileRestore( } private fun setLastModifiedOnMediaFile(mediaFile: BackupMediaFile, uri: Uri) { - @Suppress("DEPRECATION") val extDir = getExternalStorageDirectory() // re-set lastModified as we can't use the MediaStore for this (read-only property) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/DocumentScanner.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/DocumentScanner.kt index 1177e8e3..85e93718 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/DocumentScanner.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/DocumentScanner.kt @@ -56,7 +56,7 @@ public class DocumentScanner(context: Context) { queryUri, PROJECTION, null, null, null ) val documentFiles = ArrayList(cursor?.count ?: 0) - cursor?.use { it -> + cursor?.use { while (it.moveToNext()) { val id = it.getString(PROJECTION_ID) val documentUri = DocumentsContract.buildDocumentUriUsingTree(uri, id) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/FileScanner.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/FileScanner.kt index a8c86583..cc99602f 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/FileScanner.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/FileScanner.kt @@ -16,7 +16,6 @@ import org.calyxos.backup.storage.content.DocFile import org.calyxos.backup.storage.content.MediaFile import org.calyxos.backup.storage.db.UriStore import org.calyxos.backup.storage.measure -import kotlin.time.ExperimentalTime internal class FileScannerResult( val smallFiles: List, @@ -36,7 +35,6 @@ internal class FileScanner( private const val FILES_LARGE = "large" } - @OptIn(ExperimentalTime::class) fun getFiles(): FileScannerResult { // scan both APIs val mediaFiles = ArrayList() diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/MediaScanner.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/MediaScanner.kt index c2ece36e..bf9c4b5f 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/MediaScanner.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/scanner/MediaScanner.kt @@ -6,6 +6,7 @@ package org.calyxos.backup.storage.scanner import android.content.ContentResolver.QUERY_ARG_SQL_SELECTION +import android.content.ContentResolver.QUERY_ARG_SQL_SELECTION_ARGS import android.content.ContentUris import android.content.Context import android.database.Cursor @@ -21,6 +22,7 @@ import androidx.core.database.getLongOrNull import androidx.core.database.getStringOrNull import org.calyxos.backup.storage.api.BackupFile import org.calyxos.backup.storage.api.MediaType +import org.calyxos.backup.storage.backup.BackupMediaFile import org.calyxos.backup.storage.content.MediaFile import java.io.File @@ -79,6 +81,37 @@ public class MediaScanner(context: Context) { } } + internal fun existsMediaFileUnchanged(mediaFile: BackupMediaFile): Boolean { + val uri = MediaType.fromBackupMediaType(mediaFile.type).contentUri + val extras = Bundle().apply { + // search for files with same path and name + val query = StringBuilder().apply { + append("${MediaStore.MediaColumns.MIME_TYPE}!='$MIME_TYPE_DIR'") + append(" AND ") + append("${MediaStore.MediaColumns.RELATIVE_PATH}=?") + append(" AND ") + append("${MediaStore.MediaColumns.DISPLAY_NAME}=?") + } + putString(QUERY_ARG_SQL_SELECTION, query.toString()) + val args = arrayOf( + mediaFile.path + "/", // Note trailing slash that is important + mediaFile.name, + ) + putStringArray(QUERY_ARG_SQL_SELECTION_ARGS, args) + } + + contentResolver.query(uri, PROJECTION, extras, null)?.use { c -> + while (c.moveToNext()) { + val f = createMediaFile(c, uri) + // note that we get seconds, but store milliseconds + if (f.dateModified == mediaFile.lastModified / 1000 && f.size == mediaFile.size) { + return true + } + } + } + return false + } + internal fun getPath(uri: Uri): String? { val projection = arrayOf( MediaStore.MediaColumns.RELATIVE_PATH,