Implement pruning of old snapshots and unused blobs
This happens regularly after each successful backup.
This commit is contained in:
parent
307ccf57de
commit
a1baa6f9d2
8 changed files with 531 additions and 6 deletions
|
@ -14,6 +14,7 @@ import androidx.work.WorkInfo.State.RUNNING
|
||||||
import androidx.work.WorkManager
|
import androidx.work.WorkManager
|
||||||
import com.stevesoltys.seedvault.storage.StorageBackupService
|
import com.stevesoltys.seedvault.storage.StorageBackupService
|
||||||
import com.stevesoltys.seedvault.transport.ConfigurableBackupTransportService
|
import com.stevesoltys.seedvault.transport.ConfigurableBackupTransportService
|
||||||
|
import com.stevesoltys.seedvault.worker.AppBackupPruneWorker
|
||||||
import com.stevesoltys.seedvault.worker.AppBackupWorker.Companion.UNIQUE_WORK_NAME
|
import com.stevesoltys.seedvault.worker.AppBackupWorker.Companion.UNIQUE_WORK_NAME
|
||||||
import kotlinx.coroutines.flow.Flow
|
import kotlinx.coroutines.flow.Flow
|
||||||
import kotlinx.coroutines.flow.combine
|
import kotlinx.coroutines.flow.combine
|
||||||
|
@ -31,14 +32,18 @@ class BackupStateManager(
|
||||||
flow = ConfigurableBackupTransportService.isRunning,
|
flow = ConfigurableBackupTransportService.isRunning,
|
||||||
flow2 = StorageBackupService.isRunning,
|
flow2 = StorageBackupService.isRunning,
|
||||||
flow3 = workManager.getWorkInfosForUniqueWorkFlow(UNIQUE_WORK_NAME),
|
flow3 = workManager.getWorkInfosForUniqueWorkFlow(UNIQUE_WORK_NAME),
|
||||||
) { appBackupRunning, filesBackupRunning, workInfos ->
|
flow4 = workManager.getWorkInfosForUniqueWorkFlow(AppBackupPruneWorker.UNIQUE_WORK_NAME),
|
||||||
val workInfoState = workInfos.getOrNull(0)?.state
|
) { appBackupRunning, filesBackupRunning, workInfo1, workInfo2 ->
|
||||||
|
val workInfoState1 = workInfo1.getOrNull(0)?.state
|
||||||
|
val workInfoState2 = workInfo2.getOrNull(0)?.state
|
||||||
Log.i(
|
Log.i(
|
||||||
TAG, "appBackupRunning: $appBackupRunning, " +
|
TAG, "appBackupRunning: $appBackupRunning, " +
|
||||||
"filesBackupRunning: $filesBackupRunning, " +
|
"filesBackupRunning: $filesBackupRunning, " +
|
||||||
"workInfoState: ${workInfoState?.name}"
|
"appBackupWorker: ${workInfoState1?.name}, " +
|
||||||
|
"pruneBackupWorker: ${workInfoState2?.name}"
|
||||||
)
|
)
|
||||||
appBackupRunning || filesBackupRunning || workInfoState == RUNNING
|
appBackupRunning || filesBackupRunning ||
|
||||||
|
workInfoState1 == RUNNING || workInfoState2 == RUNNING
|
||||||
}
|
}
|
||||||
|
|
||||||
val isAutoRestoreEnabled: Boolean
|
val isAutoRestoreEnabled: Boolean
|
||||||
|
|
116
app/src/main/java/com/stevesoltys/seedvault/repo/Pruner.kt
Normal file
116
app/src/main/java/com/stevesoltys/seedvault/repo/Pruner.kt
Normal file
|
@ -0,0 +1,116 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||||
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.stevesoltys.seedvault.repo
|
||||||
|
|
||||||
|
import com.stevesoltys.seedvault.backend.BackendManager
|
||||||
|
import com.stevesoltys.seedvault.crypto.Crypto
|
||||||
|
import com.stevesoltys.seedvault.proto.Snapshot
|
||||||
|
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||||
|
import org.calyxos.seedvault.core.backends.AppBackupFileType
|
||||||
|
import org.calyxos.seedvault.core.backends.TopLevelFolder
|
||||||
|
import java.time.LocalDate
|
||||||
|
import java.time.temporal.ChronoField
|
||||||
|
import java.time.temporal.TemporalAdjuster
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cleans up old backups data that we do not need to retain.
|
||||||
|
*/
|
||||||
|
internal class Pruner(
|
||||||
|
private val crypto: Crypto,
|
||||||
|
private val backendManager: BackendManager,
|
||||||
|
private val snapshotManager: SnapshotManager,
|
||||||
|
) {
|
||||||
|
|
||||||
|
private val log = KotlinLogging.logger {}
|
||||||
|
private val folder get() = TopLevelFolder(crypto.repoId)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keeps the last 3 daily and 2 weekly snapshots (this and last week), removes all others.
|
||||||
|
* Then removes all blobs from the backend
|
||||||
|
* that are not referenced anymore by remaining snapshots.
|
||||||
|
*/
|
||||||
|
suspend fun removeOldSnapshotsAndPruneUnusedBlobs() {
|
||||||
|
// get snapshots currently available on backend
|
||||||
|
val snapshotHandles = mutableListOf<AppBackupFileType.Snapshot>()
|
||||||
|
backendManager.backend.list(folder, AppBackupFileType.Snapshot::class) { fileInfo ->
|
||||||
|
snapshotHandles.add(fileInfo.fileHandle as AppBackupFileType.Snapshot)
|
||||||
|
}
|
||||||
|
// load and parse snapshots
|
||||||
|
val snapshotMap = mutableMapOf<Long, AppBackupFileType.Snapshot>()
|
||||||
|
val snapshots = mutableListOf<Snapshot>()
|
||||||
|
snapshotHandles.forEach { handle ->
|
||||||
|
val snapshot = snapshotManager.loadSnapshot(handle) // exception is allowed to bubble up
|
||||||
|
snapshotMap[snapshot.token] = handle
|
||||||
|
snapshots.add(snapshot)
|
||||||
|
}
|
||||||
|
// find out which snapshots to keep
|
||||||
|
val toKeep = getTokenToKeep(snapshotMap.keys)
|
||||||
|
log.info { "Found ${snapshots.size} snapshots, keeping ${toKeep.size}." }
|
||||||
|
// remove snapshots we aren't keeping
|
||||||
|
snapshotMap.forEach { (token, handle) ->
|
||||||
|
if (token !in toKeep) {
|
||||||
|
log.info { "Removing snapshot $token ${handle.name}" }
|
||||||
|
snapshotManager.removeSnapshot(handle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// prune unused blobs
|
||||||
|
val keptSnapshots = snapshots.filter { it.token in toKeep }
|
||||||
|
pruneUnusedBlobs(keptSnapshots)
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun pruneUnusedBlobs(snapshots: List<Snapshot>) {
|
||||||
|
val blobHandles = mutableListOf<AppBackupFileType.Blob>()
|
||||||
|
backendManager.backend.list(folder, AppBackupFileType.Blob::class) { fileInfo ->
|
||||||
|
blobHandles.add(fileInfo.fileHandle as AppBackupFileType.Blob)
|
||||||
|
}
|
||||||
|
val usedBlobIds = snapshots.flatMap { snapshot ->
|
||||||
|
snapshot.blobsMap.values.map { blob ->
|
||||||
|
blob.id.hexFromProto()
|
||||||
|
}
|
||||||
|
}.toSet()
|
||||||
|
blobHandles.forEach { blobHandle ->
|
||||||
|
if (blobHandle.name !in usedBlobIds) {
|
||||||
|
log.info { "Removing blob ${blobHandle.name}" }
|
||||||
|
backendManager.backend.remove(blobHandle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getTokenToKeep(tokenSet: Set<Long>): Set<Long> {
|
||||||
|
if (tokenSet.size <= 3) return tokenSet // keep at least 3 snapshots
|
||||||
|
val tokenList = tokenSet.sortedDescending()
|
||||||
|
val toKeep = mutableSetOf<Long>()
|
||||||
|
toKeep += getToKeep(tokenList, 3) // 3 daily
|
||||||
|
toKeep += getToKeep(tokenList, 2) { temporal -> // keep one from this and last week
|
||||||
|
temporal.with(ChronoField.DAY_OF_WEEK, 1)
|
||||||
|
}
|
||||||
|
// ensure we keep at least three snapshots
|
||||||
|
val tokenIterator = tokenList.iterator()
|
||||||
|
while (toKeep.size < 3 && tokenIterator.hasNext()) toKeep.add(tokenIterator.next())
|
||||||
|
return toKeep
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getToKeep(
|
||||||
|
tokenList: List<Long>,
|
||||||
|
keep: Int,
|
||||||
|
temporalAdjuster: TemporalAdjuster? = null,
|
||||||
|
): List<Long> {
|
||||||
|
val toKeep = mutableListOf<Long>()
|
||||||
|
if (keep == 0) return toKeep
|
||||||
|
var last: LocalDate? = null
|
||||||
|
for (token in tokenList) {
|
||||||
|
val date = LocalDate.ofEpochDay(token / 1000 / 60 / 60 / 24)
|
||||||
|
val period = if (temporalAdjuster == null) date else date.with(temporalAdjuster)
|
||||||
|
if (period != last) {
|
||||||
|
toKeep.add(token)
|
||||||
|
if (toKeep.size >= keep) break
|
||||||
|
last = period
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return toKeep
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -19,4 +19,5 @@ val repoModule = module {
|
||||||
SnapshotManager(snapshotFolder, get(), get(), get())
|
SnapshotManager(snapshotFolder, get(), get(), get())
|
||||||
}
|
}
|
||||||
factory { SnapshotCreatorFactory(androidContext(), get(), get(), get(), get()) }
|
factory { SnapshotCreatorFactory(androidContext(), get(), get(), get(), get()) }
|
||||||
|
factory { Pruner(get(), get(), get()) }
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,13 +40,14 @@ private const val CHANNEL_ID_SUCCESS = "NotificationBackupSuccess"
|
||||||
private const val CHANNEL_ID_ERROR = "NotificationError"
|
private const val CHANNEL_ID_ERROR = "NotificationError"
|
||||||
private const val CHANNEL_ID_RESTORE = "NotificationRestore"
|
private const val CHANNEL_ID_RESTORE = "NotificationRestore"
|
||||||
private const val CHANNEL_ID_RESTORE_ERROR = "NotificationRestoreError"
|
private const val CHANNEL_ID_RESTORE_ERROR = "NotificationRestoreError"
|
||||||
|
private const val CHANNEL_ID_PRUNING = "NotificationPruning"
|
||||||
internal const val NOTIFICATION_ID_OBSERVER = 1
|
internal const val NOTIFICATION_ID_OBSERVER = 1
|
||||||
private const val NOTIFICATION_ID_SUCCESS = 2
|
private const val NOTIFICATION_ID_SUCCESS = 2
|
||||||
private const val NOTIFICATION_ID_ERROR = 3
|
private const val NOTIFICATION_ID_ERROR = 3
|
||||||
private const val NOTIFICATION_ID_SPACE_ERROR = 4
|
private const val NOTIFICATION_ID_SPACE_ERROR = 4
|
||||||
internal const val NOTIFICATION_ID_RESTORE = 5
|
internal const val NOTIFICATION_ID_RESTORE = 5
|
||||||
private const val NOTIFICATION_ID_RESTORE_ERROR = 6
|
private const val NOTIFICATION_ID_RESTORE_ERROR = 6
|
||||||
private const val NOTIFICATION_ID_BACKGROUND = 7
|
internal const val NOTIFICATION_ID_PRUNING = 7
|
||||||
private const val NOTIFICATION_ID_NO_MAIN_KEY_ERROR = 8
|
private const val NOTIFICATION_ID_NO_MAIN_KEY_ERROR = 8
|
||||||
|
|
||||||
private val TAG = BackupNotificationManager::class.java.simpleName
|
private val TAG = BackupNotificationManager::class.java.simpleName
|
||||||
|
@ -59,6 +60,7 @@ internal class BackupNotificationManager(private val context: Context) {
|
||||||
createNotificationChannel(getErrorChannel())
|
createNotificationChannel(getErrorChannel())
|
||||||
createNotificationChannel(getRestoreChannel())
|
createNotificationChannel(getRestoreChannel())
|
||||||
createNotificationChannel(getRestoreErrorChannel())
|
createNotificationChannel(getRestoreErrorChannel())
|
||||||
|
createNotificationChannel(getPruningChannel())
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun getObserverChannel(): NotificationChannel {
|
private fun getObserverChannel(): NotificationChannel {
|
||||||
|
@ -90,6 +92,11 @@ internal class BackupNotificationManager(private val context: Context) {
|
||||||
return NotificationChannel(CHANNEL_ID_RESTORE_ERROR, title, IMPORTANCE_HIGH)
|
return NotificationChannel(CHANNEL_ID_RESTORE_ERROR, title, IMPORTANCE_HIGH)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun getPruningChannel(): NotificationChannel {
|
||||||
|
val title = context.getString(R.string.notification_pruning_channel_title)
|
||||||
|
return NotificationChannel(CHANNEL_ID_PRUNING, title, IMPORTANCE_LOW)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This should get called for each APK we are backing up.
|
* This should get called for each APK we are backing up.
|
||||||
*/
|
*/
|
||||||
|
@ -158,7 +165,6 @@ internal class BackupNotificationManager(private val context: Context) {
|
||||||
}
|
}
|
||||||
|
|
||||||
fun onServiceDestroyed() {
|
fun onServiceDestroyed() {
|
||||||
nm.cancel(NOTIFICATION_ID_BACKGROUND)
|
|
||||||
// Cancel left-over notifications that are still ongoing.
|
// Cancel left-over notifications that are still ongoing.
|
||||||
//
|
//
|
||||||
// We have seen a race condition where the service was taken down at the same time
|
// We have seen a race condition where the service was taken down at the same time
|
||||||
|
@ -288,6 +294,17 @@ internal class BackupNotificationManager(private val context: Context) {
|
||||||
nm.cancel(NOTIFICATION_ID_RESTORE_ERROR)
|
nm.cancel(NOTIFICATION_ID_RESTORE_ERROR)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun getPruningNotification(): Notification {
|
||||||
|
return Builder(context, CHANNEL_ID_OBSERVER).apply {
|
||||||
|
setSmallIcon(R.drawable.ic_auto_delete)
|
||||||
|
setContentTitle(context.getString(R.string.notification_pruning_title))
|
||||||
|
setOngoing(true)
|
||||||
|
setShowWhen(false)
|
||||||
|
priority = PRIORITY_LOW
|
||||||
|
foregroundServiceBehavior = FOREGROUND_SERVICE_IMMEDIATE
|
||||||
|
}.build()
|
||||||
|
}
|
||||||
|
|
||||||
@SuppressLint("RestrictedApi")
|
@SuppressLint("RestrictedApi")
|
||||||
fun onNoMainKeyError() {
|
fun onNoMainKeyError() {
|
||||||
val intent = Intent(context, SettingsActivity::class.java)
|
val intent = Intent(context, SettingsActivity::class.java)
|
||||||
|
|
|
@ -23,6 +23,7 @@ import com.stevesoltys.seedvault.settings.SettingsManager
|
||||||
import com.stevesoltys.seedvault.repo.AppBackupManager
|
import com.stevesoltys.seedvault.repo.AppBackupManager
|
||||||
import com.stevesoltys.seedvault.transport.backup.PackageService
|
import com.stevesoltys.seedvault.transport.backup.PackageService
|
||||||
import com.stevesoltys.seedvault.repo.hexFromProto
|
import com.stevesoltys.seedvault.repo.hexFromProto
|
||||||
|
import com.stevesoltys.seedvault.worker.AppBackupPruneWorker
|
||||||
import com.stevesoltys.seedvault.worker.BackupRequester
|
import com.stevesoltys.seedvault.worker.BackupRequester
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import org.koin.core.component.KoinComponent
|
import org.koin.core.component.KoinComponent
|
||||||
|
@ -159,6 +160,10 @@ internal class NotificationBackupObserver(
|
||||||
}
|
}
|
||||||
} else 0L
|
} else 0L
|
||||||
nm.onBackupFinished(success, numPackagesToReport, total, size)
|
nm.onBackupFinished(success, numPackagesToReport, total, size)
|
||||||
|
if (success) {
|
||||||
|
// prune old backups
|
||||||
|
AppBackupPruneWorker.scheduleNow(context)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||||
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.stevesoltys.seedvault.worker
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
|
||||||
|
import android.util.Log
|
||||||
|
import androidx.work.BackoffPolicy
|
||||||
|
import androidx.work.CoroutineWorker
|
||||||
|
import androidx.work.ExistingWorkPolicy.REPLACE
|
||||||
|
import androidx.work.ForegroundInfo
|
||||||
|
import androidx.work.OneTimeWorkRequestBuilder
|
||||||
|
import androidx.work.OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST
|
||||||
|
import androidx.work.WorkManager
|
||||||
|
import androidx.work.WorkerParameters
|
||||||
|
import com.stevesoltys.seedvault.repo.Pruner
|
||||||
|
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
|
||||||
|
import com.stevesoltys.seedvault.ui.notification.NOTIFICATION_ID_PRUNING
|
||||||
|
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||||
|
import org.koin.core.component.KoinComponent
|
||||||
|
import org.koin.core.component.inject
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
class AppBackupPruneWorker(
|
||||||
|
appContext: Context,
|
||||||
|
workerParams: WorkerParameters,
|
||||||
|
) : CoroutineWorker(appContext, workerParams), KoinComponent {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private val TAG = AppBackupPruneWorker::class.simpleName
|
||||||
|
internal const val UNIQUE_WORK_NAME = "com.stevesoltys.seedvault.APP_BACKUP_PRUNE"
|
||||||
|
|
||||||
|
fun scheduleNow(context: Context) {
|
||||||
|
val workRequest = OneTimeWorkRequestBuilder<AppBackupPruneWorker>()
|
||||||
|
.setExpedited(RUN_AS_NON_EXPEDITED_WORK_REQUEST)
|
||||||
|
.setBackoffCriteria(BackoffPolicy.EXPONENTIAL, Duration.ofSeconds(10))
|
||||||
|
.build()
|
||||||
|
val workManager = WorkManager.getInstance(context)
|
||||||
|
Log.i(TAG, "Asking to prune app backups now...")
|
||||||
|
workManager.enqueueUniqueWork(UNIQUE_WORK_NAME, REPLACE, workRequest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val log = KotlinLogging.logger {}
|
||||||
|
private val pruner: Pruner by inject()
|
||||||
|
private val nm: BackupNotificationManager by inject()
|
||||||
|
|
||||||
|
override suspend fun doWork(): Result {
|
||||||
|
log.info { "Start worker $this ($id)" }
|
||||||
|
try {
|
||||||
|
setForeground(createForegroundInfo())
|
||||||
|
} catch (e: Exception) {
|
||||||
|
log.error(e) { "Error while running setForeground: " }
|
||||||
|
}
|
||||||
|
return try {
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
Result.success()
|
||||||
|
} catch (e: Exception) {
|
||||||
|
log.error(e) { "Error while pruning: " }
|
||||||
|
Result.retry()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun createForegroundInfo() = ForegroundInfo(
|
||||||
|
NOTIFICATION_ID_PRUNING,
|
||||||
|
nm.getPruningNotification(),
|
||||||
|
FOREGROUND_SERVICE_TYPE_DATA_SYNC,
|
||||||
|
)
|
||||||
|
}
|
|
@ -179,6 +179,9 @@
|
||||||
<string name="notification_error_no_main_key_title">Backups disabled</string>
|
<string name="notification_error_no_main_key_title">Backups disabled</string>
|
||||||
<string name="notification_error_no_main_key_text">Generate a new recovery code to complete upgrade and continue to use backups.</string>
|
<string name="notification_error_no_main_key_text">Generate a new recovery code to complete upgrade and continue to use backups.</string>
|
||||||
|
|
||||||
|
<string name="notification_pruning_channel_title">Removing old backups notification</string>
|
||||||
|
<string name="notification_pruning_title">Removing old backups…</string>
|
||||||
|
|
||||||
<!-- App Backup and Restore State -->
|
<!-- App Backup and Restore State -->
|
||||||
|
|
||||||
<string name="backup_section_system">System data</string>
|
<string name="backup_section_system">System data</string>
|
||||||
|
|
306
app/src/test/java/com/stevesoltys/seedvault/repo/PrunerTest.kt
Normal file
306
app/src/test/java/com/stevesoltys/seedvault/repo/PrunerTest.kt
Normal file
|
@ -0,0 +1,306 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2024 The Calyx Institute
|
||||||
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.stevesoltys.seedvault.repo
|
||||||
|
|
||||||
|
import com.stevesoltys.seedvault.backend.BackendManager
|
||||||
|
import com.stevesoltys.seedvault.getRandomByteArray
|
||||||
|
import com.stevesoltys.seedvault.proto.Snapshot
|
||||||
|
import com.stevesoltys.seedvault.proto.copy
|
||||||
|
import com.stevesoltys.seedvault.transport.TransportTest
|
||||||
|
import io.mockk.Runs
|
||||||
|
import io.mockk.coEvery
|
||||||
|
import io.mockk.coVerify
|
||||||
|
import io.mockk.every
|
||||||
|
import io.mockk.just
|
||||||
|
import io.mockk.mockk
|
||||||
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import org.calyxos.seedvault.core.backends.AppBackupFileType
|
||||||
|
import org.calyxos.seedvault.core.backends.Backend
|
||||||
|
import org.calyxos.seedvault.core.backends.FileInfo
|
||||||
|
import org.calyxos.seedvault.core.backends.TopLevelFolder
|
||||||
|
import org.calyxos.seedvault.core.toHexString
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import java.time.LocalDateTime
|
||||||
|
import java.time.ZoneOffset.UTC
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
internal class PrunerTest : TransportTest() {
|
||||||
|
|
||||||
|
private val backendManager: BackendManager = mockk()
|
||||||
|
private val snapshotManager: SnapshotManager = mockk()
|
||||||
|
private val backend: Backend = mockk()
|
||||||
|
|
||||||
|
private val pruner = Pruner(crypto, backendManager, snapshotManager)
|
||||||
|
private val folder = TopLevelFolder(repoId)
|
||||||
|
|
||||||
|
private val snapshotHandle1 =
|
||||||
|
AppBackupFileType.Snapshot(repoId, getRandomByteArray(32).toHexString())
|
||||||
|
private val snapshotHandle2 =
|
||||||
|
AppBackupFileType.Snapshot(repoId, getRandomByteArray(32).toHexString())
|
||||||
|
private val snapshotHandle3 =
|
||||||
|
AppBackupFileType.Snapshot(repoId, getRandomByteArray(32).toHexString())
|
||||||
|
private val snapshotHandle4 =
|
||||||
|
AppBackupFileType.Snapshot(repoId, getRandomByteArray(32).toHexString())
|
||||||
|
private val snapshotHandle5 =
|
||||||
|
AppBackupFileType.Snapshot(repoId, getRandomByteArray(32).toHexString())
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `single snapshot gets left alone`() = runBlocking {
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { token = System.currentTimeMillis() },
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
|
||||||
|
// we only find blobs that are in snapshots
|
||||||
|
expectLoadingBlobs(snapshot.blobsMap.values.map { it.id.hexFromProto() })
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `three snapshots from same day don't remove blobs`() = runBlocking {
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { token = System.currentTimeMillis() },
|
||||||
|
snapshotHandle2 to snapshot.copy { token = System.currentTimeMillis() - 1 },
|
||||||
|
snapshotHandle3 to snapshot.copy { token = System.currentTimeMillis() - 2 },
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
|
||||||
|
// we only find blobs that are in snapshots
|
||||||
|
expectLoadingBlobs(snapshot.blobsMap.values.map { it.id.hexFromProto() })
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `four snapshots from same day only remove oldest`() = runBlocking {
|
||||||
|
val now = System.currentTimeMillis()
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { token = now },
|
||||||
|
snapshotHandle2 to snapshot.copy { token = now - 1 },
|
||||||
|
snapshotHandle3 to snapshot.copy { token = now - 2 },
|
||||||
|
snapshotHandle4 to snapshot.copy { token = now - 3 },
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
|
||||||
|
// we only find blobs that are in snapshots
|
||||||
|
expectLoadingBlobs(snapshot.blobsMap.values.map { it.id.hexFromProto() })
|
||||||
|
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle4) } just Runs
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
|
||||||
|
coVerify { snapshotManager.removeSnapshot(snapshotHandle4) }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `three snapshots from different days remove blobs`() = runBlocking {
|
||||||
|
val now = System.currentTimeMillis()
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { token = now },
|
||||||
|
snapshotHandle2 to snapshot.copy { token = now - TimeUnit.DAYS.toMillis(1) },
|
||||||
|
snapshotHandle3 to snapshot.copy { token = now - TimeUnit.DAYS.toMillis(2) },
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
|
||||||
|
// we only find blobs that are in snapshots
|
||||||
|
val blob1 = getRandomByteArray(32).toHexString()
|
||||||
|
val blob2 = getRandomByteArray(32).toHexString()
|
||||||
|
val blobs = snapshot.blobsMap.values.map { it.id.hexFromProto() } + listOf(blob1, blob2)
|
||||||
|
expectLoadingBlobs(blobs)
|
||||||
|
|
||||||
|
// now extra blobs will get removed
|
||||||
|
coEvery { backend.remove(AppBackupFileType.Blob(repoId, blob1)) } just Runs
|
||||||
|
coEvery { backend.remove(AppBackupFileType.Blob(repoId, blob2)) } just Runs
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
|
||||||
|
coVerify {
|
||||||
|
backend.remove(AppBackupFileType.Blob(repoId, blob1))
|
||||||
|
backend.remove(AppBackupFileType.Blob(repoId, blob2))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `three snapshots from last weeks are still kept`() = runBlocking {
|
||||||
|
val now = System.currentTimeMillis()
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { token = now },
|
||||||
|
snapshotHandle2 to snapshot.copy { token = now - TimeUnit.DAYS.toMillis(7) },
|
||||||
|
snapshotHandle3 to snapshot.copy { token = now - TimeUnit.DAYS.toMillis(14) },
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
expectLoadingBlobs(emptyList())
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `five snapshots with two from last week, removes only one oldest`() = runBlocking {
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { // this week
|
||||||
|
token = LocalDateTime.of(2024, 9, 18, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle2 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 17, 20, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle3 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 16, 12, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle4 to snapshot.copy { // last week
|
||||||
|
token = LocalDateTime.of(2024, 9, 13, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle5 to snapshot.copy { // last week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 12, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
expectLoadingBlobs(emptyList())
|
||||||
|
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle5) } just Runs
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
|
||||||
|
coVerify {
|
||||||
|
snapshotManager.removeSnapshot(snapshotHandle5)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `five snapshots with one from last week, removes oldest from this week`() = runBlocking {
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { // this week
|
||||||
|
token = LocalDateTime.of(2024, 9, 19, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle2 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 18, 20, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle3 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 17, 12, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle4 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 16, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle5 to snapshot.copy { // last week
|
||||||
|
token = LocalDateTime.of(2024, 9, 12, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
expectLoadingBlobs(emptyList())
|
||||||
|
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle4) } just Runs
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
|
||||||
|
coVerify {
|
||||||
|
snapshotManager.removeSnapshot(snapshotHandle4)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `five snapshots with two on the same day, removes oldest from same day`() = runBlocking {
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy { // this week
|
||||||
|
token = LocalDateTime.of(2024, 9, 19, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle2 to snapshot.copy { // this week, same day
|
||||||
|
token = LocalDateTime.of(2024, 9, 19, 20, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle3 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 17, 12, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle4 to snapshot.copy { // this week, different day
|
||||||
|
token = LocalDateTime.of(2024, 9, 16, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle5 to snapshot.copy { // this week, same day
|
||||||
|
token = LocalDateTime.of(2024, 9, 16, 22, 0).toMillis()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
expectLoadingBlobs(emptyList())
|
||||||
|
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle2) } just Runs
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle5) } just Runs
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
|
||||||
|
coVerify {
|
||||||
|
snapshotManager.removeSnapshot(snapshotHandle2)
|
||||||
|
snapshotManager.removeSnapshot(snapshotHandle5)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `five snapshots from past weeks, removes two oldest weeks`() = runBlocking {
|
||||||
|
val snapshotMap = mapOf(
|
||||||
|
snapshotHandle1 to snapshot.copy {
|
||||||
|
token = LocalDateTime.of(2024, 9, 3, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle2 to snapshot.copy {
|
||||||
|
token = LocalDateTime.of(2024, 9, 10, 20, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle3 to snapshot.copy {
|
||||||
|
token = LocalDateTime.of(2024, 9, 17, 12, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle4 to snapshot.copy {
|
||||||
|
token = LocalDateTime.of(2024, 9, 24, 23, 0).toMillis()
|
||||||
|
},
|
||||||
|
snapshotHandle5 to snapshot.copy {
|
||||||
|
token = LocalDateTime.of(2024, 10, 1, 22, 0).toMillis()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expectLoadingSnapshots(snapshotMap)
|
||||||
|
expectLoadingBlobs(emptyList())
|
||||||
|
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle1) } just Runs
|
||||||
|
coEvery { snapshotManager.removeSnapshot(snapshotHandle2) } just Runs
|
||||||
|
|
||||||
|
pruner.removeOldSnapshotsAndPruneUnusedBlobs()
|
||||||
|
|
||||||
|
coVerify {
|
||||||
|
snapshotManager.removeSnapshot(snapshotHandle1)
|
||||||
|
snapshotManager.removeSnapshot(snapshotHandle2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun expectLoadingSnapshots(
|
||||||
|
snapshots: Map<AppBackupFileType.Snapshot, Snapshot>,
|
||||||
|
) {
|
||||||
|
every { crypto.repoId } returns repoId
|
||||||
|
every { backendManager.backend } returns backend
|
||||||
|
coEvery {
|
||||||
|
backend.list(folder, AppBackupFileType.Snapshot::class, callback = captureLambda())
|
||||||
|
} answers {
|
||||||
|
snapshots.keys.forEach {
|
||||||
|
val fileInfo = FileInfo(it, Random.nextLong(Long.MAX_VALUE))
|
||||||
|
lambda<(FileInfo) -> Unit>().captured.invoke(fileInfo)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
snapshots.forEach { (handle, snapshot) ->
|
||||||
|
coEvery { snapshotManager.loadSnapshot(handle) } returns snapshot
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun expectLoadingBlobs(blobIds: List<String>) {
|
||||||
|
coEvery {
|
||||||
|
backend.list(folder, AppBackupFileType.Blob::class, callback = captureLambda())
|
||||||
|
} answers {
|
||||||
|
blobIds.forEach {
|
||||||
|
val fileInfo = FileInfo(
|
||||||
|
fileHandle = AppBackupFileType.Blob(repoId, it),
|
||||||
|
size = Random.nextLong(Long.MAX_VALUE),
|
||||||
|
)
|
||||||
|
lambda<(FileInfo) -> Unit>().captured.invoke(fileInfo)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun LocalDateTime.toMillis(): Long {
|
||||||
|
return toInstant(UTC).toEpochMilli()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in a new issue