try more than once to upload metadata after APK backups

Failure to upload metadata after backup up APKs can be critical and flaky I/O can make it fail, so we try again.

app/src/main/java/com/stevesoltys/seedvault/worker/ApkBackupManager.kt

@@ -18,6 +18,7 @@ import com.stevesoltys.seedvault.transport.backup.PackageService
 import com.stevesoltys.seedvault.transport.backup.isStopped
 import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
 import com.stevesoltys.seedvault.ui.notification.getAppName
+import kotlinx.coroutines.delay
 import java.io.IOException
 import java.io.OutputStream
 
@@ -46,10 +47,13 @@ internal class ApkBackupManager(
                 backUpApks()
             }
         } finally {
-            // upload all local changes only at the end, so we don't have to re-upload the metadata
+            keepTrying {
+                // upload all local changes only at the end,
+                // so we don't have to re-upload the metadata
                 plugin.getMetadataOutputStream().use { outputStream ->
                     metadataManager.uploadMetadata(outputStream)
                 }
+            }
             nm.onApkBackupDone()
         }
     }
@@ -109,6 +113,18 @@ internal class ApkBackupManager(
         }
     }
 
+    private suspend fun keepTrying(n: Int = 3, block: suspend () -> Unit) {
+        for (i in 1..n) {
+            try {
+                block()
+            } catch (e: Exception) {
+                if (i == n) throw e
+                Log.e(TAG, "Error (#$i), we'll keep trying", e)
+                delay(1000)
+            }
+        }
+    }
+
     private suspend fun StoragePlugin.getMetadataOutputStream(token: Long? = null): OutputStream {
         val t = token ?: settingsManager.getToken() ?: throw IOException("no current token")
         return getOutputStream(t, FILE_BACKUP_METADATA)

app/src/test/java/com/stevesoltys/seedvault/worker/ApkBackupManagerTest.kt

@@ -15,6 +15,7 @@ import com.stevesoltys.seedvault.transport.TransportTest
 import com.stevesoltys.seedvault.transport.backup.PackageService
 import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
 import io.mockk.Runs
+import io.mockk.andThenJust
 import io.mockk.coEvery
 import io.mockk.coVerify
 import io.mockk.every
@@ -24,6 +25,7 @@ import io.mockk.verify
 import io.mockk.verifyAll
 import kotlinx.coroutines.runBlocking
 import org.junit.jupiter.api.Test
+import java.io.IOException
 import java.io.OutputStream
 
 internal class ApkBackupManagerTest : TransportTest() {
@@ -189,6 +191,37 @@ internal class ApkBackupManagerTest : TransportTest() {
         }
     }
 
+    @Test
+    fun `we keep trying to upload metadata at the end`() = runBlocking {
+        every { nm.onAppsNotBackedUp() } just Runs
+        every { packageService.notBackedUpPackages } returns listOf(packageInfo)
+
+        every {
+            metadataManager.getPackageMetadata(packageInfo.packageName)
+        } returns packageMetadata
+        every { packageMetadata.state } returns UNKNOWN_ERROR
+        every { metadataManager.onPackageDoesNotGetBackedUp(packageInfo, NOT_ALLOWED) } just Runs
+
+        every { settingsManager.backupApks() } returns false
+
+        // final upload
+        every { settingsManager.getToken() } returns token
+        coEvery { plugin.getOutputStream(token, FILE_BACKUP_METADATA) } returns metadataOutputStream
+        every {
+            metadataManager.uploadMetadata(metadataOutputStream)
+        } throws IOException() andThenThrows SecurityException() andThenJust Runs
+        every { metadataOutputStream.close() } just Runs
+
+        every { nm.onApkBackupDone() } just Runs
+
+        apkBackupManager.backup()
+
+        verify {
+            metadataManager.onPackageDoesNotGetBackedUp(packageInfo, NOT_ALLOWED)
+            metadataOutputStream.close()
+        }
+    }
+
     private fun expectAllAppsWillGetBackedUp() {
         every { nm.onAppsNotBackedUp() } just Runs
         every { packageService.notBackedUpPackages } returns emptyList()