Only consider apps that really opt-out of backup for early APK backup

app/src/androidTest/java/com/stevesoltys/seedvault/transport/backup/PackageServiceTest.kt

@@ -0,0 +1,23 @@
+package com.stevesoltys.seedvault.transport.backup
+
+import android.util.Log
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.koin.core.KoinComponent
+import org.koin.core.inject
+
+@RunWith(AndroidJUnit4::class)
+class PackageServiceTest : KoinComponent {
+
+    private val packageService: PackageService by inject()
+
+    @Test
+    fun testNotAllowedPackages() {
+        val packages = packageService.notAllowedPackages
+        assertTrue(packages.isNotEmpty())
+        Log.e("TEST", "Packages: $packages")
+    }
+
+}

app/src/main/java/com/stevesoltys/seedvault/NotificationBackupObserver.kt

@@ -14,9 +14,10 @@ import org.koin.core.inject
 private val TAG = NotificationBackupObserver::class.java.simpleName
 
 class NotificationBackupObserver(
-        private val context: Context,
+    private val context: Context,
-        private val expectedPackages: Int,
+    private val expectedPackages: Int,
-        private val userInitiated: Boolean) : IBackupObserver.Stub(), KoinComponent {
+    private val userInitiated: Boolean
+) : IBackupObserver.Stub(), KoinComponent {
 
     private val nm: BackupNotificationManager by inject()
     private val metadataManager: MetadataManager by inject()
@@ -25,6 +26,12 @@ class NotificationBackupObserver(
 
     init {
         // we need to show this manually as [onUpdate] isn't called for first @pm@ package
+        // TODO consider showing something else for MAGIC_PACKAGE_MANAGER,
+        //  because we also back up APKs at the beginning and this can take quite some time.
+        //  Therefore, also consider showing a more fine-grained progress bar
+        //  by (roughly) doubling the number [expectedPackages] (probably -3)
+        //  and calling back here from KvBackup and ApkBackup to update progress.
+        //  We will also need to take [PackageService#notAllowedPackages] into account.
         nm.onBackupUpdate(getAppName(MAGIC_PACKAGE_MANAGER), 0, expectedPackages, userInitiated)
     }
 
@@ -77,7 +84,9 @@ class NotificationBackupObserver(
         if (currentPackage == packageName) return
 
         if (isLoggable(TAG, INFO)) {
-            Log.i(TAG, "Showing progress notification for $currentPackage $numPackages/$expectedPackages")
+            "Showing progress notification for $currentPackage $numPackages/$expectedPackages".let {
+                Log.i(TAG, it)
+            }
         }
         currentPackage = packageName
         val app = getAppName(packageName)

app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataManager.kt

@@ -2,8 +2,6 @@ package com.stevesoltys.seedvault.metadata
 
 import android.content.Context
 import android.content.Context.MODE_PRIVATE
-import android.content.pm.ApplicationInfo.FLAG_SYSTEM
-import android.content.pm.ApplicationInfo.FLAG_UPDATED_SYSTEM_APP
 import android.content.pm.PackageInfo
 import android.util.Log
 import androidx.annotation.VisibleForTesting
@@ -12,23 +10,25 @@ import androidx.lifecycle.LiveData
 import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.distinctUntilChanged
 import com.stevesoltys.seedvault.Clock
-import com.stevesoltys.seedvault.MAGIC_PACKAGE_MANAGER
 import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA
 import com.stevesoltys.seedvault.metadata.PackageState.NOT_ALLOWED
+import com.stevesoltys.seedvault.transport.backup.isSystemApp
 import java.io.FileNotFoundException
 import java.io.IOException
 import java.io.OutputStream
 
 private val TAG = MetadataManager::class.java.simpleName
+
 @VisibleForTesting
 internal const val METADATA_CACHE_FILE = "metadata.cache"
 
 @WorkerThread
 class MetadataManager(
-        private val context: Context,
+    private val context: Context,
-        private val clock: Clock,
+    private val clock: Clock,
-        private val metadataWriter: MetadataWriter,
+    private val metadataWriter: MetadataWriter,
-        private val metadataReader: MetadataReader) {
+    private val metadataReader: MetadataReader
+) {
 
     private val uninitializedMetadata = BackupMetadata(token = 0L)
     private var metadata: BackupMetadata = uninitializedMetadata
@@ -67,7 +67,11 @@ class MetadataManager(
      */
     @Synchronized
     @Throws(IOException::class)
-    fun onApkBackedUp(packageInfo: PackageInfo, packageMetadata: PackageMetadata, metadataOutputStream: OutputStream) {
+    fun onApkBackedUp(
+        packageInfo: PackageInfo,
+        packageMetadata: PackageMetadata,
+        metadataOutputStream: OutputStream
+    ) {
         val packageName = packageInfo.packageName
         metadata.packageMetadataMap[packageName]?.let {
             check(packageMetadata.version != null) {
@@ -78,20 +82,21 @@ class MetadataManager(
             }
         }
         val oldPackageMetadata = metadata.packageMetadataMap[packageName]
-                ?: PackageMetadata()
+            ?: PackageMetadata()
         // only allow state change if backup of this package is not allowed
-        val newState = if (packageMetadata.state == NOT_ALLOWED)
+        val newState = if (packageMetadata.state == NOT_ALLOWED) {
             packageMetadata.state
-        else
+        } else {
             oldPackageMetadata.state
+        }
         modifyMetadata(metadataOutputStream) {
             metadata.packageMetadataMap[packageName] = oldPackageMetadata.copy(
-                    state = newState,
+                state = newState,
-                    system = packageInfo.isSystemApp(),
+                system = packageInfo.isSystemApp(),
-                    version = packageMetadata.version,
+                version = packageMetadata.version,
-                    installer = packageMetadata.installer,
+                installer = packageMetadata.installer,
-                    sha256 = packageMetadata.sha256,
+                sha256 = packageMetadata.sha256,
-                    signatures = packageMetadata.signatures
+                signatures = packageMetadata.signatures
             )
         }
     }
@@ -114,9 +119,9 @@ class MetadataManager(
                 metadata.packageMetadataMap[packageName]!!.state = APK_AND_DATA
             } else {
                 metadata.packageMetadataMap[packageName] = PackageMetadata(
-                        time = now,
+                    time = now,
-                        state = APK_AND_DATA,
+                    state = APK_AND_DATA,
-                        system = packageInfo.isSystemApp()
+                    system = packageInfo.isSystemApp()
                 )
             }
         }
@@ -130,7 +135,11 @@ class MetadataManager(
      */
     @Synchronized
     @Throws(IOException::class)
-    internal fun onPackageBackupError(packageInfo: PackageInfo, packageState: PackageState, metadataOutputStream: OutputStream) {
+    internal fun onPackageBackupError(
+        packageInfo: PackageInfo,
+        packageState: PackageState,
+        metadataOutputStream: OutputStream
+    ) {
         check(packageState != APK_AND_DATA) { "Backup Error with non-error package state." }
         val packageName = packageInfo.packageName
         modifyMetadata(metadataOutputStream) {
@@ -138,9 +147,9 @@ class MetadataManager(
                 metadata.packageMetadataMap[packageName]!!.state = packageState
             } else {
                 metadata.packageMetadataMap[packageName] = PackageMetadata(
-                        time = 0L,
+                    time = 0L,
-                        state = packageState,
+                    state = packageState,
-                        system = packageInfo.isSystemApp()
+                    system = packageInfo.isSystemApp()
                 )
             }
         }
@@ -219,13 +228,3 @@ class MetadataManager(
     }
 
 }
-
-fun PackageInfo.isSystemApp(): Boolean {
-    if (packageName == MAGIC_PACKAGE_MANAGER || applicationInfo == null) return true
-    return applicationInfo.flags and FLAG_SYSTEM != 0
-}
-
-fun PackageInfo.isUpdatedSystemApp(): Boolean {
-    if (packageName == MAGIC_PACKAGE_MANAGER || applicationInfo == null) return false
-    return applicationInfo.flags and FLAG_UPDATED_SYSTEM_APP != 0
-}

app/src/main/java/com/stevesoltys/seedvault/settings/SettingsViewModel.kt

@@ -21,25 +21,25 @@ import com.stevesoltys.seedvault.metadata.PackageState.NOT_ALLOWED
 import com.stevesoltys.seedvault.metadata.PackageState.NO_DATA
 import com.stevesoltys.seedvault.metadata.PackageState.QUOTA_EXCEEDED
 import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR
-import com.stevesoltys.seedvault.metadata.isSystemApp
 import com.stevesoltys.seedvault.restore.AppRestoreStatus.FAILED
 import com.stevesoltys.seedvault.restore.AppRestoreStatus.FAILED_NOT_ALLOWED
 import com.stevesoltys.seedvault.restore.AppRestoreStatus.FAILED_NO_DATA
 import com.stevesoltys.seedvault.restore.AppRestoreStatus.FAILED_QUOTA_EXCEEDED
 import com.stevesoltys.seedvault.restore.AppRestoreStatus.SUCCEEDED
+import com.stevesoltys.seedvault.transport.backup.isSystemApp
 import com.stevesoltys.seedvault.transport.requestBackup
 import com.stevesoltys.seedvault.ui.RequireProvisioningViewModel
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
-import java.util.*
+import java.util.Locale
 
 private val TAG = SettingsViewModel::class.java.simpleName
 
 class SettingsViewModel(
-        app: Application,
+    app: Application,
-        settingsManager: SettingsManager,
+    settingsManager: SettingsManager,
-        keyManager: KeyManager,
+    keyManager: KeyManager,
-        private val metadataManager: MetadataManager
+    private val metadataManager: MetadataManager
 ) : RequireProvisioningViewModel(app, settingsManager, keyManager) {
 
     override val isRestoreOperation = false

app/src/main/java/com/stevesoltys/seedvault/transport/backup/ApkBackup.kt

@@ -11,8 +11,6 @@ import com.stevesoltys.seedvault.encodeBase64
 import com.stevesoltys.seedvault.metadata.MetadataManager
 import com.stevesoltys.seedvault.metadata.PackageMetadata
 import com.stevesoltys.seedvault.metadata.PackageState
-import com.stevesoltys.seedvault.metadata.isSystemApp
-import com.stevesoltys.seedvault.metadata.isUpdatedSystemApp
 import com.stevesoltys.seedvault.settings.SettingsManager
 import java.io.File
 import java.io.FileNotFoundException
@@ -23,9 +21,10 @@ import java.security.MessageDigest
 private val TAG = ApkBackup::class.java.simpleName
 
 class ApkBackup(
-        private val pm: PackageManager,
+    private val pm: PackageManager,
-        private val settingsManager: SettingsManager,
+    private val settingsManager: SettingsManager,
-        private val metadataManager: MetadataManager) {
+    private val metadataManager: MetadataManager
+) {
 
     /**
      * Checks if a new APK needs to get backed up,
@@ -36,7 +35,11 @@ class ApkBackup(
      * @return new [PackageMetadata] if an APK backup was made or null if no backup was made.
      */
     @Throws(IOException::class)
-    suspend fun backupApkIfNecessary(packageInfo: PackageInfo, packageState: PackageState, streamGetter: suspend () -> OutputStream): PackageMetadata? {
+    suspend fun backupApkIfNecessary(
+        packageInfo: PackageInfo,
+        packageState: PackageState,
+        streamGetter: suspend () -> OutputStream
+    ): PackageMetadata? {
         // do not back up @pm@
         val packageName = packageInfo.packageName
         if (packageName == MAGIC_PACKAGE_MANAGER) return null
@@ -45,7 +48,7 @@ class ApkBackup(
         if (!settingsManager.backupApks()) return null
 
         // do not back up system apps that haven't been updated
-        if (packageInfo.isSystemApp() && !packageInfo.isUpdatedSystemApp()) {
+        if (packageInfo.isNotUpdatedSystemApp()) {
             Log.d(TAG, "Package $packageName is vanilla system app. Not backing it up.")
             return null
         }
@@ -65,15 +68,19 @@ class ApkBackup(
 
         // get cached metadata about package
         val packageMetadata = metadataManager.getPackageMetadata(packageName)
-                ?: PackageMetadata()
+            ?: PackageMetadata()
 
         // get version codes
         val version = packageInfo.longVersionCode
-        val backedUpVersion = packageMetadata.version ?: 0L  // no version will cause backup
+        val backedUpVersion = packageMetadata.version ?: 0L // no version will cause backup
 
         // do not backup if we have the version already and signatures did not change
         if (version <= backedUpVersion && !signaturesChanged(packageMetadata, signatures)) {
-            Log.d(TAG, "Package $packageName with version $version already has a backup ($backedUpVersion) with the same signature. Not backing it up.")
+            Log.d(
+                TAG,
+                "Package $packageName with version $version already has a backup ($backedUpVersion)" +
+                        " with the same signature. Not backing it up."
+            )
             return null
         }
 
@@ -91,7 +98,7 @@ class ApkBackup(
 
         // copy the APK to the storage's output and calculate SHA-256 hash while at it
         val messageDigest = MessageDigest.getInstance("SHA-256")
-        streamGetter.invoke().use { outputStream ->
+        streamGetter().use { outputStream ->
             inputStream.use { inputStream ->
                 val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
                 var bytes = inputStream.read(buffer)
@@ -107,15 +114,18 @@ class ApkBackup(
 
         // return updated metadata
         return PackageMetadata(
-                state = packageState,
+            state = packageState,
-                version = version,
+            version = version,
-                installer = pm.getInstallerPackageName(packageName),
+            installer = pm.getInstallerPackageName(packageName),
-                sha256 = sha256,
+            sha256 = sha256,
-                signatures = signatures
+            signatures = signatures
         )
     }
 
-    private fun signaturesChanged(packageMetadata: PackageMetadata, signatures: List<String>): Boolean {
+    private fun signaturesChanged(
+        packageMetadata: PackageMetadata,
+        signatures: List<String>
+    ): Boolean {
         // no signatures in package metadata counts as them not having changed
         if (packageMetadata.signatures == null) return false
         // TODO to support multiple signers check if lists differ

app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt

@@ -19,7 +19,6 @@ import com.stevesoltys.seedvault.metadata.PackageState.NOT_ALLOWED
 import com.stevesoltys.seedvault.metadata.PackageState.NO_DATA
 import com.stevesoltys.seedvault.metadata.PackageState.QUOTA_EXCEEDED
 import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR
-import com.stevesoltys.seedvault.metadata.isSystemApp
 import com.stevesoltys.seedvault.settings.SettingsManager
 import java.io.IOException
 import java.util.concurrent.TimeUnit.DAYS

app/src/main/java/com/stevesoltys/seedvault/transport/backup/PackageService.kt

@@ -1,6 +1,9 @@
 package com.stevesoltys.seedvault.transport.backup
 
 import android.app.backup.IBackupManager
+import android.content.pm.ApplicationInfo.FLAG_ALLOW_BACKUP
+import android.content.pm.ApplicationInfo.FLAG_SYSTEM
+import android.content.pm.ApplicationInfo.FLAG_UPDATED_SYSTEM_APP
 import android.content.pm.PackageInfo
 import android.content.pm.PackageManager
 import android.content.pm.PackageManager.GET_SIGNING_CERTIFICATES
@@ -20,8 +23,9 @@ private const val LOG_MAX_PACKAGES = 100
  * @author Torsten Grote
  */
 internal class PackageService(
-        private val packageManager: PackageManager,
+    private val packageManager: PackageManager,
-        private val backupManager: IBackupManager) {
+    private val backupManager: IBackupManager
+) {
 
     private val myUserId = UserHandle.myUserId()
 
@@ -30,8 +34,8 @@ internal class PackageService(
         @Throws(RemoteException::class)
         get() {
             val packages = packageManager.getInstalledPackages(0)
-                    .map { packageInfo -> packageInfo.packageName }
+                .map { packageInfo -> packageInfo.packageName }
-                    .sorted()
+                .sorted()
 
             // log packages
             if (Log.isLoggable(TAG, INFO)) {
@@ -41,14 +45,13 @@ internal class PackageService(
                 }
             }
 
-            val eligibleApps = backupManager.filterAppsEligibleForBackupForUser(myUserId, packages.toTypedArray())
+            val eligibleApps =
+                backupManager.filterAppsEligibleForBackupForUser(myUserId, packages.toTypedArray())
 
             // log eligible packages
             if (Log.isLoggable(TAG, INFO)) {
                 Log.i(TAG, "Filtering left ${eligibleApps.size} eligible packages:")
-                eligibleApps.toList().chunked(LOG_MAX_PACKAGES).forEach {
+                logPackages(eligibleApps.toList())
-                    Log.i(TAG, it.toString())
-                }
             }
 
             // add magic @pm@ package (PACKAGE_MANAGER_SENTINEL) which holds package manager data
@@ -61,16 +64,48 @@ internal class PackageService(
     val notAllowedPackages: List<PackageInfo>
         @WorkerThread
         get() {
-            val installed = packageManager.getInstalledPackages(GET_SIGNING_CERTIFICATES)
+            // We need the GET_SIGNING_CERTIFICATES flag here,
-            val installedArray = installed.map { packageInfo ->
+            // because the package info is used by [ApkBackup] which needs signing info.
-                packageInfo.packageName
+            return packageManager.getInstalledPackages(GET_SIGNING_CERTIFICATES)
-            }.toTypedArray()
+                .filter { packageInfo ->
-
+                    !packageInfo.isBackupAllowed() && // only apps that do not allow backup
-            val eligible = backupManager.filterAppsEligibleForBackupForUser(myUserId, installedArray)
+                            !packageInfo.isNotUpdatedSystemApp() // and are not vanilla system apps
-
+                }.sortedBy { packageInfo ->
-            return installed.filter { packageInfo ->
+                    packageInfo.packageName
-                packageInfo.packageName !in eligible
+                }.also { notAllowed ->
-            }.sortedBy { it.packageName }
+                    // log eligible packages
+                    if (Log.isLoggable(TAG, INFO)) {
+                        Log.i(TAG, "${notAllowed.size} apps do not allow backup:")
+                        logPackages(notAllowed.map { it.packageName })
+                    }
+                }
         }
 
+    private fun logPackages(packages: List<String>) {
+        packages.chunked(LOG_MAX_PACKAGES).forEach {
+            Log.i(TAG, it.toString())
+        }
+    }
+
+}
+
+internal fun PackageInfo.isSystemApp(): Boolean {
+    if (packageName == MAGIC_PACKAGE_MANAGER || applicationInfo == null) return true
+    return applicationInfo.flags and FLAG_SYSTEM != 0
+}
+
+/**
+ * Returns true if this is a system app that hasn't been updated.
+ * We don't back up those APKs.
+ */
+internal fun PackageInfo.isNotUpdatedSystemApp(): Boolean {
+    if (packageName == MAGIC_PACKAGE_MANAGER || applicationInfo == null) return true
+    val isSystemApp = applicationInfo.flags and FLAG_SYSTEM != 0
+    val isUpdatedSystemApp = applicationInfo.flags and FLAG_UPDATED_SYSTEM_APP != 0
+    return isSystemApp && !isUpdatedSystemApp
+}
+
+internal fun PackageInfo.isBackupAllowed(): Boolean {
+    if (packageName == MAGIC_PACKAGE_MANAGER || applicationInfo == null) return false
+    return applicationInfo.flags and FLAG_ALLOW_BACKUP != 0
 }

app/src/main/java/com/stevesoltys/seedvault/transport/restore/ApkRestore.kt

@@ -9,8 +9,8 @@ import android.util.Log
 import com.stevesoltys.seedvault.encodeBase64
 import com.stevesoltys.seedvault.metadata.PackageMetadata
 import com.stevesoltys.seedvault.metadata.PackageMetadataMap
-import com.stevesoltys.seedvault.metadata.isSystemApp
 import com.stevesoltys.seedvault.transport.backup.getSignatures
+import com.stevesoltys.seedvault.transport.backup.isSystemApp
 import com.stevesoltys.seedvault.transport.restore.ApkRestoreStatus.FAILED
 import com.stevesoltys.seedvault.transport.restore.ApkRestoreStatus.IN_PROGRESS
 import com.stevesoltys.seedvault.transport.restore.ApkRestoreStatus.QUEUED