Write metadata with new version 1

Reading still supports version 0

app/build.gradle

@@ -105,6 +105,8 @@ dependencies {
     implementation rootProject.ext.std_libs.androidx_documentfile
     implementation rootProject.ext.std_libs.com_google_android_material
 
+    implementation rootProject.ext.storage_libs.com_google_crypto_tink_android
+
     /**
      * Storage Dependencies
      */

app/src/main/java/com/stevesoltys/seedvault/crypto/Crypto.kt

@@ -1,5 +1,6 @@
 package com.stevesoltys.seedvault.crypto
 
+import com.google.crypto.tink.subtle.AesGcmHkdfStreaming
 import com.stevesoltys.seedvault.header.HeaderReader
 import com.stevesoltys.seedvault.header.HeaderWriter
 import com.stevesoltys.seedvault.header.MAX_SEGMENT_CLEARTEXT_LENGTH
@@ -7,10 +8,13 @@ import com.stevesoltys.seedvault.header.MAX_SEGMENT_LENGTH
 import com.stevesoltys.seedvault.header.MAX_VERSION_HEADER_SIZE
 import com.stevesoltys.seedvault.header.SegmentHeader
 import com.stevesoltys.seedvault.header.VersionHeader
+import org.calyxos.backup.storage.crypto.StreamCrypto
+import org.calyxos.backup.storage.crypto.StreamCrypto.deriveStreamKey
 import java.io.EOFException
 import java.io.IOException
 import java.io.InputStream
 import java.io.OutputStream
+import java.security.GeneralSecurityException
 import javax.crypto.Cipher
 import javax.crypto.spec.SecretKeySpec
 import kotlin.math.min
@@ -33,6 +37,22 @@ import kotlin.math.min
  */
 interface Crypto {
 
+    /**
+     * Returns a [AesGcmHkdfStreaming] encrypting stream
+     * that gets encrypted with the given secret.
+     */
+    @Throws(IOException::class, GeneralSecurityException::class)
+    fun newEncryptingStream(
+        outputStream: OutputStream,
+        associatedData: ByteArray = ByteArray(0)
+    ): OutputStream
+
+    @Throws(IOException::class, GeneralSecurityException::class)
+    fun newDecryptingStream(
+        inputStream: InputStream,
+        associatedData: ByteArray = ByteArray(0)
+    ): InputStream
+
     /**
      * Encrypts a backup stream header ([VersionHeader]) and writes it to the given [OutputStream].
      *
@@ -105,12 +125,35 @@ interface Crypto {
     fun verifyBackupKey(seed: ByteArray): Boolean
 }
 
+internal const val TYPE_METADATA: Byte = 0x00
+internal const val TYPE_BACKUP_KV: Byte = 0x01
+internal const val TYPE_BACKUP_FULL: Byte = 0x02
+
 internal class CryptoImpl(
+    private val keyManager: KeyManager,
     private val cipherFactory: CipherFactory,
     private val headerWriter: HeaderWriter,
     private val headerReader: HeaderReader
 ) : Crypto {
 
+    private val key: ByteArray by lazy {
+        deriveStreamKey(keyManager.getMainKey(), "app data key".toByteArray())
+    }
+
+    override fun newEncryptingStream(
+        outputStream: OutputStream,
+        associatedData: ByteArray
+    ): OutputStream {
+        return StreamCrypto.newEncryptingStream(key, outputStream, associatedData)
+    }
+
+    override fun newDecryptingStream(
+        inputStream: InputStream,
+        associatedData: ByteArray
+    ): InputStream {
+        return StreamCrypto.newDecryptingStream(key, inputStream, associatedData)
+    }
+
     @Throws(IOException::class)
     override fun encryptHeader(outputStream: OutputStream, versionHeader: VersionHeader) {
         val bytes = headerWriter.getEncodedVersionHeader(versionHeader)

app/src/main/java/com/stevesoltys/seedvault/crypto/CryptoModule.kt

@@ -15,5 +15,5 @@ val cryptoModule = module {
         }
         KeyManagerImpl(keyStore)
     }
-    single<Crypto> { CryptoImpl(get(), get(), get()) }
+    single<Crypto> { CryptoImpl(get(), get(), get(), get()) }
 }

app/src/main/java/com/stevesoltys/seedvault/header/Header.kt

@@ -2,7 +2,7 @@ package com.stevesoltys.seedvault.header
 
 import com.stevesoltys.seedvault.crypto.GCM_AUTHENTICATION_TAG_LENGTH
 
-internal const val VERSION: Byte = 0
+internal const val VERSION: Byte = 1
 internal const val MAX_PACKAGE_LENGTH_SIZE = 255
 internal const val MAX_KEY_LENGTH_SIZE = MAX_PACKAGE_LENGTH_SIZE
 internal const val MAX_VERSION_HEADER_SIZE =

app/src/main/java/com/stevesoltys/seedvault/metadata/Metadata.kt

@@ -2,9 +2,12 @@ package com.stevesoltys.seedvault.metadata
 
 import android.content.pm.ApplicationInfo.FLAG_STOPPED
 import android.os.Build
+import com.stevesoltys.seedvault.crypto.TYPE_METADATA
 import com.stevesoltys.seedvault.header.VERSION
 import com.stevesoltys.seedvault.metadata.PackageState.UNKNOWN_ERROR
+import org.calyxos.backup.storage.crypto.StreamCrypto.toByteArray
 import java.io.InputStream
+import java.nio.ByteBuffer
 
 typealias PackageMetadataMap = HashMap<String, PackageMetadata>
 
@@ -110,3 +113,9 @@ class EncryptedBackupMetadata private constructor(
      */
     constructor(token: Long) : this(token, null, true)
 }
+
+internal fun getAD(version: Byte, token: Long) = ByteBuffer.allocate(2 + 8)
+    .put(version)
+    .put(TYPE_METADATA)
+    .put(token.toByteArray())
+    .array()

app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataManager.kt

@@ -10,6 +10,7 @@ import androidx.lifecycle.LiveData
 import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.distinctUntilChanged
 import com.stevesoltys.seedvault.Clock
+import com.stevesoltys.seedvault.header.VERSION
 import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA
 import com.stevesoltys.seedvault.metadata.PackageState.NOT_ALLOWED
 import com.stevesoltys.seedvault.metadata.PackageState.NO_DATA
@@ -205,6 +206,9 @@ class MetadataManager(
     private val mLastBackupTime = MutableLiveData<Long>()
     internal val lastBackupTime: LiveData<Long> = mLastBackupTime.distinctUntilChanged()
 
+    internal val isLegacyFormat: Boolean
+        @Synchronized get() = metadata.version < VERSION
+
     @Synchronized
     fun getPackageMetadata(packageName: String): PackageMetadata? {
         return metadata.packageMetadataMap[packageName]?.copy()

app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataReader.kt

@@ -14,6 +14,7 @@ import org.json.JSONException
 import org.json.JSONObject
 import java.io.IOException
 import java.io.InputStream
+import java.security.GeneralSecurityException
 import javax.crypto.AEADBadTagException
 
 interface MetadataReader {
@@ -47,12 +48,29 @@ internal class MetadataReaderImpl(private val crypto: Crypto) : MetadataReader {
         val version = inputStream.read().toByte()
         if (version < 0) throw IOException()
         if (version > VERSION) throw UnsupportedVersionException(version)
+        if (version == 0.toByte()) return readMetadataV0(inputStream, expectedToken)
+
+        val metadataBytes = try {
+            crypto.newDecryptingStream(inputStream, getAD(version, expectedToken)).readBytes()
+        } catch (e: GeneralSecurityException) {
+            throw DecryptionFailedException(e)
+        }
+        return decode(metadataBytes, version, expectedToken)
+    }
+
+    @Throws(
+        SecurityException::class,
+        DecryptionFailedException::class,
+        UnsupportedVersionException::class,
+        IOException::class
+    )
+    private fun readMetadataV0(inputStream: InputStream, expectedToken: Long): BackupMetadata {
         val metadataBytes = try {
             crypto.decryptMultipleSegments(inputStream)
         } catch (e: AEADBadTagException) {
             throw DecryptionFailedException(e)
         }
-        return decode(metadataBytes, version, expectedToken)
+        return decode(metadataBytes, 0.toByte(), expectedToken)
     }
 
     @Throws(SecurityException::class)

app/src/main/java/com/stevesoltys/seedvault/metadata/MetadataWriter.kt

@@ -20,7 +20,9 @@ internal class MetadataWriterImpl(private val crypto: Crypto) : MetadataWriter {
     @Throws(IOException::class)
     override fun write(metadata: BackupMetadata, outputStream: OutputStream) {
         outputStream.write(ByteArray(1).apply { this[0] = metadata.version })
-        crypto.encryptMultipleSegments(outputStream, encode(metadata))
+        crypto.newEncryptingStream(outputStream, getAD(metadata.version, metadata.token)).use {
+            it.write(encode(metadata))
+        }
     }
 
     override fun encode(metadata: BackupMetadata): ByteArray {

app/src/main/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinator.kt

@@ -238,6 +238,7 @@ internal class BackupCoordinator(
         // We need to reject them manually when we can not do a backup now.
         // What else we tried can be found in: https://github.com/seedvault-app/seedvault/issues/102
         if (packageName == MAGIC_PACKAGE_MANAGER) {
+            val isIncremental = flags and FLAG_INCREMENTAL != 0
             if (!settingsManager.canDoBackupNow()) {
                 // Returning anything else here (except non-incremental-required which re-tries)
                 // will make the system consider the backup state compromised
@@ -248,9 +249,17 @@ internal class BackupCoordinator(
                 settingsManager.pmBackupNextTimeNonIncremental = true
                 data.close()
                 return TRANSPORT_OK
-            } else if (flags and FLAG_INCREMENTAL != 0 &&
+            } else if (metadataManager.isLegacyFormat) {
-                settingsManager.pmBackupNextTimeNonIncremental
+                // start a new restore set to upgrade from legacy format
-            ) {
+                // by starting a clean backup with all files using the new version
+                try {
+                    startNewRestoreSet()
+                } catch (e: IOException) {
+                    Log.e(TAG, "Error starting new restore set", e)
+                }
+                // this causes a backup error, but things should go back to normal afterwards
+                return TRANSPORT_NOT_INITIALIZED
+            } else if (isIncremental && settingsManager.pmBackupNextTimeNonIncremental) {
                 settingsManager.pmBackupNextTimeNonIncremental = false
                 data.close()
                 return TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED

app/src/sharedTest/java/com/stevesoltys/seedvault/crypto/KeyManagerTestImpl.kt

@@ -29,12 +29,8 @@ class KeyManagerTestImpl(private val customKey: SecretKey? = null) : KeyManager
         throw NotImplementedError("not implemented")
     }
 
-    override fun getBackupKey(): SecretKey {
+    override fun getBackupKey(): SecretKey = key
-        return key
-    }
 
-    override fun getMainKey(): SecretKey {
+    override fun getMainKey(): SecretKey = key
-        throw NotImplementedError("not implemented")
-    }
 
 }

app/src/test/java/com/stevesoltys/seedvault/TestApp.kt

@@ -21,7 +21,7 @@ class TestApp : App() {
     private val testCryptoModule = module {
         factory<CipherFactory> { CipherFactoryImpl(get()) }
         single<KeyManager> { KeyManagerTestImpl() }
-        single<Crypto> { CryptoImpl(get(), get(), get()) }
+        single<Crypto> { CryptoImpl(get(), get(), get(), get()) }
     }
     private val appModule = module {
         single { Clock() }

app/src/test/java/com/stevesoltys/seedvault/crypto/CryptoImplTest.kt

@@ -20,11 +20,12 @@ import kotlin.random.Random
 @TestInstance(PER_METHOD)
 class CryptoImplTest {
 
+    private val keyManager = mockk<KeyManager>()
     private val cipherFactory = mockk<CipherFactory>()
     private val headerWriter = HeaderWriterImpl()
     private val headerReader = HeaderReaderImpl()
 
-    private val crypto = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val crypto = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
 
     private val cipher = mockk<Cipher>()
 

app/src/test/java/com/stevesoltys/seedvault/crypto/CryptoIntegrationTest.kt

@@ -21,7 +21,7 @@ class CryptoIntegrationTest {
     private val headerWriter = HeaderWriterImpl()
     private val headerReader = HeaderReaderImpl()
 
-    private val crypto = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val crypto = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
 
     private val cleartext = byteArrayOf(0x01, 0x02, 0x03)
 

app/src/test/java/com/stevesoltys/seedvault/crypto/CryptoTest.kt

@@ -36,11 +36,12 @@ import kotlin.random.Random
 @TestInstance(PER_METHOD)
 class CryptoTest {
 
+    private val keyManager = mockk<KeyManager>()
     private val cipherFactory = mockk<CipherFactory>()
     private val headerWriter = mockk<HeaderWriter>()
     private val headerReader = mockk<HeaderReader>()
 
-    private val crypto = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val crypto = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
 
     private val cipher = mockk<Cipher>()
 

app/src/test/java/com/stevesoltys/seedvault/metadata/MetadataReadWriteTest.kt

@@ -29,7 +29,7 @@ internal class MetadataReadWriteTest {
     private val cipherFactory = CipherFactoryImpl(keyManager)
     private val headerWriter = HeaderWriterImpl()
     private val headerReader = HeaderReaderImpl()
-    private val cryptoImpl = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val cryptoImpl = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
 
     private val writer = MetadataWriterImpl(cryptoImpl)
     private val reader = MetadataReaderImpl(cryptoImpl)
@@ -48,7 +48,6 @@ internal class MetadataReadWriteTest {
 
         val inputStream = ByteArrayInputStream(outputStream.toByteArray())
 
-
         assertEquals(metadata, reader.readMetadata(inputStream, metadata.token))
     }
 

app/src/test/java/com/stevesoltys/seedvault/metadata/MetadataV0ReadTest.kt

@@ -6,7 +6,6 @@ import com.stevesoltys.seedvault.crypto.KEY_SIZE_BYTES
 import com.stevesoltys.seedvault.crypto.KeyManagerTestImpl
 import com.stevesoltys.seedvault.header.HeaderReaderImpl
 import com.stevesoltys.seedvault.header.HeaderWriterImpl
-import com.stevesoltys.seedvault.header.VERSION
 import com.stevesoltys.seedvault.metadata.PackageState.APK_AND_DATA
 import com.stevesoltys.seedvault.metadata.PackageState.WAS_STOPPED
 import com.stevesoltys.seedvault.toByteArrayFromHex
@@ -30,7 +29,7 @@ internal class MetadataV0ReadTest {
     private val cipherFactory = CipherFactoryImpl(keyManager)
     private val headerWriter = HeaderWriterImpl()
     private val headerReader = HeaderReaderImpl()
-    private val cryptoImpl = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val cryptoImpl = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
 
     private val reader = MetadataReaderImpl(cryptoImpl)
 
@@ -55,7 +54,7 @@ internal class MetadataV0ReadTest {
     private fun getMetadata(
         packageMetadata: HashMap<String, PackageMetadata> = HashMap()
     ) = BackupMetadata(
-        version = VERSION,
+        version = 0x00,
         token = 1337L,
         time = 2342L,
         androidVersion = 30,

app/src/test/java/com/stevesoltys/seedvault/transport/CoordinatorIntegrationTest.kt

@@ -59,7 +59,7 @@ internal class CoordinatorIntegrationTest : TransportTest() {
     private val cipherFactory = CipherFactoryImpl(keyManager)
     private val headerWriter = HeaderWriterImpl()
     private val headerReader = HeaderReaderImpl()
-    private val cryptoImpl = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val cryptoImpl = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
     private val metadataReader = MetadataReaderImpl(cryptoImpl)
     private val notificationManager = mockk<BackupNotificationManager>()
 

app/src/test/java/com/stevesoltys/seedvault/transport/backup/BackupCoordinatorTest.kt

@@ -3,6 +3,7 @@ package com.stevesoltys.seedvault.transport.backup
 import android.app.backup.BackupTransport.FLAG_INCREMENTAL
 import android.app.backup.BackupTransport.TRANSPORT_ERROR
 import android.app.backup.BackupTransport.TRANSPORT_NON_INCREMENTAL_BACKUP_REQUIRED
+import android.app.backup.BackupTransport.TRANSPORT_NOT_INITIALIZED
 import android.app.backup.BackupTransport.TRANSPORT_OK
 import android.app.backup.BackupTransport.TRANSPORT_PACKAGE_REJECTED
 import android.app.backup.BackupTransport.TRANSPORT_QUOTA_EXCEEDED
@@ -152,6 +153,7 @@ internal class BackupCoordinatorTest : BackupTest() {
         assertEquals(TRANSPORT_OK, backup.performIncrementalBackup(packageInfo, data, 0))
 
         every { settingsManager.canDoBackupNow() } returns true
+        every { metadataManager.isLegacyFormat } returns false
         every { settingsManager.pmBackupNextTimeNonIncremental } returns true
         every { settingsManager.pmBackupNextTimeNonIncremental = false } just Runs
 
@@ -162,6 +164,27 @@ internal class BackupCoordinatorTest : BackupTest() {
         )
     }
 
+    @Test
+    fun `performIncrementalBackup of @pm@ causes re-init when legacy format`() = runBlocking {
+        val packageInfo = PackageInfo().apply { packageName = MAGIC_PACKAGE_MANAGER }
+
+        every { settingsManager.canDoBackupNow() } returns true
+        every { metadataManager.isLegacyFormat } returns true
+
+        // start new restore set
+        every { clock.time() } returns token + 1
+        every { settingsManager.setNewToken(token + 1) } just Runs
+        coEvery { plugin.startNewRestoreSet(token + 1) } just Runs
+
+        every { data.close() } just Runs
+
+        // returns TRANSPORT_NOT_INITIALIZED to re-init next time
+        assertEquals(
+            TRANSPORT_NOT_INITIALIZED,
+            backup.performIncrementalBackup(packageInfo, data, 0)
+        )
+    }
+
     @Test
     fun `getBackupQuota() delegates to right plugin`() = runBlocking {
         val isFullBackup = Random.nextBoolean()
@@ -354,6 +377,7 @@ internal class BackupCoordinatorTest : BackupTest() {
         val packageMetadata: PackageMetadata = mockk()
 
         every { settingsManager.canDoBackupNow() } returns true
+        every { metadataManager.isLegacyFormat } returns false
         // do actual @pm@ backup
         coEvery { kv.performBackup(packageInfo, fileDescriptor, 0) } returns TRANSPORT_OK
         // now check if we have opt-out apps that we need to back up APKs for

app/src/test/java/com/stevesoltys/seedvault/transport/restore/RestoreV0IntegrationTest.kt

@@ -44,7 +44,7 @@ internal class RestoreV0IntegrationTest : TransportTest() {
     private val cipherFactory = CipherFactoryImpl(keyManager)
     private val headerWriter = HeaderWriterImpl()
     private val headerReader = HeaderReaderImpl()
-    private val cryptoImpl = CryptoImpl(cipherFactory, headerWriter, headerReader)
+    private val cryptoImpl = CryptoImpl(keyManager, cipherFactory, headerWriter, headerReader)
     private val metadataReader = MetadataReaderImpl(cryptoImpl)
     private val notificationManager = mockk<BackupNotificationManager>()
 

gradle/dependencies.gradle

@@ -83,7 +83,7 @@ ext.std_libs = [
 ]
 
 ext.lint_libs = [
-    exceptions: 'com.github.thirdegg:lint-rules:0.0.6-beta'
+    exceptions: 'com.github.thirdegg:lint-rules:0.0.7-beta'
 ]
 
 ext.storage_libs = [

storage/lib/src/main/java/org/calyxos/backup/storage/crypto/StreamCrypto.kt

@@ -77,7 +77,7 @@ public object StreamCrypto {
         ).newDecryptingStream(inputStream, associatedData)
     }
 
-    private fun Long.toByteArray() = ByteArray(8).apply {
+    public fun Long.toByteArray(): ByteArray = ByteArray(8).apply {
         var l = this@toByteArray
         for (i in 7 downTo 0) {
             this[i] = (l and 0xFF).toByte()