Merge pull request #582 from seedvault-app/feature/e2e-test-logs

Fix and improve E2E tests

.cirrus.yml

@@ -1,7 +1,7 @@
 task:
   name: Build with AOSP
   only_if: $CIRRUS_PR_LABELS =~ ".*aosp-build.*"
-  timeout_in: 120m
+  timeout_in: 70m
   container:
     image: ubuntu:23.04
     cpu: 8
@@ -10,4 +10,4 @@ task:
     - ./.github/scripts/build_aosp.sh aosp_arm64 userdebug android-14.0.0_r1
   always:
     seedvault_artifacts:
-      path: $CIRRUS_WORKING_DIR/Seedvault.apk
+      path: Seedvault.apk

.github/scripts/run_tests.sh

@@ -0,0 +1,35 @@
+adb root
+sleep 5
+adb remount
+
+echo "Installing Seedvault app..."
+adb shell mkdir -p /system/priv-app/Seedvault
+adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk
+
+echo "Installing Seedvault permissions..."
+adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml
+adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml
+
+echo "Setting Seedvault transport..."
+sleep 10
+adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport
+
+large_test_exit_code=0
+./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$?
+
+adb pull /sdcard/seedvault_test_results
+
+if [ "$large_test_exit_code" -ne 0 ]; then
+    echo 'Large tests failed.'
+    exit 1
+fi
+
+medium_test_exit_code=0
+./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$?
+
+if [ "$medium_test_exit_code" -ne 0 ]; then
+    echo 'Medium tests failed.'
+    exit 1
+fi
+
+exit 0

.github/workflows/build.yml

@@ -1,6 +1,10 @@
 name: Build
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     name: Build

.github/workflows/test.yml

@@ -13,14 +13,13 @@ concurrency:
 jobs:
   instrumentation_tests:
     runs-on: macos-11
+    if: github.repository == 'seedvault-app/seedvault'
+    timeout-minutes: 80
     strategy:
       fail-fast: false
       matrix:
-        android_target: [33, 34]
-        emulator_type: [default, google_apis]
-        exclude:
-          - android_target: 34
-            emulator_type: default
+        android_target: [ 33, 34 ]
+        emulator_type: [ default ]
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -31,88 +30,33 @@ jobs:
           java-version: '17'
           cache: 'gradle'
 
-      - name: AVD cache
-        uses: actions/cache@v3
-        id: avd-cache
-        with:
-          path: |
-            ~/.android/avd/*
-            ~/.android/adb*
-          key: aosp-${{ matrix.emulator_type }}-${{ matrix.android_target }}-${{ runner.os }}
-
       - name: Build Release APK
         run: ./gradlew :app:assembleRelease
 
-      - name: Create AVD snapshot
-        if: steps.avd-cache.outputs.cache-hit != 'true'
-        uses: reactivecircus/android-emulator-runner@v2
-        with:
-          api-level: ${{ matrix.android_target }}
-          target: ${{ matrix.emulator_type }}
-          arch: x86_64
-          force-avd-creation: false
-          emulator-options: -writable-system -no-snapshot-load -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
-          disable-animations: true
-          script: |
-            ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64"
-            echo "Generated AVD snapshot for caching."
-
       - name: Assemble tests
         run: ./gradlew :app:assembleAndroidTest
 
       - name: Run tests
-        uses: reactivecircus/android-emulator-runner@v2
+        uses: Wandalen/wretry.action@v1.3.0
         with:
-          api-level: ${{ matrix.android_target }}
-          target: ${{ matrix.emulator_type }}
-          arch: x86_64
-          force-avd-creation: false
-          emulator-options: -writable-system -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
-          profile: pixel_6a
-          heap-size: '512M'
-          ram-size: '4096M'
-          disk-size: '14G'
-          sdcard-path-or-size: '4096M'
-          cores: 3
-          disable-animations: false
-          script: |
-            adb root
-            sleep 5
-            adb remount
+          attempt_limit: 3
+          action: reactivecircus/android-emulator-runner@v2
+          with: |
+            api-level: ${{ matrix.android_target }}
+            target: ${{ matrix.emulator_type }}
+            arch: x86_64
+            force-avd-creation: true
+            emulator-options: -cores 2 -writable-system -no-snapshot-load -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+            disk-size: '14G'
+            sdcard-path-or-size: '4096M'
+            disable-animations: true
+            script: |
+              ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64"
+              ./.github/scripts/run_tests.sh
 
-            echo "Installing Seedvault app..."
-            adb shell mkdir -p /system/priv-app/Seedvault
-            adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk
-
-            echo "Installing Seedvault permissions..."
-            adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml
-            adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml
-
-            echo "Setting Seedvault transport..."
-            sleep 10
-            adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport
-
-            wget --quiet https://github.com/seedvault-app/seedvault-test-data/releases/download/1/backup.tar.gz
-            adb shell mkdir -p /sdcard/seedvault_baseline
-            adb push backup.tar.gz /sdcard/seedvault_baseline
-            adb wait-for-device
-            adb shell tar xzf /sdcard/seedvault_baseline/backup.tar.gz --directory=/sdcard/seedvault_baseline
-            adb shell rm /sdcard/seedvault_baseline/backup.tar.gz
-
-            large_test_exit_code=0
-            ./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$?
-
-            medium_test_exit_code=0
-            ./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$?
-
-            adb pull /sdcard/seedvault_test_videos
-
-            if [ $large_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi
-            if [ $medium_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi
-
-      - name: Upload screenshots and videos
+      - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: seedvault_test_videos
-          path: seedvault_test_videos/**/*.mp4
+          name: ${{ matrix.emulator_type }}-${{ matrix.android_target }}-results
+          path: seedvault_test_results/**/*

app/build.gradle

@@ -162,7 +162,7 @@ dependencies {
     androidTestImplementation 'androidx.test:runner:1.4.0'
     androidTestImplementation 'androidx.test:rules:1.4.0'
     androidTestImplementation 'androidx.test.ext:junit:1.1.3'
-    androidTestImplementation "io.mockk:mockk-android:$mockk_version"
+    androidTestImplementation "io.mockk:mockk-android:1.13.8"
     androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0'
 }
 
@@ -200,7 +200,7 @@ tasks.register('provisionEmulator', Exec) {
     doFirst {
         commandLine "${project.projectDir}/development/scripts/provision_emulator.sh",
                 "seedvault",
-                "system-images;android-34;google_apis;x86_64"
+                "system-images;android-34;default;x86_64"
 
         environment "ANDROID_HOME", android.sdkDirectory.absolutePath
         environment "JAVA_HOME", System.properties['java.home']

app/development/scripts/provision_emulator.sh

@@ -20,7 +20,7 @@ DEVELOPMENT_DIR=$SCRIPT_DIR/..
 ROOT_PROJECT_DIR=$SCRIPT_DIR/../../..
 
 echo "Downloading system image..."
-$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE"
+yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE"
 
 # create AVD if it doesn't exist
 if $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager list avd | grep -q "$EMULATOR_NAME"; then

app/development/scripts/start_emulator.sh

@@ -14,9 +14,5 @@ fi
 
 EMULATOR_NAME=$1
 
-SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
-DEVELOPMENT_DIR=$SCRIPT_DIR/..
-ROOT_PROJECT_DIR=$SCRIPT_DIR/../../..
-
 echo "Starting emulator..."
 nohup $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -gpu swiftshader_indirect -writable-system >/dev/null 2>&1 &

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt

@@ -44,7 +44,7 @@ internal interface LargeTestBase : KoinComponent {
 
     companion object {
         private const val TEST_STORAGE_FOLDER = "seedvault_test"
-        private const val TEST_VIDEO_FOLDER = "seedvault_test_videos"
+        private const val TEST_VIDEO_FOLDER = "seedvault_test_results"
     }
 
     val externalStorageDir: String get() = Environment.getExternalStorageDirectory().absolutePath
@@ -106,19 +106,23 @@ internal interface LargeTestBase : KoinComponent {
         uiAutomation.executeShellCommand(command).close()
     }
 
+    fun testResultFilename(testName: String): String {
+        val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss")
+        val timeStamp = simpleDateFormat.format(Calendar.getInstance().time)
+        return "${timeStamp}_${testName.replace(" ", "_")}"
+    }
+
     @OptIn(DelicateCoroutinesApi::class)
     @WorkerThread
-    suspend fun startScreenRecord(
+    suspend fun startRecordingTest(
         keepRecordingScreen: AtomicBoolean,
         testName: String,
     ) {
-        val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss")
-        val timeStamp = simpleDateFormat.format(Calendar.getInstance().time)
-        val fileName = "${timeStamp}_${testName.replace(" ", "_")}"
-
         val folder = testVideoPath
         runCommand("mkdir -p $folder")
 
+        val fileName = testResultFilename(testName)
+
         // screen record automatically stops after 3 minutes
         // we need to block on a loop and split it into multiple files
         GlobalScope.launch(Dispatchers.IO) {
@@ -131,10 +135,16 @@ internal interface LargeTestBase : KoinComponent {
     }
 
     @WorkerThread
-    fun stopScreenRecord(keepRecordingScreen: AtomicBoolean) {
+    fun stopRecordingTest(
+        keepRecordingScreen: AtomicBoolean,
+        testName: String,
+    ) {
         keepRecordingScreen.set(false)
-
         runCommand("pkill -2 screenrecord")
+
+        // write logcat to file
+        val fileName = testResultFilename(testName)
+        runCommand("logcat -d -f $testVideoPath/$fileName.log")
     }
 
     fun uninstallPackages(packages: Collection<PackageInfo>) {

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt

@@ -38,13 +38,13 @@ internal abstract class SeedvaultLargeTest :
         resetApplicationState()
         clearTestBackups()
 
-        startScreenRecord(keepRecordingScreen, name.methodName)
+        startRecordingTest(keepRecordingScreen, name.methodName)
         restoreBaselineBackup()
     }
 
     @After
     open fun tearDown() {
-        stopScreenRecord(keepRecordingScreen)
+        stopRecordingTest(keepRecordingScreen, name.methodName)
     }
 
     /**

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt

@@ -11,6 +11,10 @@ import java.lang.Thread.sleep
 
 abstract class UiDeviceScreen<T> {
 
+    companion object {
+        private const val SELECTOR_TIMEOUT = 180000L
+    }
+
     operator fun invoke(function: T.() -> Unit) {
         function.invoke(this as T)
     }
@@ -18,8 +22,11 @@ abstract class UiDeviceScreen<T> {
     fun UiObject.scrollTo(
         scrollSelector: UiSelector = UiSelector().className(ScrollView::class.java),
     ): UiObject {
-        UiScrollable(scrollSelector).scrollIntoView(this)
-        waitForExists(15000)
+        val uiScrollable = UiScrollable(scrollSelector)
+        uiScrollable.waitForExists(SELECTOR_TIMEOUT)
+        uiScrollable.scrollIntoView(this)
+        waitForExists(SELECTOR_TIMEOUT)
+
         sleep(2000)
         return this
     }
@@ -32,6 +39,6 @@ abstract class UiDeviceScreen<T> {
 
     private fun device() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
         .also {
-            Configurator.getInstance().waitForSelectorTimeout = 60000
+            Configurator.getInstance().waitForSelectorTimeout = SELECTOR_TIMEOUT
         }
 }