From 8297bb70879cb3e9d2fe1da3768fb2e9d8bf0688 Mon Sep 17 00:00:00 2001 From: Steve Soltys Date: Thu, 12 Oct 2023 02:13:34 -0400 Subject: [PATCH] Fix and improve E2E tests --- .github/scripts/run_tests.sh | 35 ++++++ .github/workflows/build.yml | 4 + .github/workflows/test.yml | 100 ++++-------------- app/build.gradle | 4 +- app/development/scripts/provision_emulator.sh | 2 +- app/development/scripts/start_emulator.sh | 4 - .../seedvault/e2e/LargeTestBase.kt | 26 +++-- .../seedvault/e2e/SeedvaultLargeTest.kt | 4 +- .../seedvault/e2e/screen/UiDeviceScreen.kt | 13 ++- 9 files changed, 94 insertions(+), 98 deletions(-) create mode 100755 .github/scripts/run_tests.sh diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh new file mode 100755 index 00000000..48a86572 --- /dev/null +++ b/.github/scripts/run_tests.sh @@ -0,0 +1,35 @@ +adb root +sleep 5 +adb remount + +echo "Installing Seedvault app..." +adb shell mkdir -p /system/priv-app/Seedvault +adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk + +echo "Installing Seedvault permissions..." +adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml +adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml + +echo "Setting Seedvault transport..." +sleep 10 +adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport + +large_test_exit_code=0 +./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$? + +adb pull /sdcard/seedvault_test_results + +if [ "$large_test_exit_code" -ne 0 ]; then + echo 'Large tests failed.' + exit 1 +fi + +medium_test_exit_code=0 +./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$? + +if [ "$medium_test_exit_code" -ne 0 ]; then + echo 'Medium tests failed.' + exit 1 +fi + +exit 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82ca5c42..52dd750b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,10 @@ name: Build on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: name: Build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 925a65b1..a624c775 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,14 +13,13 @@ concurrency: jobs: instrumentation_tests: runs-on: macos-11 + if: github.repository == 'seedvault-app/seedvault' + timeout-minutes: 80 strategy: fail-fast: false matrix: - android_target: [33, 34] - emulator_type: [default, google_apis] - exclude: - - android_target: 34 - emulator_type: default + android_target: [ 33, 34 ] + emulator_type: [ default ] steps: - name: Checkout Code uses: actions/checkout@v3 @@ -31,88 +30,33 @@ jobs: java-version: '17' cache: 'gradle' - - name: AVD cache - uses: actions/cache@v3 - id: avd-cache - with: - path: | - ~/.android/avd/* - ~/.android/adb* - key: aosp-${{ matrix.emulator_type }}-${{ matrix.android_target }}-${{ runner.os }} - - name: Build Release APK run: ./gradlew :app:assembleRelease - - name: Create AVD snapshot - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: ${{ matrix.android_target }} - target: ${{ matrix.emulator_type }} - arch: x86_64 - force-avd-creation: false - emulator-options: -writable-system -no-snapshot-load -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - disable-animations: true - script: | - ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64" - echo "Generated AVD snapshot for caching." - - name: Assemble tests run: ./gradlew :app:assembleAndroidTest - name: Run tests - uses: reactivecircus/android-emulator-runner@v2 + uses: Wandalen/wretry.action@v1.3.0 with: - api-level: ${{ matrix.android_target }} - target: ${{ matrix.emulator_type }} - arch: x86_64 - force-avd-creation: false - emulator-options: -writable-system -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - profile: pixel_6a - heap-size: '512M' - ram-size: '4096M' - disk-size: '14G' - sdcard-path-or-size: '4096M' - cores: 3 - disable-animations: false - script: | - adb root - sleep 5 - adb remount + attempt_limit: 3 + action: reactivecircus/android-emulator-runner@v2 + with: | + api-level: ${{ matrix.android_target }} + target: ${{ matrix.emulator_type }} + arch: x86_64 + force-avd-creation: true + emulator-options: -cores 2 -writable-system -no-snapshot-load -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disk-size: '14G' + sdcard-path-or-size: '4096M' + disable-animations: true + script: | + ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64" + ./.github/scripts/run_tests.sh - echo "Installing Seedvault app..." - adb shell mkdir -p /system/priv-app/Seedvault - adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk - - echo "Installing Seedvault permissions..." - adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml - adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml - - echo "Setting Seedvault transport..." - sleep 10 - adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport - - wget --quiet https://github.com/seedvault-app/seedvault-test-data/releases/download/1/backup.tar.gz - adb shell mkdir -p /sdcard/seedvault_baseline - adb push backup.tar.gz /sdcard/seedvault_baseline - adb wait-for-device - adb shell tar xzf /sdcard/seedvault_baseline/backup.tar.gz --directory=/sdcard/seedvault_baseline - adb shell rm /sdcard/seedvault_baseline/backup.tar.gz - - large_test_exit_code=0 - ./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$? - - medium_test_exit_code=0 - ./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$? - - adb pull /sdcard/seedvault_test_videos - - if [ $large_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi - if [ $medium_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi - - - name: Upload screenshots and videos + - name: Upload test results if: always() uses: actions/upload-artifact@v3 with: - name: seedvault_test_videos - path: seedvault_test_videos/**/*.mp4 + name: ${{ matrix.emulator_type }}-${{ matrix.android_target }}-results + path: seedvault_test_results/**/* diff --git a/app/build.gradle b/app/build.gradle index 56508f33..fcb46e4e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -162,7 +162,7 @@ dependencies { androidTestImplementation 'androidx.test:runner:1.4.0' androidTestImplementation 'androidx.test:rules:1.4.0' androidTestImplementation 'androidx.test.ext:junit:1.1.3' - androidTestImplementation "io.mockk:mockk-android:$mockk_version" + androidTestImplementation "io.mockk:mockk-android:1.13.8" androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0' } @@ -200,7 +200,7 @@ tasks.register('provisionEmulator', Exec) { doFirst { commandLine "${project.projectDir}/development/scripts/provision_emulator.sh", "seedvault", - "system-images;android-34;google_apis;x86_64" + "system-images;android-34;default;x86_64" environment "ANDROID_HOME", android.sdkDirectory.absolutePath environment "JAVA_HOME", System.properties['java.home'] diff --git a/app/development/scripts/provision_emulator.sh b/app/development/scripts/provision_emulator.sh index cfdcc887..fef04b8b 100755 --- a/app/development/scripts/provision_emulator.sh +++ b/app/development/scripts/provision_emulator.sh @@ -20,7 +20,7 @@ DEVELOPMENT_DIR=$SCRIPT_DIR/.. ROOT_PROJECT_DIR=$SCRIPT_DIR/../../.. echo "Downloading system image..." -$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE" +yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE" # create AVD if it doesn't exist if $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager list avd | grep -q "$EMULATOR_NAME"; then diff --git a/app/development/scripts/start_emulator.sh b/app/development/scripts/start_emulator.sh index 4c071184..b2113487 100755 --- a/app/development/scripts/start_emulator.sh +++ b/app/development/scripts/start_emulator.sh @@ -14,9 +14,5 @@ fi EMULATOR_NAME=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -DEVELOPMENT_DIR=$SCRIPT_DIR/.. -ROOT_PROJECT_DIR=$SCRIPT_DIR/../../.. - echo "Starting emulator..." nohup $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -gpu swiftshader_indirect -writable-system >/dev/null 2>&1 & diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt index b386214f..86f14a28 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt @@ -44,7 +44,7 @@ internal interface LargeTestBase : KoinComponent { companion object { private const val TEST_STORAGE_FOLDER = "seedvault_test" - private const val TEST_VIDEO_FOLDER = "seedvault_test_videos" + private const val TEST_VIDEO_FOLDER = "seedvault_test_results" } val externalStorageDir: String get() = Environment.getExternalStorageDirectory().absolutePath @@ -106,19 +106,23 @@ internal interface LargeTestBase : KoinComponent { uiAutomation.executeShellCommand(command).close() } + fun testResultFilename(testName: String): String { + val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss") + val timeStamp = simpleDateFormat.format(Calendar.getInstance().time) + return "${timeStamp}_${testName.replace(" ", "_")}" + } + @OptIn(DelicateCoroutinesApi::class) @WorkerThread - suspend fun startScreenRecord( + suspend fun startRecordingTest( keepRecordingScreen: AtomicBoolean, testName: String, ) { - val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss") - val timeStamp = simpleDateFormat.format(Calendar.getInstance().time) - val fileName = "${timeStamp}_${testName.replace(" ", "_")}" - val folder = testVideoPath runCommand("mkdir -p $folder") + val fileName = testResultFilename(testName) + // screen record automatically stops after 3 minutes // we need to block on a loop and split it into multiple files GlobalScope.launch(Dispatchers.IO) { @@ -131,10 +135,16 @@ internal interface LargeTestBase : KoinComponent { } @WorkerThread - fun stopScreenRecord(keepRecordingScreen: AtomicBoolean) { + fun stopRecordingTest( + keepRecordingScreen: AtomicBoolean, + testName: String, + ) { keepRecordingScreen.set(false) - runCommand("pkill -2 screenrecord") + + // write logcat to file + val fileName = testResultFilename(testName) + runCommand("logcat -d -f $testVideoPath/$fileName.log") } fun uninstallPackages(packages: Collection) { diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt index 423c461e..2d2be5f1 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt @@ -38,13 +38,13 @@ internal abstract class SeedvaultLargeTest : resetApplicationState() clearTestBackups() - startScreenRecord(keepRecordingScreen, name.methodName) + startRecordingTest(keepRecordingScreen, name.methodName) restoreBaselineBackup() } @After open fun tearDown() { - stopScreenRecord(keepRecordingScreen) + stopRecordingTest(keepRecordingScreen, name.methodName) } /** diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt index fdcdd67b..bc13414f 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt @@ -11,6 +11,10 @@ import java.lang.Thread.sleep abstract class UiDeviceScreen { + companion object { + private const val SELECTOR_TIMEOUT = 180000L + } + operator fun invoke(function: T.() -> Unit) { function.invoke(this as T) } @@ -18,8 +22,11 @@ abstract class UiDeviceScreen { fun UiObject.scrollTo( scrollSelector: UiSelector = UiSelector().className(ScrollView::class.java), ): UiObject { - UiScrollable(scrollSelector).scrollIntoView(this) - waitForExists(15000) + val uiScrollable = UiScrollable(scrollSelector) + uiScrollable.waitForExists(SELECTOR_TIMEOUT) + uiScrollable.scrollIntoView(this) + waitForExists(SELECTOR_TIMEOUT) + sleep(2000) return this } @@ -32,6 +39,6 @@ abstract class UiDeviceScreen { private fun device() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) .also { - Configurator.getInstance().waitForSelectorTimeout = 60000 + Configurator.getInstance().waitForSelectorTimeout = SELECTOR_TIMEOUT } }