Add end-to-end emulator test running on CI

This commit is contained in:
Steve Soltys 2023-09-12 22:17:39 +00:00
parent c96f1c961a
commit f1238977d0
21 changed files with 519 additions and 20 deletions

53
.cirrus.yml Normal file
View file

@ -0,0 +1,53 @@
container:
image: ghcr.io/cirruslabs/android-sdk:33
kvm: true
cpu: 8
memory: 16G
check_android_task:
skip: "!changesInclude('.cirrus.yml', '*.gradle', '*.gradle.kts', '**/*.gradle', '**/*.gradle.kts', '*.properties', '**/*.properties', '**/*.kt', '**/*.xml')"
create_avd_script:
start_avd_background_script:
sdkmanager --install "system-images;android-33;google_apis;x86_64";
echo no | avdmanager create avd -n seedvault -k "system-images;android-33;google_apis;x86_64";
$ANDROID_HOME/emulator/emulator
-avd seedvault
-no-audio
-no-boot-anim
-gpu swiftshader_indirect
-no-snapshot
-no-window
-writable-system;
provision_avd_background_script:
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
wget --output-document etar.apk https://f-droid.org/repo/ws.xsoh.etar_35.apk;
adb install etar.apk
adb root;
sleep 5;
adb remount;
adb reboot;
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
adb root;
sleep 5;
adb remount;
assemble_release_script:
./gradlew :app:assembleRelease :app:assembleAndroidTest
install_app_script:
timeout 180s bash -c 'while [[ -z $(adb shell mount | grep "/system " | grep "(rw,") ]]; do sleep 1; done;';
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
adb shell mkdir -p /system/priv-app/Seedvault;
adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk;
adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml;
adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml;
adb shell bmgr enable true;
adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport;
adb reboot;
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
run_large_tests_script: ./gradlew -Pinstrumented_test_size=large :app:connectedAndroidTest
run_medium_tests_script: ./gradlew -Pinstrumented_test_size=medium :app:connectedAndroidTest
always:
pull_screenshots_script:
adb pull /sdcard/Documents/screenshots
screenshots_artifacts:
path: "screenshots/**/*.png"

View file

@ -17,7 +17,7 @@
<option name="TARGET_SELECTION_MODE" value="SHOW_DIALOG" />
<option name="SELECTED_CLOUD_MATRIX_CONFIGURATION_ID" value="-1" />
<option name="SELECTED_CLOUD_MATRIX_PROJECT_ID" value="" />
<option name="DEBUGGER_TYPE" value="Auto" />
<option name="DEBUGGER_TYPE" value="Java" />
<Auto>
<option name="USE_JAVA_AWARE_DEBUGGER" value="false" />
<option name="SHOW_STATIC_VARS" value="true" />
@ -37,7 +37,7 @@
<option name="DEBUG_SANDBOX_SDK" value="false" />
</Hybrid>
<Java>
<option name="ATTACH_ON_WAIT_FOR_DEBUGGER" value="false" />
<option name="ATTACH_ON_WAIT_FOR_DEBUGGER" value="true" />
<option name="DEBUG_SANDBOX_SDK" value="false" />
</Java>
<Native>
@ -62,7 +62,7 @@
<option name="SEARCH_ACTIVITY_IN_GLOBAL_SCOPE" value="false" />
<option name="SKIP_ACTIVITY_VALIDATION" value="false" />
<method v="2">
<option name="RunConfigurationTask" enabled="false" run_configuration_name="seedvault [installEmulatorRelease]" run_configuration_type="GradleRunConfiguration" />
<option name="Gradle.BeforeRunTask" enabled="false" tasks="installEmulatorRelease" externalProjectPath="$PROJECT_DIR$/app" vmOptions="" scriptParameters="" />
</method>
</configuration>
</component>

View file

@ -21,8 +21,15 @@ android {
minSdk 32 // leave at 32 for robolectric tests
targetSdk rootProject.ext.targetSdk
versionNameSuffix "-$gitDescribe"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunner "com.stevesoltys.seedvault.KoinInstrumentationTestRunner"
testInstrumentationRunnerArguments disableAnalytics: 'true'
if (project.hasProperty('instrumented_test_size')) {
final testSize = project.getProperty('instrumented_test_size')
println("Instrumented test size: $testSize")
testInstrumentationRunnerArguments size: testSize
}
}
buildTypes {
@ -150,10 +157,13 @@ dependencies {
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit5_version"
testRuntimeOnly "org.junit.vintage:junit-vintage-engine:$junit5_version"
androidTestImplementation rootProject.ext.aosp_libs
androidTestImplementation 'androidx.test:runner:1.4.0'
androidTestImplementation 'androidx.test:rules:1.4.0'
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
androidTestImplementation "io.mockk:mockk-android:$mockk_version"
androidTestImplementation 'com.kaspersky.android-components:kaspresso:1.5.3'
}
apply from: "${rootProject.rootDir}/gradle/ktlint.gradle"
@ -210,3 +220,14 @@ tasks.register('installEmulatorRelease', Exec) {
environment "JAVA_HOME", System.properties['java.home']
}
}
tasks.register('clearEmulatorAppData', Exec) {
group("emulator")
doFirst {
commandLine "${project.projectDir}/development/scripts/clear_app_data.sh"
environment "ANDROID_SDK_HOME", android.sdkDirectory.absolutePath
environment "JAVA_HOME", System.properties['java.home']
}
}

View file

@ -0,0 +1,22 @@
#!/usr/bin/env bash
# assert ANDROID_HOME is set
if [ -z "$ANDROID_SDK_HOME" ]; then
echo "ANDROID_SDK_HOME is not set"
exit 1
fi
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
DEVELOPMENT_DIR=$SCRIPT_DIR/..
ROOT_PROJECT_DIR=$SCRIPT_DIR/../../..
EMULATOR_DEVICE_NAME=$($ANDROID_SDK_HOME/platform-tools/adb devices | grep emulator | cut -f1)
if [ -z "$EMULATOR_DEVICE_NAME" ]; then
echo "Emulator device name not found"
exit 1
fi
ADB="$ANDROID_SDK_HOME/platform-tools/adb -s $EMULATOR_DEVICE_NAME"
$ADB shell pm clear com.stevesoltys.seedvault

View file

@ -31,7 +31,6 @@ else
sleep 1
fi
echo "Starting emulator..."
$SCRIPT_DIR/start_emulator.sh "$EMULATOR_NAME"
sleep 3

View file

@ -0,0 +1,35 @@
package com.stevesoltys.seedvault
import androidx.test.platform.app.InstrumentationRegistry
import com.stevesoltys.seedvault.restore.RestoreViewModel
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import io.mockk.spyk
import org.koin.core.module.Module
import org.koin.dsl.module
private val spyBackupNotificationManager = spyk(
BackupNotificationManager(
InstrumentationRegistry.getInstrumentation()
.targetContext.applicationContext
)
)
class KoinInstrumentationTestApp : App() {
override fun appModules(): List<Module> {
val testModule = module {
single { spyBackupNotificationManager }
single {
spyk(
RestoreViewModel(
this@KoinInstrumentationTestApp,
get(), get(), get(), get(), get(), get()
)
)
}
}
return super.appModules().plus(testModule)
}
}

View file

@ -0,0 +1,20 @@
package com.stevesoltys.seedvault
import android.app.Application
import android.content.Context
import androidx.test.runner.AndroidJUnitRunner
class KoinInstrumentationTestRunner : AndroidJUnitRunner() {
override fun newApplication(
classLoader: ClassLoader?,
className: String?,
context: Context?,
): Application {
return super.newApplication(
classLoader,
KoinInstrumentationTestApp::class.java.name,
context
)
}
}

View file

@ -2,6 +2,7 @@ package com.stevesoltys.seedvault
import androidx.test.core.content.pm.PackageInfoBuilder
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.MediumTest
import androidx.test.platform.app.InstrumentationRegistry
import com.stevesoltys.seedvault.plugins.LegacyStoragePlugin
import com.stevesoltys.seedvault.plugins.StoragePlugin
@ -28,6 +29,7 @@ import org.koin.core.component.inject
@RunWith(AndroidJUnit4::class)
@Suppress("BlockingMethodInNonBlockingContext")
@MediumTest
class PluginTest : KoinComponent {
private val context = InstrumentationRegistry.getInstrumentation().targetContext

View file

@ -0,0 +1,139 @@
package com.stevesoltys.seedvault.e2e
import androidx.test.filters.LargeTest
import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
import com.stevesoltys.seedvault.restore.RestoreViewModel
import com.stevesoltys.seedvault.transport.backup.PackageService
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import io.mockk.every
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import org.junit.Test
import org.koin.core.component.inject
import java.util.concurrent.atomic.AtomicBoolean
@LargeTest
class BackupRestoreTest : LargeTestBase() {
private val packageService: PackageService by inject()
private val spyBackupNotificationManager: BackupNotificationManager by inject()
private val restoreViewModel: RestoreViewModel by inject()
companion object {
private const val BACKUP_TIMEOUT = 360 * 1000L
private const val RESTORE_TIMEOUT = 360 * 1000L
}
@Test
fun `backup and restore applications`() = run {
launchBackupActivity()
verifyCode()
chooseBackupLocation()
val eligiblePackages = launchAllEligibleApps()
performBackup(eligiblePackages)
uninstallPackages(eligiblePackages)
performRestore()
val packagesAfterRestore = getEligibleApps()
assert(eligiblePackages == packagesAfterRestore)
}
private fun getEligibleApps() = packageService.userApps
.map { it.packageName }.toSet()
private fun launchAllEligibleApps(): Set<String> {
return getEligibleApps().onEach {
val intent = device.targetContext.packageManager.getLaunchIntentForPackage(it)
device.targetContext.startActivity(intent)
waitUntilIdle()
}
}
private fun performBackup(expectedPackages: Set<String>) = run {
val backupResult = spyOnBackup(expectedPackages)
startBackup()
waitForBackupResult(backupResult)
screenshot("backup result")
}
private fun spyOnBackup(expectedPackages: Set<String>): AtomicBoolean {
val finishedBackup = AtomicBoolean(false)
every {
spyBackupNotificationManager.onBackupFinished(any(), any())
} answers {
val success = firstArg<Boolean>()
assert(success) { "Backup failed." }
val packageCount = secondArg<Int>()
assert(packageCount == expectedPackages.size) {
"Expected ${expectedPackages.size} apps, got $packageCount."
}
this.callOriginal()
finishedBackup.set(true)
}
return finishedBackup
}
private fun waitForBackupResult(finishedBackup: AtomicBoolean) = run {
step("Wait for backup completion") {
runBlocking {
withTimeout(BACKUP_TIMEOUT) {
while (!finishedBackup.get()) {
delay(100)
}
}
}
}
}
private fun performRestore() = run {
step("Start restore and await completion") {
RestoreScreen {
startRestore()
waitForInstallResult()
screenshot("restore app apks result")
nextButton.click()
waitForRestoreResult()
screenshot("restore app data result")
finishButton.click()
}
}
}
private fun waitForInstallResult() = runBlocking {
withTimeout(RESTORE_TIMEOUT) {
while (restoreViewModel.installResult.value == null) {
delay(100)
}
val restoreResultValue = restoreViewModel.installResult.value!!
assert(!restoreResultValue.hasFailed) { "Failed to install packages" }
}
}
private fun waitForRestoreResult() = runBlocking {
withTimeout(RESTORE_TIMEOUT) {
while (restoreViewModel.restoreBackupResult.value == null) {
delay(100)
}
val restoreResultValue = restoreViewModel.restoreBackupResult.value!!
assert(!restoreResultValue.hasError()) {
"Restore failed: ${restoreResultValue.errorMsg}"
}
}
}
}

View file

@ -0,0 +1,110 @@
package com.stevesoltys.seedvault.e2e
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import com.kaspersky.kaspresso.testcases.api.testcase.TestCase
import com.stevesoltys.seedvault.e2e.screen.impl.BackupScreen
import com.stevesoltys.seedvault.e2e.screen.impl.DocumentPickerScreen
import com.stevesoltys.seedvault.e2e.screen.impl.RecoveryCodeScreen
import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
import org.junit.After
import org.junit.Before
import org.junit.runner.RunWith
import org.koin.core.component.KoinComponent
import java.lang.Thread.sleep
@RunWith(AndroidJUnit4::class)
abstract class LargeTestBase : TestCase(), KoinComponent {
@Before
open fun setUp() {
// reset document picker state, and delete old backups
runCommand("pm clear com.google.android.documentsui")
runCommand("rm -Rf /sdcard/seedvault")
}
@After
open fun tearDown() {
screenshot("end")
}
protected fun launchBackupActivity() = run {
runCommand("am start -n ${device.targetContext.packageName}/.settings.SettingsActivity")
waitUntilIdle()
}
protected fun launchRestoreActivity() = run {
runCommand("am start -n ${device.targetContext.packageName}/.restore.RestoreActivity")
waitUntilIdle()
}
protected fun waitUntilIdle() {
device.uiDevice.waitForIdle()
sleep(3000)
}
protected fun verifyCode() = run {
RecoveryCodeScreen {
step("Confirm code") {
screenshot("confirm code")
confirmCodeButton.click()
}
step("Verify code") {
screenshot("verify code")
verifyCodeButton.scrollTo()
verifyCodeButton.click()
}
}
}
protected fun chooseBackupLocation() = run {
step("Choose backup location") {
waitUntilIdle()
screenshot("choose backup location")
DocumentPickerScreen {
createNewFolderButton.click()
textBox.text = "seedvault"
okButton.click()
useThisFolderButton.click()
allowButton.click()
}
}
}
protected fun startBackup() = run {
launchBackupActivity()
step("Run backup") {
BackupScreen {
backupMenu.clickAndWaitForNewWindow()
backupNowButton.clickAndWaitForNewWindow()
backupStatusButton.clickAndWaitForNewWindow()
}
}
}
protected fun startRestore() = run {
launchRestoreActivity()
step("Restore backup") {
RestoreScreen {
backupListItem.clickAndWaitForNewWindow()
}
}
}
protected fun uninstallPackages(packages: Set<String>) {
packages.forEach { runCommand("pm uninstall $it") }
}
protected fun runCommand(command: String) {
InstrumentationRegistry.getInstrumentation().uiAutomation
.executeShellCommand(command)
.close()
}
protected fun screenshot(name: String) {
device.screenshots.take(name.replace(" ", "_"))
}
}

View file

@ -0,0 +1,20 @@
package com.stevesoltys.seedvault.e2e.screen
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.uiautomator.UiDevice
import androidx.test.uiautomator.UiObject
import androidx.test.uiautomator.UiSelector
abstract class UiDeviceScreen<T> {
operator fun invoke(function: T.() -> Unit) {
function.invoke(this as T)
}
protected fun findObject(
block: UiSelector.() -> UiSelector,
): UiObject = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
.findObject(
UiSelector().let { it.block() }
)
}

View file

@ -0,0 +1,12 @@
package com.stevesoltys.seedvault.e2e.screen.impl
import com.stevesoltys.seedvault.e2e.screen.UiDeviceScreen
object BackupScreen : UiDeviceScreen<BackupScreen>() {
val backupMenu = findObject { description("More options") }
val backupNowButton = findObject { text("Backup now") }
val backupStatusButton = findObject { text("Backup status") }
}

View file

@ -0,0 +1,17 @@
package com.stevesoltys.seedvault.e2e.screen.impl
import android.widget.EditText
import com.stevesoltys.seedvault.e2e.screen.UiDeviceScreen
object DocumentPickerScreen : UiDeviceScreen<DocumentPickerScreen>() {
val createNewFolderButton = findObject { text("CREATE NEW FOLDER") }
val useThisFolderButton = findObject { text("USE THIS FOLDER") }
val textBox = findObject { className(EditText::class.java) }
val okButton = findObject { text("OK") }
val allowButton = findObject { text("ALLOW") }
}

View file

@ -0,0 +1,15 @@
package com.stevesoltys.seedvault.e2e.screen.impl
import com.kaspersky.kaspresso.screens.KScreen
import com.stevesoltys.seedvault.R
import io.github.kakaocup.kakao.text.KButton
object RecoveryCodeScreen : KScreen<RecoveryCodeScreen>() {
override val layoutId: Int? = null
override val viewClass: Class<*>? = null
val confirmCodeButton = KButton { withId(R.id.confirmCodeButton) }
val verifyCodeButton = KButton { withId(R.id.doneButton) }
}

View file

@ -0,0 +1,12 @@
package com.stevesoltys.seedvault.e2e.screen.impl
import com.stevesoltys.seedvault.e2e.screen.UiDeviceScreen
object RestoreScreen : UiDeviceScreen<RestoreScreen>() {
val backupListItem = findObject { textContains("Last backup 0 hr") }
val nextButton = findObject { text("Next") }
val finishButton = findObject { text("Finish") }
}

View file

@ -7,6 +7,7 @@ import android.os.Bundle
import android.provider.DocumentsContract.EXTRA_LOADING
import androidx.documentfile.provider.DocumentFile
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.MediumTest
import androidx.test.platform.app.InstrumentationRegistry
import com.stevesoltys.seedvault.assertReadEquals
import com.stevesoltys.seedvault.coAssertThrows
@ -39,6 +40,7 @@ import kotlin.random.Random
@RunWith(AndroidJUnit4::class)
@Suppress("BlockingMethodInNonBlockingContext")
@MediumTest
class DocumentsStorageTest : KoinComponent {
private val context = InstrumentationRegistry.getInstrumentation().targetContext

View file

@ -2,12 +2,14 @@ package com.stevesoltys.seedvault.transport.backup
import android.util.Log
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.MediumTest
import org.junit.Test
import org.junit.runner.RunWith
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
@RunWith(AndroidJUnit4::class)
@MediumTest
class PackageServiceTest : KoinComponent {
private val packageService: PackageService by inject()

View file

@ -84,8 +84,10 @@ open class App : Application() {
protected open fun startKoin() = startKoin {
androidLogger(Level.ERROR)
androidContext(this@App)
modules(
listOf(
modules(appModules())
}
open fun appModules() = listOf(
cryptoModule,
headerModule,
metadataModule,
@ -96,8 +98,6 @@ open class App : Application() {
storageModule,
appModule
)
)
}
private val settingsManager: SettingsManager by inject()
private val metadataManager: MetadataManager by inject()

View file

@ -1,5 +1,8 @@
org.gradle.jvmargs=-Xmx1g
org.gradle.jvmargs=-Xmx4g
org.gradle.configureondemand=true
org.gradle.caching=true
org.gradle.parallel=true
org.gradle.daemon=true
android.useAndroidX=true
android.enableJetifier=false
kotlin.code.style=official

View file

@ -23,7 +23,7 @@ ext.aosp_libs = fileTree(include: [
'android.jar',
// out/target/common/obj/JAVA_LIBRARIES/core-libart.com.android.art_intermediates/classes.jar
'libcore.jar',
], dir: "$projectDir/app/libs")
], dir: "${rootProject.projectDir}/app/libs")
ext.kotlin_libs = [
std: [

View file

@ -18,3 +18,18 @@ include ':app'
include ':contactsbackup'
include ':storage:lib'
include ':storage:demo'
ext.isCiServer = System.getenv().containsKey("CIRRUS_CI")
ext.isMasterBranch = System.getenv().getOrDefault("CIRRUS_BRANCH", "").matches("android[0-9]+")
ext.buildCacheHost = System.getenv().getOrDefault("CIRRUS_HTTP_CACHE_HOST", "localhost:12321")
buildCache {
local {
enabled = !isCiServer
}
remote(HttpBuildCache) {
url = "http://${buildCacheHost}/"
enabled = isCiServer
push = isMasterBranch
}
}