Address review comments, add screen recording, use test backup data

This commit is contained in:
Steve Soltys 2023-09-14 06:37:38 +00:00
parent f1238977d0
commit 59cef31183
14 changed files with 392 additions and 244 deletions

View file

@ -4,9 +4,9 @@ container:
cpu: 8
memory: 16G
check_android_task:
instrumentation_tests_task:
name: "Cirrus CI Instrumentation Tests"
skip: "!changesInclude('.cirrus.yml', '*.gradle', '*.gradle.kts', '**/*.gradle', '**/*.gradle.kts', '*.properties', '**/*.properties', '**/*.kt', '**/*.xml')"
create_avd_script:
start_avd_background_script:
sdkmanager --install "system-images;android-33;google_apis;x86_64";
echo no | avdmanager create avd -n seedvault -k "system-images;android-33;google_apis;x86_64";
@ -19,10 +19,9 @@ check_android_task:
-no-window
-writable-system;
provision_avd_background_script:
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
wget --output-document etar.apk https://f-droid.org/repo/ws.xsoh.etar_35.apk;
adb install etar.apk
wget https://github.com/seedvault-app/seedvault-test-data/releases/download/1/backup.tar.gz;
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
adb root;
sleep 5;
adb remount;
@ -31,11 +30,17 @@ check_android_task:
adb root;
sleep 5;
adb remount;
assemble_release_script:
sleep 5;
assemble_script:
./gradlew :app:assembleRelease :app:assembleAndroidTest
install_app_script:
timeout 180s bash -c 'while [[ -z $(adb shell mount | grep "/system " | grep "(rw,") ]]; do sleep 1; done;';
adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
adb shell mkdir -p /sdcard/seedvault_baseline;
adb push backup.tar.gz /sdcard/seedvault_baseline/backup.tar.gz;
adb shell tar xzf /sdcard/seedvault_baseline/backup.tar.gz --directory=/sdcard/seedvault_baseline;
adb shell mkdir -p /system/priv-app/Seedvault;
adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk;
adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml;
@ -48,6 +53,6 @@ check_android_task:
run_medium_tests_script: ./gradlew -Pinstrumented_test_size=medium :app:connectedAndroidTest
always:
pull_screenshots_script:
adb pull /sdcard/Documents/screenshots
adb pull /sdcard/seedvault_test_videos
screenshots_artifacts:
path: "screenshots/**/*.png"
path: "seedvault_test_videos/**/*.mp4"

View file

@ -162,8 +162,7 @@ dependencies {
androidTestImplementation 'androidx.test:rules:1.4.0'
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
androidTestImplementation "io.mockk:mockk-android:$mockk_version"
androidTestImplementation 'com.kaspersky.android-components:kaspresso:1.5.3'
androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0'
}
apply from: "${rootProject.rootDir}/gradle/ktlint.gradle"

View file

@ -67,4 +67,13 @@ echo "Rebooting emulator..."
$ADB reboot
$ADB wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;'
echo "Downloading and extracting test backup to '/sdcard/seedvault'..."
wget https://github.com/seedvault-app/seedvault-test-data/releases/download/1/backup.tar.gz
$ADB push backup.tar.gz /sdcard/
rm backup.tar.gz
$ADB shell mkdir -p /sdcard/seedvault_baseline
$ADB shell tar xzf /sdcard/backup.tar.gz --directory=/sdcard/seedvault_baseline
$ADB shell rm /sdcard/backup.tar.gz
echo "Emulator '$EMULATOR_NAME' has been provisioned with Seedvault!"

View file

@ -1,5 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<application android:extractNativeLibs="true" />
</manifest>

View file

@ -1,139 +0,0 @@
package com.stevesoltys.seedvault.e2e
import androidx.test.filters.LargeTest
import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
import com.stevesoltys.seedvault.restore.RestoreViewModel
import com.stevesoltys.seedvault.transport.backup.PackageService
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import io.mockk.every
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import org.junit.Test
import org.koin.core.component.inject
import java.util.concurrent.atomic.AtomicBoolean
@LargeTest
class BackupRestoreTest : LargeTestBase() {
private val packageService: PackageService by inject()
private val spyBackupNotificationManager: BackupNotificationManager by inject()
private val restoreViewModel: RestoreViewModel by inject()
companion object {
private const val BACKUP_TIMEOUT = 360 * 1000L
private const val RESTORE_TIMEOUT = 360 * 1000L
}
@Test
fun `backup and restore applications`() = run {
launchBackupActivity()
verifyCode()
chooseBackupLocation()
val eligiblePackages = launchAllEligibleApps()
performBackup(eligiblePackages)
uninstallPackages(eligiblePackages)
performRestore()
val packagesAfterRestore = getEligibleApps()
assert(eligiblePackages == packagesAfterRestore)
}
private fun getEligibleApps() = packageService.userApps
.map { it.packageName }.toSet()
private fun launchAllEligibleApps(): Set<String> {
return getEligibleApps().onEach {
val intent = device.targetContext.packageManager.getLaunchIntentForPackage(it)
device.targetContext.startActivity(intent)
waitUntilIdle()
}
}
private fun performBackup(expectedPackages: Set<String>) = run {
val backupResult = spyOnBackup(expectedPackages)
startBackup()
waitForBackupResult(backupResult)
screenshot("backup result")
}
private fun spyOnBackup(expectedPackages: Set<String>): AtomicBoolean {
val finishedBackup = AtomicBoolean(false)
every {
spyBackupNotificationManager.onBackupFinished(any(), any())
} answers {
val success = firstArg<Boolean>()
assert(success) { "Backup failed." }
val packageCount = secondArg<Int>()
assert(packageCount == expectedPackages.size) {
"Expected ${expectedPackages.size} apps, got $packageCount."
}
this.callOriginal()
finishedBackup.set(true)
}
return finishedBackup
}
private fun waitForBackupResult(finishedBackup: AtomicBoolean) = run {
step("Wait for backup completion") {
runBlocking {
withTimeout(BACKUP_TIMEOUT) {
while (!finishedBackup.get()) {
delay(100)
}
}
}
}
}
private fun performRestore() = run {
step("Start restore and await completion") {
RestoreScreen {
startRestore()
waitForInstallResult()
screenshot("restore app apks result")
nextButton.click()
waitForRestoreResult()
screenshot("restore app data result")
finishButton.click()
}
}
}
private fun waitForInstallResult() = runBlocking {
withTimeout(RESTORE_TIMEOUT) {
while (restoreViewModel.installResult.value == null) {
delay(100)
}
val restoreResultValue = restoreViewModel.installResult.value!!
assert(!restoreResultValue.hasFailed) { "Failed to install packages" }
}
}
private fun waitForRestoreResult() = runBlocking {
withTimeout(RESTORE_TIMEOUT) {
while (restoreViewModel.restoreBackupResult.value == null) {
delay(100)
}
val restoreResultValue = restoreViewModel.restoreBackupResult.value!!
assert(!restoreResultValue.hasError()) {
"Restore failed: ${restoreResultValue.errorMsg}"
}
}
}
}

View file

@ -0,0 +1,71 @@
package com.stevesoltys.seedvault.e2e
import com.stevesoltys.seedvault.e2e.screen.impl.BackupScreen
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import io.mockk.clearMocks
import io.mockk.every
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import java.util.concurrent.atomic.AtomicBoolean
internal interface LargeBackupTestBase : LargeTestBase {
companion object {
private const val BACKUP_TIMEOUT = 360 * 1000L
}
val spyBackupNotificationManager: BackupNotificationManager
fun launchBackupActivity() {
runCommand("am start -n ${targetContext.packageName}/.settings.SettingsActivity")
waitUntilIdle()
}
fun startBackup() {
BackupScreen {
backupMenu.clickAndWaitForNewWindow()
waitUntilIdle()
backupNowButton.clickAndWaitForNewWindow()
waitUntilIdle()
backupStatusButton.clickAndWaitForNewWindow()
waitUntilIdle()
}
}
fun performBackup(expectedPackages: Set<String>) {
val backupResult = spyOnBackup(expectedPackages)
startBackup()
waitForBackupResult(backupResult)
}
private fun spyOnBackup(expectedPackages: Set<String>): AtomicBoolean {
val finishedBackup = AtomicBoolean(false)
clearMocks(spyBackupNotificationManager)
every {
spyBackupNotificationManager.onBackupFinished(any(), any())
} answers {
val success = firstArg<Boolean>()
assert(success) { "Backup failed." }
this.callOriginal()
finishedBackup.set(true)
}
return finishedBackup
}
private fun waitForBackupResult(finishedBackup: AtomicBoolean) {
runBlocking {
withTimeout(BACKUP_TIMEOUT) {
while (!finishedBackup.get()) {
delay(100)
}
}
}
}
}

View file

@ -0,0 +1,86 @@
package com.stevesoltys.seedvault.e2e
import com.stevesoltys.seedvault.e2e.screen.impl.RecoveryCodeScreen
import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
import com.stevesoltys.seedvault.restore.RestoreViewModel
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
internal interface LargeRestoreTestBase : LargeTestBase {
companion object {
private const val RESTORE_TIMEOUT = 360 * 1000L
}
val spyRestoreViewModel: RestoreViewModel
fun launchRestoreActivity() {
runCommand("am start -n ${targetContext.packageName}/.restore.RestoreActivity")
waitUntilIdle()
}
fun typeInRestoreCode(code: List<String>) {
assert(code.size == 12) { "Code must have 12 words." }
RecoveryCodeScreen {
waitUntilIdle()
code.forEachIndexed { index, word ->
wordTextField(index).text = word
}
waitUntilIdle()
verifyCodeButton.scrollTo().click()
}
}
fun performRestore() {
RestoreScreen {
backupListItem.clickAndWaitForNewWindow()
waitUntilIdle()
waitForInstallResult()
nextButton.clickAndWaitForNewWindow()
waitForRestoreDataResult()
finishButton.clickAndWaitForNewWindow()
skipButton.clickAndWaitForNewWindow()
waitUntilIdle()
}
}
private fun waitForInstallResult() = runBlocking {
withTimeout(RESTORE_TIMEOUT) {
while (spyRestoreViewModel.installResult.value == null ||
spyRestoreViewModel.nextButtonEnabled.value == false
) {
delay(100)
}
}
val restoreResultValue = spyRestoreViewModel.installResult.value
?: error("Restore APKs timed out")
assert(!restoreResultValue.hasFailed) { "Failed to install packages" }
waitUntilIdle()
}
private fun waitForRestoreDataResult() = runBlocking {
withTimeout(RESTORE_TIMEOUT) {
while (spyRestoreViewModel.restoreBackupResult.value == null) {
delay(100)
}
}
val restoreResultValue = spyRestoreViewModel.restoreBackupResult.value
?: error("Restore app data timed out")
assert(!restoreResultValue.hasError()) {
"Restore failed: ${restoreResultValue.errorMsg}"
}
waitUntilIdle()
}
}

View file

@ -1,110 +1,100 @@
package com.stevesoltys.seedvault.e2e
import androidx.test.ext.junit.runners.AndroidJUnit4
import android.app.UiAutomation
import android.content.Context
import android.os.Environment
import androidx.annotation.WorkerThread
import androidx.test.platform.app.InstrumentationRegistry
import com.kaspersky.kaspresso.testcases.api.testcase.TestCase
import com.stevesoltys.seedvault.e2e.screen.impl.BackupScreen
import androidx.test.uiautomator.UiDevice
import com.stevesoltys.seedvault.e2e.screen.impl.DocumentPickerScreen
import com.stevesoltys.seedvault.e2e.screen.impl.RecoveryCodeScreen
import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
import org.junit.After
import org.junit.Before
import org.junit.runner.RunWith
import org.koin.core.component.KoinComponent
import java.lang.Thread.sleep
import java.text.SimpleDateFormat
import java.util.Calendar
@RunWith(AndroidJUnit4::class)
abstract class LargeTestBase : TestCase(), KoinComponent {
interface LargeTestBase {
@Before
open fun setUp() {
// reset document picker state, and delete old backups
runCommand("pm clear com.google.android.documentsui")
runCommand("rm -Rf /sdcard/seedvault")
companion object {
private const val TEST_STORAGE_FOLDER = "seedvault_test"
private const val TEST_VIDEO_FOLDER = "seedvault_test_videos"
}
@After
open fun tearDown() {
screenshot("end")
}
fun externalStorageDir(): String = Environment.getExternalStorageDirectory().absolutePath
protected fun launchBackupActivity() = run {
runCommand("am start -n ${device.targetContext.packageName}/.settings.SettingsActivity")
waitUntilIdle()
}
fun testStoragePath(): String = "${externalStorageDir()}/$TEST_STORAGE_FOLDER"
protected fun launchRestoreActivity() = run {
runCommand("am start -n ${device.targetContext.packageName}/.restore.RestoreActivity")
waitUntilIdle()
}
fun testVideoPath(): String = "${externalStorageDir()}/$TEST_VIDEO_FOLDER"
protected fun waitUntilIdle() {
device.uiDevice.waitForIdle()
val targetContext: Context
get() = InstrumentationRegistry.getInstrumentation().targetContext
val uiAutomation: UiAutomation
get() = InstrumentationRegistry.getInstrumentation().uiAutomation
val device: UiDevice
get() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
fun waitUntilIdle() {
device.waitForIdle()
sleep(3000)
}
protected fun verifyCode() = run {
RecoveryCodeScreen {
step("Confirm code") {
screenshot("confirm code")
confirmCodeButton.click()
}
step("Verify code") {
screenshot("verify code")
verifyCodeButton.scrollTo()
verifyCodeButton.click()
}
}
fun runCommand(command: String) {
uiAutomation.executeShellCommand(command).close()
}
protected fun chooseBackupLocation() = run {
step("Choose backup location") {
waitUntilIdle()
screenshot("choose backup location")
@WorkerThread
fun startScreenRecord(testName: String) {
val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss")
val timeStamp = simpleDateFormat.format(Calendar.getInstance().time)
val fileName = "${timeStamp}_${testName.replace(" ", "_")}"
DocumentPickerScreen {
createNewFolderButton.click()
textBox.text = "seedvault"
okButton.click()
useThisFolderButton.click()
allowButton.click()
}
}
val folder = testVideoPath()
runCommand("mkdir -p $folder")
runCommand("screenrecord $folder/$fileName.mp4")
}
protected fun startBackup() = run {
launchBackupActivity()
step("Run backup") {
BackupScreen {
backupMenu.clickAndWaitForNewWindow()
backupNowButton.clickAndWaitForNewWindow()
backupStatusButton.clickAndWaitForNewWindow()
}
}
@WorkerThread
fun stopScreenRecord() {
runCommand("pkill -2 screenrecord")
}
protected fun startRestore() = run {
launchRestoreActivity()
step("Restore backup") {
RestoreScreen {
backupListItem.clickAndWaitForNewWindow()
}
}
}
protected fun uninstallPackages(packages: Set<String>) {
fun uninstallPackages(packages: Set<String>) {
packages.forEach { runCommand("pm uninstall $it") }
}
protected fun runCommand(command: String) {
InstrumentationRegistry.getInstrumentation().uiAutomation
.executeShellCommand(command)
.close()
fun clearDocumentPickerAppData() {
runCommand("pm clear com.google.android.documentsui")
}
protected fun screenshot(name: String) {
device.screenshots.take(name.replace(" ", "_"))
fun clearTestBackups() {
runCommand("rm -Rf ${testStoragePath()}")
}
fun chooseStorageLocation(
folderName: String = TEST_STORAGE_FOLDER,
exists: Boolean = false,
) {
DocumentPickerScreen {
if (exists) {
existingFolder(folderName).scrollTo().clickAndWaitForNewWindow()
} else {
createNewFolderButton.clickAndWaitForNewWindow()
textBox.text = folderName
okButton.clickAndWaitForNewWindow()
}
useThisFolderButton.clickAndWaitForNewWindow()
allowButton.clickAndWaitForNewWindow()
}
}
fun confirmCode() {
RecoveryCodeScreen {
confirmCodeButton.click()
verifyCodeButton.scrollTo().click()
}
}
}

View file

@ -0,0 +1,73 @@
package com.stevesoltys.seedvault.e2e
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.stevesoltys.seedvault.restore.RestoreViewModel
import com.stevesoltys.seedvault.ui.notification.BackupNotificationManager
import org.junit.After
import org.junit.Before
import org.junit.Rule
import org.junit.rules.TestName
import org.junit.runner.RunWith
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.File
@RunWith(AndroidJUnit4::class)
internal abstract class SeedvaultLargeTest :
LargeBackupTestBase, LargeRestoreTestBase, KoinComponent {
@JvmField
@Rule
var name = TestName()
companion object {
private const val BASELINE_BACKUP_FOLDER = "seedvault_baseline"
private const val RECOVERY_CODE_FILE = "recovery-code.txt"
}
override val spyBackupNotificationManager: BackupNotificationManager by inject()
override val spyRestoreViewModel: RestoreViewModel by inject()
private val baselineBackupFolderPath = "${this.externalStorageDir()}/$BASELINE_BACKUP_FOLDER"
private val baselineRecoveryCodePath = "$baselineBackupFolderPath/$RECOVERY_CODE_FILE"
@Before
open fun setUp() {
clearDocumentPickerAppData()
clearTestBackups()
startScreenRecord(name.methodName)
restoreBaselineBackup()
}
@After
open fun tearDown() {
stopScreenRecord()
}
/**
* Restore the baseline backup, if it exists.
*
* This is a hand-crafted backup containing various apps and app data that we use for
* provisioning tests: https://github.com/seedvault-app/seedvault-test-data
*/
private fun restoreBaselineBackup() {
if (File(baselineBackupFolderPath).exists()) {
launchRestoreActivity()
chooseStorageLocation(folderName = BASELINE_BACKUP_FOLDER, exists = true)
typeInRestoreCode(baselineBackupRecoveryCode())
performRestore()
}
}
private fun baselineBackupRecoveryCode(): List<String> {
val recoveryCodeFile = File(baselineRecoveryCodePath)
return recoveryCodeFile.readLines()
.filter { it.isNotBlank() }
.joinToString(separator = " ") { it.trim() }
.split(" ")
}
}

View file

@ -0,0 +1,41 @@
package com.stevesoltys.seedvault.e2e.impl
import androidx.test.filters.LargeTest
import com.stevesoltys.seedvault.e2e.SeedvaultLargeTest
import com.stevesoltys.seedvault.settings.SettingsManager
import com.stevesoltys.seedvault.transport.backup.PackageService
import org.junit.Test
import org.koin.core.component.inject
@LargeTest
internal class BackupRestoreTest : SeedvaultLargeTest() {
private val packageService: PackageService by inject()
private val settingsManager: SettingsManager by inject()
@Test
fun `backup and restore applications`() {
launchBackupActivity()
if (settingsManager.getStorage() == null) {
confirmCode()
chooseStorageLocation()
}
val eligiblePackages = getEligibleApps()
performBackup(eligiblePackages)
uninstallPackages(eligiblePackages)
launchRestoreActivity()
performRestore()
// TODO: Get some real assertions in here..
// val packagesAfterRestore = getEligibleApps()
// assert(eligiblePackages == packagesAfterRestore)
}
private fun getEligibleApps() = packageService.userApps
.map { it.packageName }.toSet()
}

View file

@ -1,9 +1,12 @@
package com.stevesoltys.seedvault.e2e.screen
import android.widget.ScrollView
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.uiautomator.UiDevice
import androidx.test.uiautomator.UiObject
import androidx.test.uiautomator.UiScrollable
import androidx.test.uiautomator.UiSelector
import java.lang.Thread.sleep
abstract class UiDeviceScreen<T> {
@ -11,10 +14,20 @@ abstract class UiDeviceScreen<T> {
function.invoke(this as T)
}
protected fun findObject(
fun UiObject.scrollTo(
scrollSelector: UiSelector = UiSelector().className(ScrollView::class.java),
): UiObject {
UiScrollable(scrollSelector).scrollIntoView(this)
waitForExists(15000)
sleep(2000)
return this
}
fun findObject(
block: UiSelector.() -> UiSelector,
): UiObject = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
.findObject(
): UiObject = device().findObject(
UiSelector().let { it.block() }
)
private fun device() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
}

View file

@ -14,4 +14,6 @@ object DocumentPickerScreen : UiDeviceScreen<DocumentPickerScreen>() {
val okButton = findObject { text("OK") }
val allowButton = findObject { text("ALLOW") }
fun existingFolder(folderName: String) = findObject { text(folderName) }
}

View file

@ -1,15 +1,12 @@
package com.stevesoltys.seedvault.e2e.screen.impl
import com.kaspersky.kaspresso.screens.KScreen
import com.stevesoltys.seedvault.R
import io.github.kakaocup.kakao.text.KButton
import com.stevesoltys.seedvault.e2e.screen.UiDeviceScreen
object RecoveryCodeScreen : KScreen<RecoveryCodeScreen>() {
object RecoveryCodeScreen : UiDeviceScreen<RecoveryCodeScreen>() {
override val layoutId: Int? = null
override val viewClass: Class<*>? = null
val confirmCodeButton = findObject { text("Confirm code") }
val confirmCodeButton = KButton { withId(R.id.confirmCodeButton) }
val verifyCodeButton = findObject { text("Verify") }
val verifyCodeButton = KButton { withId(R.id.doneButton) }
fun wordTextField(index: Int) = findObject { text("Word ${index + 1}") }
}

View file

@ -4,9 +4,11 @@ import com.stevesoltys.seedvault.e2e.screen.UiDeviceScreen
object RestoreScreen : UiDeviceScreen<RestoreScreen>() {
val backupListItem = findObject { textContains("Last backup 0 hr") }
val backupListItem = findObject { textContains("Last backup") }
val nextButton = findObject { text("Next") }
val finishButton = findObject { text("Finish") }
val skipButton = findObject { text("Skip restoring files") }
}