From ea9bad7cff588c00297bd1a4ba788b45093b2a47 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:28:41 +0200 Subject: [PATCH] ci: run maestro flows on the ios simulator --- .github/workflows/e2e-ios.yml | 167 ++++++++++++++++++++++++++++++++++ .maestro/launch.yaml | 21 +++++ .maestro/signed-in.yaml | 99 ++++++++++++++++++++ 3 files changed, 287 insertions(+) create mode 100644 .github/workflows/e2e-ios.yml create mode 100644 .maestro/launch.yaml create mode 100644 .maestro/signed-in.yaml diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml new file mode 100644 index 00000000..e47923e5 --- /dev/null +++ b/.github/workflows/e2e-ios.yml @@ -0,0 +1,167 @@ +name: iOS simulator tests + +# Builds the app for the iOS Simulator (no signing needed), runs the Maestro +# flows in .maestro/ against it and uploads screenshots and a screen recording. +# +# The signed-in flow needs the E2E_SERVER and E2E_USERNAME variables and the +# E2E_PASSWORD secret (a test account on a server the runner can reach). +# Without them only the launch flow runs. + +on: + workflow_dispatch: + release: + types: [published] + +concurrency: + group: e2e-ios-${{ github.ref }} + cancel-in-progress: true + +jobs: + simulator: + runs-on: macos-latest + timeout-minutes: 75 + env: + EXPO_NO_TELEMETRY: "1" + OUT: ${{ github.workspace }}/e2e-output + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Cache CocoaPods + uses: actions/cache@v4 + with: + path: | + ~/Library/Caches/CocoaPods + ~/.cocoapods/repos + key: pods-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + restore-keys: pods-${{ runner.os }}- + + - name: Install JS dependencies + run: npm ci + + - name: Generate the iOS project + run: npx expo prebuild --platform ios --clean + + - name: Build for the simulator + run: | + WORKSPACE=$(ls -d ios/*.xcworkspace | head -1) + SCHEME=$(basename "$WORKSPACE" .xcworkspace) + # Release bundles the JS into the app, so no Metro server is needed. + # arm64 only: the runner is Apple silicon, and a second simulator + # architecture would double the build time. + mkdir -p "$OUT" + if ! xcodebuild build \ + -workspace "$WORKSPACE" \ + -scheme "$SCHEME" \ + -configuration Release \ + -sdk iphonesimulator \ + -destination 'generic/platform=iOS Simulator' \ + -derivedDataPath build \ + ARCHS=arm64 \ + CODE_SIGNING_ALLOWED=NO \ + > "$OUT/xcodebuild.log" 2>&1; then + grep -E "error:|fatal error" "$OUT/xcodebuild.log" | head -n 50 || true + tail -n 80 "$OUT/xcodebuild.log" + exit 1 + fi + APP=$(ls -d build/Build/Products/Release-iphonesimulator/*.app | head -1) + echo "APP=$APP" >> "$GITHUB_ENV" + echo "Built $APP" + + - name: Upload the simulator app + uses: actions/upload-artifact@v4 + with: + name: bulwark-mobile-simulator-${{ github.sha }} + path: build/Build/Products/Release-iphonesimulator/*.app + if-no-files-found: error + + - name: Boot a simulator + run: | + # The newest iPhone on the newest runtime the image ships. + UDID=$(xcrun simctl list devices available -j | jq -r ' + [.devices | to_entries[] + | select(.key | test("SimRuntime.iOS")) + | .value[] | select(.name | startswith("iPhone")) | .udid] | last') + if [ -z "$UDID" ] || [ "$UDID" = "null" ]; then + xcrun simctl list devices available + echo "::error::No iPhone simulator available" + exit 1 + fi + xcrun simctl boot "$UDID" + xcrun simctl bootstatus "$UDID" -b + xcrun simctl install "$UDID" "$APP" + echo "UDID=$UDID" >> "$GITHUB_ENV" + xcrun simctl list devices | grep "$UDID" + + - name: Install Maestro + run: | + curl -fsSL "https://get.maestro.mobile.dev" | bash + echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" + + - name: Run the flows + env: + SERVER: ${{ vars.E2E_SERVER }} + USERNAME: ${{ vars.E2E_USERNAME }} + PASSWORD: ${{ secrets.E2E_PASSWORD }} + # The XCTest driver takes a while to come up on a cold runner. + MAESTRO_DRIVER_STARTUP_TIMEOUT: "240000" + MAESTRO_CLI_NO_ANALYTICS: "1" + run: | + maestro --version + mkdir -p "$OUT/screenshots" + + xcrun simctl io "$UDID" recordVideo --codec=h264 --force "$OUT/run.mp4" & + RECORDER=$! + + ARGS=() + if [ -n "$SERVER" ] && [ -n "$USERNAME" ] && [ -n "$PASSWORD" ]; then + ARGS+=(-e "SERVER=$SERVER" -e "USERNAME=$USERNAME" -e "PASSWORD=$PASSWORD") + else + echo "::notice::E2E_SERVER, E2E_USERNAME or E2E_PASSWORD is not set - running the launch flow only." + ARGS+=(--exclude-tags signed-in) + fi + + # Screenshots land in the working directory, so run from the output + # folder. Debug output is left out on purpose: it logs typed text, + # including the password, and artifacts here are public. + set +e + (cd "$OUT/screenshots" && maestro --device "$UDID" test \ + --format junit --output "$OUT/report.xml" \ + "${ARGS[@]}" "$GITHUB_WORKSPACE/.maestro") + STATUS=$? + set -e + + kill -INT "$RECORDER" || true + wait "$RECORDER" || true + + # Newer Maestro versions write screenshots (and a screenshot of any + # failed step) under ~/.maestro/tests instead. Copy only the images: + # the JSON next to them holds the typed password. + find "$HOME/.maestro/tests" -name '*.png' -exec cp {} "$OUT/screenshots/" \; 2>/dev/null || true + ls -la "$OUT/screenshots" + exit $STATUS + + - name: Grab the app log on failure + if: failure() && env.UDID != '' + run: | + xcrun simctl spawn "$UDID" log show --last 10m --style compact \ + --predicate 'process CONTAINS "Bulwark"' > "$OUT/app.log" 2>&1 || true + tail -n 100 "$OUT/app.log" || true + + - name: Upload screenshots and recording + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-ios-${{ github.run_number }} + path: e2e-output/ + if-no-files-found: ignore diff --git a/.maestro/launch.yaml b/.maestro/launch.yaml new file mode 100644 index 00000000..857dfd3a --- /dev/null +++ b/.maestro/launch.yaml @@ -0,0 +1,21 @@ +# Fresh install: the app starts and shows the sign-in choices. +# Needs no server, so it also runs when the test account secrets are missing. +appId: org.bulwarkmail.mobile +name: Launch to sign-in +tags: + - smoke +--- +- launchApp: + clearState: true + clearKeychain: true +# A cold start of a release build on a CI simulator can be slow. +- extendedWaitUntil: + visible: "Enter server details manually" + timeout: 120000 +- takeScreenshot: 01-sign-in-choices +- tapOn: "Enter server details manually" +- extendedWaitUntil: + visible: + id: login-server-input + timeout: 10000 +- takeScreenshot: 02-server-step diff --git a/.maestro/signed-in.yaml b/.maestro/signed-in.yaml new file mode 100644 index 00000000..2997457d --- /dev/null +++ b/.maestro/signed-in.yaml @@ -0,0 +1,99 @@ +# Signs in with a password and walks the main screens. +# Run with: maestro test -e SERVER=... -e USERNAME=... -e PASSWORD=... .maestro/signed-in.yaml +appId: org.bulwarkmail.mobile +name: Sign in and browse +tags: + - signed-in +--- +- launchApp: + clearState: true + clearKeychain: true + # The calendar tab asks for notification permission for reminders; + # granting it up front keeps the system alert from covering the screen. + permissions: + notifications: allow +- extendedWaitUntil: + visible: "Enter server details manually" + timeout: 120000 +- tapOn: "Enter server details manually" + +- tapOn: + id: login-server-input +- inputText: ${SERVER} +- pressKey: Enter + +# Servers that offer OAuth land on a confirm step first; switch to the +# password form there. Others go straight to it. +- extendedWaitUntil: + visible: "Sign in with a password instead|Email or username" + timeout: 30000 +- runFlow: + when: + visible: "Sign in with a password instead" + commands: + - tapOn: "Sign in with a password instead" + +- tapOn: + id: login-username-input +- eraseText +- inputText: ${USERNAME} +- tapOn: + id: login-password-input +- inputText: ${PASSWORD} +- pressKey: Enter + +- extendedWaitUntil: + visible: + id: tab-mail + timeout: 60000 +- runFlow: + when: + visible: "Not now" + commands: + - tapOn: "Not now" + +- extendedWaitUntil: + visible: + id: email-row + timeout: 60000 +- takeScreenshot: 10-inbox + +- tapOn: + id: email-row + index: 0 +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: 11-message +- tapOn: "Back" +- extendedWaitUntil: + visible: + id: email-row + timeout: 10000 + +- tapOn: + id: tab-calendar +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: 20-calendar + +- tapOn: + id: tab-contacts +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: 30-contacts + +- tapOn: + id: tab-files +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: 40-files + +- tapOn: + id: tab-settings +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: 50-settings + +# Still alive after the walk: the tab bar is there and the app didn't crash. +- assertVisible: + id: tab-mail