From 273051e5f59369603cdc750801ca6e80b4f2d346 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:54:03 +0200 Subject: [PATCH] ci: run sign-in flows for every method on iphone and ipad simulators --- .github/actions/e2e-ios-leg/action.yml | 92 +++++ .github/workflows/e2e-ios.yml | 295 +++++++------- .maestro/launch.yaml | 20 +- .maestro/scripts/helper.js | 8 + .maestro/signed-in.yaml | 100 ----- .maestro/signin/01-password.yaml | 32 ++ .maestro/signin/02-password-errors.yaml | 48 +++ .maestro/signin/03-totp-password.yaml | 24 ++ .maestro/signin/04-oauth.yaml | 20 + .maestro/signin/05-oauth-cancel.yaml | 39 ++ .maestro/signin/06-webmail.yaml | 20 + .maestro/signin/07-webmail-totp.yaml | 22 + .maestro/signin/08-link-password.yaml | 17 + .maestro/signin/09-paste-password.yaml | 16 + .maestro/signin/10-link-totp.yaml | 15 + .maestro/signin/11-paste-totp.yaml | 15 + .maestro/signin/12-link-expired.yaml | 21 + .maestro/signin/13-link-used.yaml | 44 ++ .maestro/signin/14-scanner-camera.yaml | 32 ++ .maestro/signin/15-email-discovery.yaml | 23 ++ .maestro/signin/16-add-password.yaml | 30 ++ .maestro/signin/17-add-oauth.yaml | 42 ++ .maestro/signin/18-add-webmail.yaml | 43 ++ .maestro/signin/19-add-link.yaml | 30 ++ .maestro/subflows/allow-auth-session.yaml | 11 + .maestro/subflows/check-active-account.yaml | 16 + .maestro/subflows/confirm-link.yaml | 10 + .../subflows/finish-two-factor-if-asked.yaml | 12 + .maestro/subflows/inbox.yaml | 18 + .maestro/subflows/manual-server.yaml | 18 + .maestro/subflows/open-add-account.yaml | 17 + .maestro/subflows/password-form.yaml | 19 + .maestro/subflows/password-sign-in.yaml | 11 + .maestro/subflows/paste-link.yaml | 22 + .maestro/subflows/sign-out.yaml | 16 + .maestro/subflows/stalwart-login-page.yaml | 25 ++ .maestro/subflows/start.yaml | 20 + .maestro/subflows/two-factor.yaml | 21 + .maestro/subflows/type-secret.yaml | 17 + .maestro/subflows/type.yaml | 25 ++ .maestro/subflows/webmail-login-page.yaml | 31 ++ scripts/e2e/boot-sim.sh | 51 +++ scripts/e2e/failed-steps.mjs | 71 ++++ scripts/e2e/helper.mjs | 384 ++++++++++++++++++ scripts/e2e/run-flows.sh | 95 +++++ 45 files changed, 1696 insertions(+), 262 deletions(-) create mode 100644 .github/actions/e2e-ios-leg/action.yml create mode 100644 .maestro/scripts/helper.js delete mode 100644 .maestro/signed-in.yaml create mode 100644 .maestro/signin/01-password.yaml create mode 100644 .maestro/signin/02-password-errors.yaml create mode 100644 .maestro/signin/03-totp-password.yaml create mode 100644 .maestro/signin/04-oauth.yaml create mode 100644 .maestro/signin/05-oauth-cancel.yaml create mode 100644 .maestro/signin/06-webmail.yaml create mode 100644 .maestro/signin/07-webmail-totp.yaml create mode 100644 .maestro/signin/08-link-password.yaml create mode 100644 .maestro/signin/09-paste-password.yaml create mode 100644 .maestro/signin/10-link-totp.yaml create mode 100644 .maestro/signin/11-paste-totp.yaml create mode 100644 .maestro/signin/12-link-expired.yaml create mode 100644 .maestro/signin/13-link-used.yaml create mode 100644 .maestro/signin/14-scanner-camera.yaml create mode 100644 .maestro/signin/15-email-discovery.yaml create mode 100644 .maestro/signin/16-add-password.yaml create mode 100644 .maestro/signin/17-add-oauth.yaml create mode 100644 .maestro/signin/18-add-webmail.yaml create mode 100644 .maestro/signin/19-add-link.yaml create mode 100644 .maestro/subflows/allow-auth-session.yaml create mode 100644 .maestro/subflows/check-active-account.yaml create mode 100644 .maestro/subflows/confirm-link.yaml create mode 100644 .maestro/subflows/finish-two-factor-if-asked.yaml create mode 100644 .maestro/subflows/inbox.yaml create mode 100644 .maestro/subflows/manual-server.yaml create mode 100644 .maestro/subflows/open-add-account.yaml create mode 100644 .maestro/subflows/password-form.yaml create mode 100644 .maestro/subflows/password-sign-in.yaml create mode 100644 .maestro/subflows/paste-link.yaml create mode 100644 .maestro/subflows/sign-out.yaml create mode 100644 .maestro/subflows/stalwart-login-page.yaml create mode 100644 .maestro/subflows/start.yaml create mode 100644 .maestro/subflows/two-factor.yaml create mode 100644 .maestro/subflows/type-secret.yaml create mode 100644 .maestro/subflows/type.yaml create mode 100644 .maestro/subflows/webmail-login-page.yaml create mode 100644 scripts/e2e/boot-sim.sh create mode 100644 scripts/e2e/failed-steps.mjs create mode 100644 scripts/e2e/helper.mjs create mode 100644 scripts/e2e/run-flows.sh diff --git a/.github/actions/e2e-ios-leg/action.yml b/.github/actions/e2e-ios-leg/action.yml new file mode 100644 index 00000000..6be8ec0e --- /dev/null +++ b/.github/actions/e2e-ios-leg/action.yml @@ -0,0 +1,92 @@ +name: iOS simulator test leg +description: Boot a simulator, install the app and run one suite of Maestro flows + +inputs: + suite: + description: Folder under .maestro (signin or features) + required: true + name: + description: Leg name, used for the artifact + required: true + app_run_id: + description: Take the simulator app from this run instead of the current one + required: false + default: "" + flows: + description: Comma list of flow names to run instead of the whole suite + required: false + default: "" + playwright: + description: Install Playwright for the helper's webmail pairing + required: false + default: "false" + +runs: + using: composite + steps: + - name: Download the simulator app (this run) + if: inputs.app_run_id == '' + uses: actions/download-artifact@v4 + with: + name: simulator-app + path: ${{ runner.temp }} + + - name: Download the simulator app (earlier run) + if: inputs.app_run_id != '' + uses: actions/download-artifact@v4 + with: + name: simulator-app + run-id: ${{ inputs.app_run_id }} + github-token: ${{ github.token }} + path: ${{ runner.temp }} + + - name: Unpack the simulator app + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/app" + tar -C "$RUNNER_TEMP/app" -xzf "$RUNNER_TEMP/simulator-app.tar.gz" + echo "APP=$RUNNER_TEMP/app/BulwarkMobile.app" >> "$GITHUB_ENV" + + - name: Boot a simulator + shell: bash + run: bash scripts/e2e/boot-sim.sh "$DEVICE" "$APPEARANCE" "$APP" + + - name: Install Maestro + shell: bash + run: | + curl -fsSL "https://get.maestro.mobile.dev" | bash + echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" + + - name: Install Playwright for the helper + if: inputs.playwright == 'true' + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/pw" + cd "$RUNNER_TEMP/pw" + npm init -y >/dev/null + npm install --no-audit --no-fund playwright@1.59.1 >/dev/null + npx playwright install chromium >/dev/null + echo "PLAYWRIGHT_DIR=$RUNNER_TEMP/pw/node_modules" >> "$GITHUB_ENV" + + - name: Start the test helper + shell: bash + run: | + mkdir -p "$OUT/helper" + HELPER_SHOTS="$OUT/helper" nohup node scripts/e2e/helper.mjs > "$OUT/helper/helper.log" 2>&1 & + for i in $(seq 1 20); do + curl -fs http://127.0.0.1:8787/health >/dev/null && break + sleep 0.5 + done + curl -fs http://127.0.0.1:8787/health + + - name: Run the flows + shell: bash + run: bash scripts/e2e/run-flows.sh "${{ inputs.suite }}" "${{ inputs.flows }}" + + - name: Upload screenshots, recording and logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-ios-${{ inputs.name }} + path: e2e-output/ + if-no-files-found: ignore diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml index a1089ca4..0f7c57c9 100644 --- a/.github/workflows/e2e-ios.yml +++ b/.github/workflows/e2e-ios.yml @@ -1,11 +1,20 @@ name: iOS simulator tests -# Builds the app for the iOS Simulator (no signing needed), runs the Maestro -# flows in .maestro/ against it and uploads screenshots and a screen recording. +# Builds the app for the iOS Simulator (no signing needed) and runs the +# Maestro flows in .maestro/ on iPhone and iPad simulators: # -# The signed-in flow needs the E2E_SERVER and E2E_USERNAME variables and the -# E2E_PASSWORD secret (a test account on a server the runner can reach). -# Without them only the launch flow runs. +# - signin: every sign-in method (.maestro/signin), on an iPhone and an iPad +# - features: the app feature by feature (.maestro/features), on iPhone and +# iPad in light and dark, plus the iPad in landscape (split layout) +# +# Each leg signs in with its own account on the test server (e2e-* accounts +# on VM 106, password E2E_PASSWORD), so parallel legs never share a mailbox. +# scripts/e2e/helper.mjs runs next to the simulator for what a flow can't do +# itself: TOTP codes, fresh webmail pairing links, mail sent by userb. +# +# Artifacts per leg: screenshots, a screen recording, the app log and a JUnit +# report. They are public; see scripts/e2e/run-flows.sh for how secrets are +# kept out of them. on: workflow_dispatch: @@ -14,6 +23,14 @@ on: description: "Reuse the simulator app from this earlier run instead of building (run id)" type: string required: false + legs: + description: "Legs to run: all, signin, features, or a comma list (signin-phone, features-tablet-dark, ...)" + type: string + default: all + flows: + description: "Only these flows (comma list of file names without .yaml); empty runs all" + type: string + required: false release: types: [published] @@ -21,33 +38,55 @@ concurrency: group: e2e-ios-${{ github.ref }} cancel-in-progress: true -jobs: - simulator: - runs-on: macos-latest - timeout-minutes: 75 - permissions: - contents: read - actions: read - env: - EXPO_NO_TELEMETRY: "1" - OUT: ${{ github.workspace }}/e2e-output +env: + EXPO_NO_TELEMETRY: "1" + MAESTRO_CLI_NO_ANALYTICS: "1" + # The XCTest driver takes minutes to come up on a cold runner. + MAESTRO_DRIVER_STARTUP_TIMEOUT: "600000" +jobs: + plan: + runs-on: ubuntu-latest + outputs: + signin: ${{ steps.plan.outputs.signin }} + features: ${{ steps.plan.outputs.features }} + steps: + - id: plan + env: + LEGS: ${{ inputs.legs || 'all' }} + run: | + node -e ' + const legs = (process.env.LEGS || "all").split(",").map((s) => s.trim()).filter(Boolean); + const want = (name, group) => legs.includes("all") || legs.includes(group) || legs.includes(name); + const signin = ["phone", "tablet"] + .map((device) => ({ name: `signin-${device}`, device })) + .filter((l) => want(l.name, "signin")); + const features = [ + { name: "features-phone-light", device: "phone", appearance: "light", orientation: "portrait", account: "e2e-phone-light" }, + { name: "features-phone-dark", device: "phone", appearance: "dark", orientation: "portrait", account: "e2e-phone-dark" }, + { name: "features-tablet-light", device: "tablet", appearance: "light", orientation: "portrait", account: "e2e-tablet-light" }, + { name: "features-tablet-dark", device: "tablet", appearance: "dark", orientation: "portrait", account: "e2e-tablet-dark" }, + { name: "features-tablet-landscape", device: "tablet", appearance: "light", orientation: "landscape", account: "e2e-tablet-land" }, + ].filter((l) => want(l.name, "features")); + const fs = require("fs"); + fs.appendFileSync(process.env.GITHUB_OUTPUT, + `signin=${JSON.stringify({ include: signin })}\nfeatures=${JSON.stringify({ include: features })}\n`); + console.log({ signin: signin.map((l) => l.name), features: features.map((l) => l.name) }); + ' + + build: + if: inputs.app_run_id == '' + runs-on: macos-latest + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 - if: inputs.app_run_id == '' with: node-version: 20 cache: npm - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 17 - - name: Cache CocoaPods - if: inputs.app_run_id == '' uses: actions/cache@v4 with: path: | @@ -57,22 +96,19 @@ jobs: restore-keys: pods-${{ runner.os }}- - name: Install JS dependencies - if: inputs.app_run_id == '' run: npm ci - name: Generate the iOS project - if: inputs.app_run_id == '' run: npx expo prebuild --platform ios --clean - name: Build for the simulator - if: inputs.app_run_id == '' run: | WORKSPACE=$(ls -d ios/*.xcworkspace | head -1) SCHEME=$(basename "$WORKSPACE" .xcworkspace) + mkdir -p e2e-build # Release bundles the JS into the app, so no Metro server is needed. # arm64 only: the runner is Apple silicon, and a second simulator # architecture would double the build time. - mkdir -p "$OUT" if ! xcodebuild build \ -workspace "$WORKSPACE" \ -scheme "$SCHEME" \ @@ -82,145 +118,98 @@ jobs: -derivedDataPath build \ ARCHS=arm64 \ CODE_SIGNING_ALLOWED=NO \ - > "$OUT/xcodebuild.log" 2>&1; then - grep -E "error:|fatal error" "$OUT/xcodebuild.log" | head -n 50 || true - tail -n 80 "$OUT/xcodebuild.log" + > e2e-build/xcodebuild.log 2>&1; then + grep -E "error:|fatal error" e2e-build/xcodebuild.log | head -n 50 || true + tail -n 80 e2e-build/xcodebuild.log exit 1 fi # Packed as a tar: artifacts drop the executable bit, which would # leave a reused app unable to launch. tar -C build/Build/Products/Release-iphonesimulator -czf "$RUNNER_TEMP/simulator-app.tar.gz" BulwarkMobile.app + ls -la "$RUNNER_TEMP/simulator-app.tar.gz" - name: Upload the simulator app - if: inputs.app_run_id == '' uses: actions/upload-artifact@v4 with: name: simulator-app path: ${{ runner.temp }}/simulator-app.tar.gz if-no-files-found: error - - name: Download the simulator app from an earlier run - if: inputs.app_run_id != '' - uses: actions/download-artifact@v4 - with: - name: simulator-app - run-id: ${{ inputs.app_run_id }} - github-token: ${{ github.token }} - path: ${{ runner.temp }} - - - name: Unpack the simulator app - run: | - mkdir -p "$RUNNER_TEMP/app" - tar -C "$RUNNER_TEMP/app" -xzf "$RUNNER_TEMP/simulator-app.tar.gz" - echo "APP=$RUNNER_TEMP/app/BulwarkMobile.app" >> "$GITHUB_ENV" - - - name: Boot a simulator - run: | - # The newest iPhone on the newest runtime the image ships. - UDID=$(xcrun simctl list devices available -j | jq -r ' - [.devices | to_entries[] - | select(.key | test("SimRuntime.iOS")) - | .value[] | select(.name | startswith("iPhone")) | .udid] | last') - if [ -z "$UDID" ] || [ "$UDID" = "null" ]; then - xcrun simctl list devices available - echo "::error::No iPhone simulator available" - exit 1 - fi - xcrun simctl boot "$UDID" - xcrun simctl bootstatus "$UDID" -b - # A fresh simulator shows the slide-to-type tip over the keyboard the - # first time a text field gets focus, and typed text goes nowhere. - xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences DidShowContinuousPathIntroduction -bool true - xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences KeyboardContinuousPathEnabled -bool false - # Predictions, autocorrect and password AutoFill can swallow or - # rewrite what XCTest types. - xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences KeyboardPrediction -bool false - xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences KeyboardAutocorrection -bool false - xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences KeyboardAutocapitalization -bool false - xcrun simctl spawn "$UDID" defaults write com.apple.WebUI AutoFillPasswords -bool false || true - xcrun simctl install "$UDID" "$APP" - echo "UDID=$UDID" >> "$GITHUB_ENV" - xcrun simctl list devices | grep "$UDID" - - - name: Install Maestro - run: | - curl -fsSL "https://get.maestro.mobile.dev" | bash - echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - - - name: Run the flows - env: - SERVER: ${{ vars.E2E_SERVER }} - USERNAME: ${{ vars.E2E_USERNAME }} - PASSWORD: ${{ secrets.E2E_PASSWORD }} - # The XCTest driver takes a while to come up on a cold runner. - MAESTRO_DRIVER_STARTUP_TIMEOUT: "600000" - MAESTRO_CLI_NO_ANALYTICS: "1" - run: | - maestro --version - mkdir -p "$OUT/screenshots" - - xcrun simctl io "$UDID" recordVideo --codec=h264 --force "$OUT/run.mp4" & - RECORDER=$! - - ARGS=() - if [ -n "$SERVER" ] && [ -n "$USERNAME" ] && [ -n "$PASSWORD" ]; then - ARGS+=(-e "SERVER=$SERVER" -e "USERNAME=$USERNAME" -e "PASSWORD=$PASSWORD") - else - echo "::notice::E2E_SERVER, E2E_USERNAME or E2E_PASSWORD is not set - running the launch flow only." - ARGS+=(--exclude-tags signed-in) - fi - - # Warm-up with the launch flow alone. It types nothing, so its debug - # log holds no secrets and can be printed when the driver won't start. - # The real run below keeps debug output off: it would log the password, - # and artifacts here are public. - set +e - touch "$RUNNER_TEMP/flows-started" - (cd "$OUT/screenshots" && maestro --device "$UDID" test \ - --debug-output "$RUNNER_TEMP/maestro-warmup" "$GITHUB_WORKSPACE/.maestro/launch.yaml") - STATUS=$? - echo "::group::Maestro warm-up log (typing)" - find "$RUNNER_TEMP/maestro-warmup" -name 'maestro.log' -exec grep -n -i -E "inputText|typeText|keyboard|input_text|Exception|error" {} \; | tail -n 120 - echo "::endgroup::" - maestro --device "$UDID" hierarchy > "$OUT/hierarchy-warmup.json" 2>/dev/null || true - if [ $STATUS -ne 0 ]; then - echo "::group::Maestro warm-up log" - find "$RUNNER_TEMP/maestro-warmup" -name 'maestro.log' -exec tail -n 300 {} \; - echo "::endgroup::" - echo "::group::xcodebuild logs of the driver" - find "$HOME/Library/Logs" "$HOME/.maestro" "$TMPDIR" -name '*.log' -newer "$RUNNER_TEMP/flows-started" 2>/dev/null \ - | grep -i -E 'xctest|xcodebuild|maestro' | grep -v maestro-warmup | head -5 \ - | while read -r f; do echo "== $f"; tail -n 80 "$f"; done - echo "::endgroup::" - else - (cd "$OUT/screenshots" && maestro --device "$UDID" test \ - --format junit --output "$OUT/report.xml" \ - "${ARGS[@]}" "$GITHUB_WORKSPACE/.maestro") - STATUS=$? - fi - set -e - - kill -INT "$RECORDER" || true - wait "$RECORDER" || true - - # Newer Maestro versions write screenshots (and a screenshot of any - # failed step) under ~/.maestro/tests instead. Copy only the images: - # the JSON next to them holds the typed password. - find "$HOME/.maestro/tests" -name '*.png' -exec cp {} "$OUT/screenshots/" \; 2>/dev/null || true - ls -la "$OUT/screenshots" - exit $STATUS - - - name: Grab the app log on failure - if: failure() && env.UDID != '' - run: | - xcrun simctl spawn "$UDID" log show --last 10m --style compact \ - --predicate 'process CONTAINS "Bulwark"' > "$OUT/app.log" 2>&1 || true - tail -n 100 "$OUT/app.log" || true - - - name: Upload screenshots and recording - if: always() + - name: Upload the build log + if: failure() uses: actions/upload-artifact@v4 with: - name: e2e-ios-${{ github.run_number }} - path: e2e-output/ - if-no-files-found: ignore + name: xcodebuild-log + path: e2e-build/ + + signin: + needs: [plan, build] + if: >- + always() && !cancelled() && + (needs.build.result == 'success' || needs.build.result == 'skipped') && + fromJson(needs.plan.outputs.signin).include[0] != null + runs-on: macos-latest + timeout-minutes: 90 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.signin) }} + env: + OUT: ${{ github.workspace }}/e2e-output + DEVICE: ${{ matrix.device }} + APPEARANCE: light + ORIENTATION: portrait + E2E_SERVER: ${{ vars.E2E_SERVER }} + E2E_WEBMAIL: https://bulwark.internal.bulwarkmail.org + E2E_ACCOUNT: e2e-login-${{ matrix.device }} + E2E_ACCOUNT2: e2e-second-${{ matrix.device }} + E2E_TOTP_ACCOUNT: e2e-totp-${{ matrix.device }} + E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} + E2E_USERB_PASSWORD: ${{ secrets.E2E_USERB_PASSWORD }} + E2E_TOTP_SECRET: ${{ secrets.E2E_TOTP_SECRET }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/e2e-ios-leg + with: + suite: signin + app_run_id: ${{ inputs.app_run_id }} + flows: ${{ inputs.flows }} + playwright: "true" + name: ${{ matrix.name }} + + features: + needs: [plan, build] + if: >- + always() && !cancelled() && + (needs.build.result == 'success' || needs.build.result == 'skipped') && + fromJson(needs.plan.outputs.features).include[0] != null + runs-on: macos-latest + timeout-minutes: 120 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.features) }} + env: + OUT: ${{ github.workspace }}/e2e-output + DEVICE: ${{ matrix.device }} + APPEARANCE: ${{ matrix.appearance }} + ORIENTATION: ${{ matrix.orientation }} + E2E_SERVER: ${{ vars.E2E_SERVER }} + E2E_WEBMAIL: https://bulwark.internal.bulwarkmail.org + E2E_ACCOUNT: ${{ matrix.account }} + E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} + E2E_USERB_PASSWORD: ${{ secrets.E2E_USERB_PASSWORD }} + E2E_TOTP_SECRET: ${{ secrets.E2E_TOTP_SECRET }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/e2e-ios-leg + with: + suite: features + app_run_id: ${{ inputs.app_run_id }} + flows: ${{ inputs.flows }} + playwright: "false" + name: ${{ matrix.name }} diff --git a/.maestro/launch.yaml b/.maestro/launch.yaml index bdef3056..4dfcf872 100644 --- a/.maestro/launch.yaml +++ b/.maestro/launch.yaml @@ -1,6 +1,7 @@ -# Fresh install: the app starts and shows the sign-in choices. -# Needs no server, so it also runs when the test account secrets are missing. -# It types only a made-up address, so its debug log is safe to print. +# Fresh install: the app starts and shows the sign-in choices, and typing +# into a field works. Needs no server, so it runs before every suite as the +# warm-up for the XCTest driver. It types only a made-up address, so its +# debug log is safe to print. appId: org.bulwarkmail.mobile name: Launch to sign-in tags: @@ -13,15 +14,16 @@ tags: - extendedWaitUntil: visible: "Enter server details manually" timeout: 120000 -- takeScreenshot: 01-sign-in-choices +- takeScreenshot: warmup-01-sign-in-choices - tapOn: "Enter server details manually" - extendedWaitUntil: visible: id: login-server-input timeout: 10000 -- tapOn: - id: login-server-input - runFlow: subflows/dismiss-keyboard-tip.yaml -- inputText: "mail.example.com" -- takeScreenshot: 02-server-step -- assertVisible: "mail.example.com" +- runFlow: + file: subflows/type.yaml + env: + FIELD: login-server-input + TEXT: typing-probe.example.net +- takeScreenshot: warmup-02-typed diff --git a/.maestro/scripts/helper.js b/.maestro/scripts/helper.js new file mode 100644 index 00000000..b170a6c4 --- /dev/null +++ b/.maestro/scripts/helper.js @@ -0,0 +1,8 @@ +// Calls the test helper (scripts/e2e/helper.mjs) on the runner. +// env CALL: path and query, e.g. "/totp" or "/pair?user=x&deliver=openurl". +// The answer lands in output.helper. +const response = http.get(HELPER + CALL); +if (!response.ok) { + throw new Error('helper ' + CALL.split('?')[0] + ' failed: ' + response.status + ' ' + response.body); +} +output.helper = json(response.body); diff --git a/.maestro/signed-in.yaml b/.maestro/signed-in.yaml deleted file mode 100644 index 1f8ff7b1..00000000 --- a/.maestro/signed-in.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# Signs in with a password and walks the main screens. -# Run with: maestro test -e SERVER=... -e USERNAME=... -e PASSWORD=... .maestro/signed-in.yaml -appId: org.bulwarkmail.mobile -name: Sign in and browse -tags: - - signed-in ---- -- launchApp: - clearState: true - clearKeychain: true - # The calendar tab asks for notification permission for reminders; - # granting it up front keeps the system alert from covering the screen. - permissions: - notifications: allow -- extendedWaitUntil: - visible: "Enter server details manually" - timeout: 120000 -- tapOn: "Enter server details manually" - -- tapOn: - id: login-server-input -- runFlow: subflows/dismiss-keyboard-tip.yaml -- inputText: ${SERVER} -- pressKey: Enter - -# Servers that offer OAuth land on a confirm step first; switch to the -# password form there. Others go straight to it. -- extendedWaitUntil: - visible: "Sign in with a password instead|Email or username" - timeout: 30000 -- runFlow: - when: - visible: "Sign in with a password instead" - commands: - - tapOn: "Sign in with a password instead" - -- tapOn: - id: login-username-input -- eraseText -- inputText: ${USERNAME} -- tapOn: - id: login-password-input -- inputText: ${PASSWORD} -- pressKey: Enter - -- extendedWaitUntil: - visible: - id: tab-mail - timeout: 60000 -- runFlow: - when: - visible: "Not now" - commands: - - tapOn: "Not now" - -- extendedWaitUntil: - visible: - id: email-row - timeout: 60000 -- takeScreenshot: 10-inbox - -- tapOn: - id: email-row - index: 0 -- waitForAnimationToEnd: - timeout: 5000 -- takeScreenshot: 11-message -- tapOn: "Back" -- extendedWaitUntil: - visible: - id: email-row - timeout: 10000 - -- tapOn: - id: tab-calendar -- waitForAnimationToEnd: - timeout: 5000 -- takeScreenshot: 20-calendar - -- tapOn: - id: tab-contacts -- waitForAnimationToEnd: - timeout: 5000 -- takeScreenshot: 30-contacts - -- tapOn: - id: tab-files -- waitForAnimationToEnd: - timeout: 5000 -- takeScreenshot: 40-files - -- tapOn: - id: tab-settings -- waitForAnimationToEnd: - timeout: 5000 -- takeScreenshot: 50-settings - -# Still alive after the walk: the tab bar is there and the app didn't crash. -- assertVisible: - id: tab-mail diff --git a/.maestro/signin/01-password.yaml b/.maestro/signin/01-password.yaml new file mode 100644 index 00000000..f5aa3656 --- /dev/null +++ b/.maestro/signin/01-password.yaml @@ -0,0 +1,32 @@ +# Password sign-in as the first account, sign out, sign in again. +appId: org.bulwarkmail.mobile +name: Password, sign out, sign in again +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- takeScreenshot: password-01-choices +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${SERVER} +- takeScreenshot: password-02-confirm +- tapOn: "Sign in with a password instead" +- runFlow: + file: ../subflows/password-form.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: password-03-inbox + +- runFlow: ../subflows/sign-out.yaml +- extendedWaitUntil: + visible: "Enter server details manually" + timeout: 30000 +- takeScreenshot: password-04-signed-out + +- runFlow: + file: ../subflows/password-sign-in.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- takeScreenshot: password-05-signed-in-again diff --git a/.maestro/signin/02-password-errors.yaml b/.maestro/signin/02-password-errors.yaml new file mode 100644 index 00000000..5bbbf2b5 --- /dev/null +++ b/.maestro/signin/02-password-errors.yaml @@ -0,0 +1,48 @@ +# A wrong password and an unreachable server: the app says what went wrong +# and stays on the form. +appId: org.bulwarkmail.mobile +name: Password errors +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${SERVER} +- tapOn: "Sign in with a password instead" +- runFlow: + file: ../subflows/password-form.yaml + env: + LOGIN: ${ACCOUNT}@example.org + PASSWORD: wrong-password-123 +- extendedWaitUntil: + visible: "That didn't work" + timeout: 30000 +- takeScreenshot: errors-01-wrong-password +- assertVisible: + id: login-password-input + +# Back to the server step, then a host that doesn't resolve. +- tapOn: "Back" +- tapOn: "Back" +- runFlow: + file: ../subflows/type.yaml + env: + FIELD: login-server-input + TEXT: mail.does-not-exist.invalid +- tapOn: "Continue" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 +- tapOn: "Sign in with a password instead" +- runFlow: + file: ../subflows/password-form.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- extendedWaitUntil: + visible: "Can't reach mail.does-not-exist.invalid|No mail server at mail.does-not-exist.invalid|Sign-in failed" + timeout: 60000 +- takeScreenshot: errors-02-unreachable +- assertVisible: + id: login-password-input diff --git a/.maestro/signin/03-totp-password.yaml b/.maestro/signin/03-totp-password.yaml new file mode 100644 index 00000000..85514778 --- /dev/null +++ b/.maestro/signin/03-totp-password.yaml @@ -0,0 +1,24 @@ +# A TOTP account signs in with a password: Stalwart answers 402 for the right +# password, and the app asks for the code. +appId: org.bulwarkmail.mobile +name: Password with TOTP +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${SERVER} +- tapOn: "Sign in with a password instead" +- runFlow: + file: ../subflows/password-form.yaml + env: + LOGIN: ${TOTP_ACCOUNT}@example.org +- extendedWaitUntil: + visible: "Two-factor code" + timeout: 30000 +- takeScreenshot: totp-01-code-asked +- runFlow: ../subflows/two-factor.yaml +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: totp-02-inbox diff --git a/.maestro/signin/04-oauth.yaml b/.maestro/signin/04-oauth.yaml new file mode 100644 index 00000000..1a4a29eb --- /dev/null +++ b/.maestro/signin/04-oauth.yaml @@ -0,0 +1,20 @@ +# OAuth with Stalwart's own authorization server: the confirm step's +# Continue opens an ASWebAuthenticationSession on Stalwart's sign-in page. +appId: org.bulwarkmail.mobile +name: OAuth (Stalwart) +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${SERVER} +- tapOn: "Continue" +- runFlow: ../subflows/allow-auth-session.yaml +- runFlow: + file: ../subflows/stalwart-login-page.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: oauth-01-inbox diff --git a/.maestro/signin/05-oauth-cancel.yaml b/.maestro/signin/05-oauth-cancel.yaml new file mode 100644 index 00000000..fe757e43 --- /dev/null +++ b/.maestro/signin/05-oauth-cancel.yaml @@ -0,0 +1,39 @@ +# Backing out of OAuth: Cancel on the system alert, and closing the sheet +# halfway. Both leave the user on the confirm step without an error. +appId: org.bulwarkmail.mobile +name: OAuth cancelled +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${SERVER} +- tapOn: "Continue" +- extendedWaitUntil: + visible: ".*Wants to Use.*to Sign In.*" + timeout: 30000 +- tapOn: + text: "Cancel" + below: ".*Wants to Use.*to Sign In.*" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 +- assertNotVisible: "Sign-in failed" +- assertNotVisible: "Sign-in was interrupted" +- takeScreenshot: oauth-cancel-01-alert-cancelled + +# Accept the alert, then close the sheet on Stalwart's page. +- tapOn: "Continue" +- runFlow: ../subflows/allow-auth-session.yaml +- extendedWaitUntil: + visible: "you@example.com" + timeout: 60000 +- tapOn: "Cancel" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 +- assertNotVisible: "Sign-in failed" +- assertNotVisible: "Sign-in was interrupted" +- takeScreenshot: oauth-cancel-02-sheet-closed diff --git a/.maestro/signin/06-webmail.yaml b/.maestro/signin/06-webmail.yaml new file mode 100644 index 00000000..6814d955 --- /dev/null +++ b/.maestro/signin/06-webmail.yaml @@ -0,0 +1,20 @@ +# Sign in via webmail: the confirm step on a Bulwark webmail address hands +# off to the webmail's sign-in page in an auth session. +appId: org.bulwarkmail.mobile +name: Webmail hand-off +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${WEBMAIL} +- tapOn: "Continue" +- runFlow: ../subflows/allow-auth-session.yaml +- runFlow: + file: ../subflows/webmail-login-page.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: webmail-01-inbox diff --git a/.maestro/signin/07-webmail-totp.yaml b/.maestro/signin/07-webmail-totp.yaml new file mode 100644 index 00000000..604bc9f7 --- /dev/null +++ b/.maestro/signin/07-webmail-totp.yaml @@ -0,0 +1,22 @@ +# Webmail hand-off for a TOTP account: the webmail asks for the code; if it +# hands back the password, the app finishes with a code of its own. +appId: org.bulwarkmail.mobile +name: Webmail hand-off with TOTP +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${WEBMAIL} +- tapOn: "Continue" +- runFlow: ../subflows/allow-auth-session.yaml +- runFlow: + file: ../subflows/webmail-login-page.yaml + env: + LOGIN: ${TOTP_ACCOUNT}@example.org + TOTP: "yes" +- runFlow: ../subflows/finish-two-factor-if-asked.yaml +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: webmail-totp-01-inbox diff --git a/.maestro/signin/08-link-password.yaml b/.maestro/signin/08-link-password.yaml new file mode 100644 index 00000000..90615e17 --- /dev/null +++ b/.maestro/signin/08-link-password.yaml @@ -0,0 +1,17 @@ +# Sign-in link from another app (simctl openurl), password account. The +# helper signs in to the test webmail, creates a fresh "Link mobile app" code +# and opens its bulwarkmail://pair link on the simulator. +appId: org.bulwarkmail.mobile +name: Sign-in link from another app +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair?user=${ACCOUNT}&deliver=openurl +- runFlow: ../subflows/confirm-link.yaml +- runFlow: ../subflows/finish-two-factor-if-asked.yaml +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: link-password-01-inbox diff --git a/.maestro/signin/09-paste-password.yaml b/.maestro/signin/09-paste-password.yaml new file mode 100644 index 00000000..d74671f3 --- /dev/null +++ b/.maestro/signin/09-paste-password.yaml @@ -0,0 +1,16 @@ +# "Paste sign-in link" with a fresh pairing link on the clipboard (simctl +# pbcopy), password account. +appId: org.bulwarkmail.mobile +name: Paste sign-in link +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair?user=${ACCOUNT}&deliver=pbcopy +- runFlow: ../subflows/paste-link.yaml +- runFlow: ../subflows/finish-two-factor-if-asked.yaml +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: paste-password-01-inbox diff --git a/.maestro/signin/10-link-totp.yaml b/.maestro/signin/10-link-totp.yaml new file mode 100644 index 00000000..a50bc8ee --- /dev/null +++ b/.maestro/signin/10-link-totp.yaml @@ -0,0 +1,15 @@ +# Sign-in link from another app for a TOTP account. +appId: org.bulwarkmail.mobile +name: Sign-in link, TOTP account +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair?user=${TOTP_ACCOUNT}&deliver=openurl +- runFlow: ../subflows/confirm-link.yaml +- runFlow: ../subflows/finish-two-factor-if-asked.yaml +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: link-totp-01-inbox diff --git a/.maestro/signin/11-paste-totp.yaml b/.maestro/signin/11-paste-totp.yaml new file mode 100644 index 00000000..8fc369e6 --- /dev/null +++ b/.maestro/signin/11-paste-totp.yaml @@ -0,0 +1,15 @@ +# "Paste sign-in link" for a TOTP account. +appId: org.bulwarkmail.mobile +name: Paste sign-in link, TOTP account +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair?user=${TOTP_ACCOUNT}&deliver=pbcopy +- runFlow: ../subflows/paste-link.yaml +- runFlow: ../subflows/finish-two-factor-if-asked.yaml +- runFlow: ../subflows/inbox.yaml +- takeScreenshot: paste-totp-01-inbox diff --git a/.maestro/signin/12-link-expired.yaml b/.maestro/signin/12-link-expired.yaml new file mode 100644 index 00000000..a6ea5299 --- /dev/null +++ b/.maestro/signin/12-link-expired.yaml @@ -0,0 +1,21 @@ +# An expired code (codes live 2 minutes): the helper creates one now and +# opens it after 130 s. The app says so and stays on the sign-in screen. +appId: org.bulwarkmail.mobile +name: Expired sign-in code +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair-later?user=${ACCOUNT}&deliver=openurl&delay=130 +- runFlow: + file: ../subflows/confirm-link.yaml + env: + LINK_WAIT: 200000 +- extendedWaitUntil: + visible: "That code has expired.*" + timeout: 30000 +- takeScreenshot: link-expired-01-notice +- assertVisible: "Enter server details manually" diff --git a/.maestro/signin/13-link-used.yaml b/.maestro/signin/13-link-used.yaml new file mode 100644 index 00000000..798bc0eb --- /dev/null +++ b/.maestro/signin/13-link-used.yaml @@ -0,0 +1,44 @@ +# A code that was already used: sign in with it, sign out, open the same link +# again (same app process, then after a restart, which asks the server). +appId: org.bulwarkmail.mobile +name: Used sign-in code +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair?user=${ACCOUNT}&deliver=openurl +- runFlow: ../subflows/confirm-link.yaml +- runFlow: ../subflows/inbox.yaml +- runFlow: ../subflows/sign-out.yaml +- extendedWaitUntil: + visible: "Enter server details manually" + timeout: 30000 + +- runScript: + file: ../scripts/helper.js + env: + CALL: /redeliver?user=${ACCOUNT}&deliver=openurl +- runFlow: ../subflows/confirm-link.yaml +- extendedWaitUntil: + visible: "That code was already used|That code has expired or was already used" + timeout: 30000 +- takeScreenshot: link-used-01-same-process + +- stopApp +- launchApp +- extendedWaitUntil: + visible: "Enter server details manually" + timeout: 60000 +- runScript: + file: ../scripts/helper.js + env: + CALL: /redeliver?user=${ACCOUNT}&deliver=openurl +- runFlow: ../subflows/confirm-link.yaml +- extendedWaitUntil: + visible: "That code was already used|That code has expired or was already used" + timeout: 30000 +- takeScreenshot: link-used-02-after-restart +- assertVisible: "Enter server details manually" diff --git a/.maestro/signin/14-scanner-camera.yaml b/.maestro/signin/14-scanner-camera.yaml new file mode 100644 index 00000000..6e81d171 --- /dev/null +++ b/.maestro/signin/14-scanner-camera.yaml @@ -0,0 +1,32 @@ +# The QR scanner with camera access denied, then allowed (the simulator has +# no camera, so nothing can be scanned). Neither may crash the app. +appId: org.bulwarkmail.mobile +name: Scanner without camera +tags: + - signin +--- +- runFlow: + file: ../subflows/start.yaml + env: + CAMERA: deny +- tapOn: "Scan a sign-in code.*" +- extendedWaitUntil: + visible: "Camera access is turned off.*|Scanning a sign-in QR code uses the camera.*" + timeout: 15000 +- takeScreenshot: scanner-01-denied +- tapOn: "Close" +- assertVisible: "Enter server details manually" + +- runFlow: + file: ../subflows/start.yaml + env: + CAMERA: allow +- tapOn: "Scan a sign-in code.*" +- extendedWaitUntil: + visible: "Sign-in code" + timeout: 15000 +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: scanner-02-allowed +- tapOn: "Close" +- assertVisible: "Enter server details manually" diff --git a/.maestro/signin/15-email-discovery.yaml b/.maestro/signin/15-email-discovery.yaml new file mode 100644 index 00000000..52170516 --- /dev/null +++ b/.maestro/signin/15-email-discovery.yaml @@ -0,0 +1,23 @@ +# "Use my email address": example.org has no autodiscovery, so the app moves +# on to the server step and names the domain. +appId: org.bulwarkmail.mobile +name: Email address discovery +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- tapOn: "Use my email address.*" +- extendedWaitUntil: + visible: "What's your email address." + timeout: 10000 +- waitForAnimationToEnd: + timeout: 3000 +- inputText: ${ACCOUNT}@example.org +- assertVisible: ${ACCOUNT}@example.org +- tapOn: "Continue" +- extendedWaitUntil: + visible: "We couldn't find a server for example.org.*" + timeout: 60000 +- takeScreenshot: discovery-01-not-found +- assertVisible: + id: login-server-input diff --git a/.maestro/signin/16-add-password.yaml b/.maestro/signin/16-add-password.yaml new file mode 100644 index 00000000..829c1093 --- /dev/null +++ b/.maestro/signin/16-add-password.yaml @@ -0,0 +1,30 @@ +# Second account by password via "Another account on ", then switch +# between the two. +appId: org.bulwarkmail.mobile +name: Add account, password +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/password-sign-in.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runFlow: ../subflows/open-add-account.yaml +- takeScreenshot: add-password-01-choices +- tapOn: "Another account on .*" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 +- tapOn: "Sign in with a password instead" +- runFlow: + file: ../subflows/password-form.yaml + env: + LOGIN: ${ACCOUNT2}@example.org +- runFlow: ../subflows/inbox.yaml +- runFlow: + file: ../subflows/check-active-account.yaml + env: + ACTIVE: ${ACCOUNT2}@example.org +- assertVisible: ${ACCOUNT}@example.org +- takeScreenshot: add-password-02-two-accounts diff --git a/.maestro/signin/17-add-oauth.yaml b/.maestro/signin/17-add-oauth.yaml new file mode 100644 index 00000000..f48ba879 --- /dev/null +++ b/.maestro/signin/17-add-oauth.yaml @@ -0,0 +1,42 @@ +# Second account by OAuth. Adding uses an ephemeral auth session, so the +# first account's Stalwart sign-in must not be reused. +appId: org.bulwarkmail.mobile +name: Add account, OAuth +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/manual-server.yaml + env: + HOST: ${SERVER} +- tapOn: "Continue" +- runFlow: ../subflows/allow-auth-session.yaml +- runFlow: + file: ../subflows/stalwart-login-page.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runFlow: ../subflows/inbox.yaml + +- runFlow: ../subflows/open-add-account.yaml +- tapOn: "Another account on .*" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 +- tapOn: "Continue" +- runFlow: + when: + visible: ".*Wants to Use.*to Sign In.*" + commands: + - runFlow: ../subflows/allow-auth-session.yaml +- runFlow: + file: ../subflows/stalwart-login-page.yaml + env: + LOGIN: ${ACCOUNT2}@example.org +- runFlow: ../subflows/inbox.yaml +- runFlow: + file: ../subflows/check-active-account.yaml + env: + ACTIVE: ${ACCOUNT2}@example.org +- assertVisible: ${ACCOUNT}@example.org +- takeScreenshot: add-oauth-01-two-accounts diff --git a/.maestro/signin/18-add-webmail.yaml b/.maestro/signin/18-add-webmail.yaml new file mode 100644 index 00000000..65b2e934 --- /dev/null +++ b/.maestro/signin/18-add-webmail.yaml @@ -0,0 +1,43 @@ +# Second account via the webmail hand-off, on top of a password account. +appId: org.bulwarkmail.mobile +name: Add account, webmail +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/password-sign-in.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runFlow: ../subflows/open-add-account.yaml +- tapOn: "An account somewhere else.*" +- extendedWaitUntil: + visible: "What's the other address." + timeout: 10000 +- tapOn: "I know my server address" +- runFlow: + file: ../subflows/type.yaml + env: + FIELD: login-server-input + TEXT: ${WEBMAIL} +- tapOn: "Continue" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 +- tapOn: "Continue" +- runFlow: + when: + visible: ".*Wants to Use.*to Sign In.*" + commands: + - runFlow: ../subflows/allow-auth-session.yaml +- runFlow: + file: ../subflows/webmail-login-page.yaml + env: + LOGIN: ${ACCOUNT2}@example.org +- runFlow: ../subflows/inbox.yaml +- runFlow: + file: ../subflows/check-active-account.yaml + env: + ACTIVE: ${ACCOUNT2}@example.org +- assertVisible: ${ACCOUNT}@example.org +- takeScreenshot: add-webmail-01-two-accounts diff --git a/.maestro/signin/19-add-link.yaml b/.maestro/signin/19-add-link.yaml new file mode 100644 index 00000000..a786d251 --- /dev/null +++ b/.maestro/signin/19-add-link.yaml @@ -0,0 +1,30 @@ +# Second account from a sign-in link opened by another app while signed in. +appId: org.bulwarkmail.mobile +name: Add account, sign-in link +tags: + - signin +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/password-sign-in.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- runScript: + file: ../scripts/helper.js + env: + CALL: /pair?user=${ACCOUNT2}&deliver=openurl +- extendedWaitUntil: + visible: "Sign in with a link." + timeout: 60000 +- takeScreenshot: add-link-01-prompt +- assertVisible: "This link adds an account.*" +- tapOn: + text: "Continue" + below: "Sign in with a link." +- runFlow: ../subflows/inbox.yaml +- runFlow: + file: ../subflows/check-active-account.yaml + env: + ACTIVE: ${ACCOUNT2}@example.org +- assertVisible: ${ACCOUNT}@example.org +- takeScreenshot: add-link-02-two-accounts diff --git a/.maestro/subflows/allow-auth-session.yaml b/.maestro/subflows/allow-auth-session.yaml new file mode 100644 index 00000000..3cb482df --- /dev/null +++ b/.maestro/subflows/allow-auth-session.yaml @@ -0,0 +1,11 @@ +# iOS asks before an app opens a sign-in session in the system browser: +# "“Bulwark Mobile” Wants to Use “” to Sign In". Accept it. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: ".*Wants to Use.*to Sign In.*" + timeout: 30000 +- takeScreenshot: auth-session-alert +- tapOn: + text: "Continue" + below: ".*Wants to Use.*to Sign In.*" diff --git a/.maestro/subflows/check-active-account.yaml b/.maestro/subflows/check-active-account.yaml new file mode 100644 index 00000000..d8fd9de3 --- /dev/null +++ b/.maestro/subflows/check-active-account.yaml @@ -0,0 +1,16 @@ +# Settings → Account lists the accounts on this device; the active one is +# ACTIVE. Goes back to Mail afterwards. +appId: org.bulwarkmail.mobile +--- +- tapOn: + id: tab-settings +- extendedWaitUntil: + visible: "Account" + timeout: 10000 +- tapOn: "Account" +- extendedWaitUntil: + visible: ${ACTIVE} + timeout: 20000 +- takeScreenshot: accounts-${ACTIVE} +- tapOn: + id: tab-mail diff --git a/.maestro/subflows/confirm-link.yaml b/.maestro/subflows/confirm-link.yaml new file mode 100644 index 00000000..d0917c13 --- /dev/null +++ b/.maestro/subflows/confirm-link.yaml @@ -0,0 +1,10 @@ +# A sign-in link from another app asks first. Continue. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: "Sign in with a link.|Open the sign-in page of.*" + timeout: ${LINK_WAIT || 60000} +- takeScreenshot: link-confirm-prompt +- tapOn: + text: "Continue" + below: "Sign in with a link." diff --git a/.maestro/subflows/finish-two-factor-if-asked.yaml b/.maestro/subflows/finish-two-factor-if-asked.yaml new file mode 100644 index 00000000..d6bd8129 --- /dev/null +++ b/.maestro/subflows/finish-two-factor-if-asked.yaml @@ -0,0 +1,12 @@ +# Some routes hand the app a password for a TOTP account; the app then asks +# for a code itself. Answer it when it shows up. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: "Two-factor code|Inbox" + timeout: 90000 +- runFlow: + when: + visible: "Two-factor code" + commands: + - runFlow: two-factor.yaml diff --git a/.maestro/subflows/inbox.yaml b/.maestro/subflows/inbox.yaml new file mode 100644 index 00000000..0e132000 --- /dev/null +++ b/.maestro/subflows/inbox.yaml @@ -0,0 +1,18 @@ +# Signed in: the tab bar is there and the inbox shows messages. Dismisses the +# one-time prompts on the way. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: + id: tab-mail + timeout: 90000 +- runFlow: + when: + visible: "Not now" + commands: + - tapOn: "Not now" +- extendedWaitUntil: + visible: + id: email-row + timeout: 60000 +- assertVisible: ${INBOX_TEXT || 'Your verification code'} diff --git a/.maestro/subflows/manual-server.yaml b/.maestro/subflows/manual-server.yaml new file mode 100644 index 00000000..4959e03b --- /dev/null +++ b/.maestro/subflows/manual-server.yaml @@ -0,0 +1,18 @@ +# From the sign-in choices: enter a server address by hand and continue to the +# confirm step. env HOST: the address as a user types it. +appId: org.bulwarkmail.mobile +--- +- tapOn: "Enter server details manually" +- extendedWaitUntil: + visible: + id: login-server-input + timeout: 10000 +- runFlow: + file: type.yaml + env: + FIELD: login-server-input + TEXT: ${HOST} +- tapOn: "Continue" +- extendedWaitUntil: + visible: "Sign in with a password instead" + timeout: 20000 diff --git a/.maestro/subflows/open-add-account.yaml b/.maestro/subflows/open-add-account.yaml new file mode 100644 index 00000000..92a22102 --- /dev/null +++ b/.maestro/subflows/open-add-account.yaml @@ -0,0 +1,17 @@ +# Signed in: Settings → Account → Add account, up to the add-mode choices. +appId: org.bulwarkmail.mobile +--- +- tapOn: + id: tab-settings +- extendedWaitUntil: + visible: "Account" + timeout: 10000 +- tapOn: "Account" +- scrollUntilVisible: + element: "Add account" + direction: DOWN + timeout: 20000 +- tapOn: "Add account" +- extendedWaitUntil: + visible: "Add another account" + timeout: 10000 diff --git a/.maestro/subflows/password-form.yaml b/.maestro/subflows/password-form.yaml new file mode 100644 index 00000000..5ba26f5b --- /dev/null +++ b/.maestro/subflows/password-form.yaml @@ -0,0 +1,19 @@ +# On the password step: fill in LOGIN and PASSWORD and sign in. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: + id: login-username-input + timeout: 20000 +- runFlow: + file: type.yaml + env: + FIELD: login-username-input + TEXT: ${LOGIN} +- runFlow: + file: type-secret.yaml + env: + FIELD: login-password-input + TEXT: ${PASSWORD} +# Return on the password field signs in. +- pressKey: Enter diff --git a/.maestro/subflows/password-sign-in.yaml b/.maestro/subflows/password-sign-in.yaml new file mode 100644 index 00000000..ebd6af1f --- /dev/null +++ b/.maestro/subflows/password-sign-in.yaml @@ -0,0 +1,11 @@ +# From the sign-in choices to the inbox: manual server, password. +# env LOGIN: the account's address. +appId: org.bulwarkmail.mobile +--- +- runFlow: + file: manual-server.yaml + env: + HOST: ${SERVER} +- tapOn: "Sign in with a password instead" +- runFlow: password-form.yaml +- runFlow: inbox.yaml diff --git a/.maestro/subflows/paste-link.yaml b/.maestro/subflows/paste-link.yaml new file mode 100644 index 00000000..ef3c6cfa --- /dev/null +++ b/.maestro/subflows/paste-link.yaml @@ -0,0 +1,22 @@ +# The helper has put a sign-in link on the simulator's clipboard: open +# "Paste sign-in link", paste it and sign in. +appId: org.bulwarkmail.mobile +--- +- tapOn: "Paste sign-in link.*" +- extendedWaitUntil: + visible: "Paste" + timeout: 10000 +- tapOn: "Paste" +# iOS asks before an app reads the clipboard. +- runFlow: + when: + visible: "Allow Paste" + commands: + - tapOn: "Allow Paste" +- extendedWaitUntil: + visible: "bulwarkmail://pair.*" + timeout: 10000 +- takeScreenshot: paste-link-filled +- tapOn: + text: "Sign in" + below: "bulwarkmail://pair.*" diff --git a/.maestro/subflows/sign-out.yaml b/.maestro/subflows/sign-out.yaml new file mode 100644 index 00000000..6f016d4f --- /dev/null +++ b/.maestro/subflows/sign-out.yaml @@ -0,0 +1,16 @@ +# Signed in: sign out of the active account from the end of Settings. +appId: org.bulwarkmail.mobile +--- +- tapOn: + id: tab-settings +- scrollUntilVisible: + element: "Sign Out" + direction: DOWN + timeout: 30000 +- tapOn: "Sign Out" +- extendedWaitUntil: + visible: "The account is removed from this device.*" + timeout: 10000 +- tapOn: + text: "Sign out" + below: "The account is removed from this device.*" diff --git a/.maestro/subflows/stalwart-login-page.yaml b/.maestro/subflows/stalwart-login-page.yaml new file mode 100644 index 00000000..d8db349a --- /dev/null +++ b/.maestro/subflows/stalwart-login-page.yaml @@ -0,0 +1,25 @@ +# Stalwart's own sign-in page inside the auth session. env LOGIN, TOTP (any +# non-empty value: the account needs a one-time code). +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: "you@example.com" + timeout: 60000 +- takeScreenshot: stalwart-login-page +- tapOn: "you@example.com" +- inputText: ${LOGIN} +- tapOn: "••••••••" +- inputText: ${PASSWORD} +- runFlow: + when: + true: ${TOTP == 'yes'} + commands: + - runScript: + file: ../scripts/helper.js + env: + CALL: /totp + - tapOn: "000000" + - inputText: ${output.helper.code} +- tapOn: + text: "Sign in" + index: 1 diff --git a/.maestro/subflows/start.yaml b/.maestro/subflows/start.yaml new file mode 100644 index 00000000..c85d5e1f --- /dev/null +++ b/.maestro/subflows/start.yaml @@ -0,0 +1,20 @@ +# Fresh install state, rotated for the landscape leg, on the sign-in choices. +# env CAMERA: allow (default) or deny. +appId: org.bulwarkmail.mobile +--- +- launchApp: + clearState: true + clearKeychain: true + permissions: + notifications: allow + camera: ${CAMERA || 'allow'} + photos: allow +- runFlow: + when: + true: ${ORIENTATION == 'landscape'} + commands: + - setOrientation: LANDSCAPE_LEFT +# A cold start of a release build on a CI simulator can be slow. +- extendedWaitUntil: + visible: "Enter server details manually" + timeout: 120000 diff --git a/.maestro/subflows/two-factor.yaml b/.maestro/subflows/two-factor.yaml new file mode 100644 index 00000000..6e419252 --- /dev/null +++ b/.maestro/subflows/two-factor.yaml @@ -0,0 +1,21 @@ +# The two-factor field is showing (it focuses itself): type a fresh code from +# the helper and sign in. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: "Two-factor code" + timeout: 30000 +- runScript: + file: ../scripts/helper.js + env: + CALL: /totp +- waitForAnimationToEnd: + timeout: 3000 +- inputText: ${output.helper.code} +# The number pad has no return key. +- runFlow: + when: + notVisible: "Sign in" + commands: + - hideKeyboard +- tapOn: "Sign in" diff --git a/.maestro/subflows/type-secret.yaml b/.maestro/subflows/type-secret.yaml new file mode 100644 index 00000000..f4bd7a2f --- /dev/null +++ b/.maestro/subflows/type-secret.yaml @@ -0,0 +1,17 @@ +# Type a secret into the field with id FIELD. Same focus handling as +# type.yaml, but the value can't be checked (a secure field shows dots), and +# it must never be shown, so a lost keystroke surfaces as a failed sign-in. +appId: org.bulwarkmail.mobile +--- +- runFlow: + when: + notVisible: + id: ${FIELD} + focused: true + commands: + - tapOn: + id: ${FIELD} +- waitForAnimationToEnd: + timeout: 3000 +- eraseText: 80 +- inputText: ${TEXT} diff --git a/.maestro/subflows/type.yaml b/.maestro/subflows/type.yaml new file mode 100644 index 00000000..5ca5d6e8 --- /dev/null +++ b/.maestro/subflows/type.yaml @@ -0,0 +1,25 @@ +# Type TEXT into the field with id FIELD and check it arrived in full. +# +# iOS drops keystrokes while the edit menu (the "AutoFill" bubble) opens, +# which happens when a flow taps a field that already has focus; an +# auto-focused field is left alone. Retyped up to twice if text got lost. +appId: org.bulwarkmail.mobile +--- +- retry: + maxRetries: 2 + commands: + - runFlow: + when: + notVisible: + id: ${FIELD} + focused: true + commands: + - tapOn: + id: ${FIELD} + - waitForAnimationToEnd: + timeout: 3000 + - eraseText: 80 + - inputText: ${TEXT} + - assertVisible: + id: ${FIELD} + text: ${TEXT_PATTERN || TEXT} diff --git a/.maestro/subflows/webmail-login-page.yaml b/.maestro/subflows/webmail-login-page.yaml new file mode 100644 index 00000000..4ef5dddf --- /dev/null +++ b/.maestro/subflows/webmail-login-page.yaml @@ -0,0 +1,31 @@ +# The Bulwark webmail's sign-in page inside the auth session (the app's +# "Sign in via webmail" hand-off). env LOGIN, TOTP=yes for a TOTP account. +appId: org.bulwarkmail.mobile +--- +- extendedWaitUntil: + visible: "user@example.com" + timeout: 60000 +- takeScreenshot: webmail-login-page +- tapOn: "user@example.com" +- inputText: ${LOGIN} +- tapOn: "Enter your password" +- inputText: ${PASSWORD} +- tapOn: + text: "Sign in" + index: 1 +- runFlow: + when: + true: ${TOTP == 'yes'} + commands: + - extendedWaitUntil: + visible: "000000" + timeout: 30000 + - runScript: + file: ../scripts/helper.js + env: + CALL: /totp + - tapOn: "000000" + - inputText: ${output.helper.code} + - tapOn: + text: "Sign in|Verify" + index: 1 diff --git a/scripts/e2e/boot-sim.sh b/scripts/e2e/boot-sim.sh new file mode 100644 index 00000000..0547dd88 --- /dev/null +++ b/scripts/e2e/boot-sim.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Boot a fresh simulator for one test leg and install the app. +# +# scripts/e2e/boot-sim.sh +# +# Writes UDID and SIM_NAME to $GITHUB_ENV. Every simulator gets the keyboard +# set up for XCTest typing: a fresh one shows the slide-to-type tip over the +# keyboard the first time a field gets focus, and predictions, autocorrect and +# password AutoFill can swallow or rewrite typed text. +set -euo pipefail +DEVICE=$1 +APPEARANCE=$2 +APP=$3 + +case "$DEVICE" in + phone) PATTERN='^iPhone' ;; + tablet) PATTERN='^iPad Pro 11|^iPad Air 11|^iPad \(' ;; + *) echo "unknown device $DEVICE" >&2; exit 1 ;; +esac + +# The newest matching device type on the newest iOS runtime the image ships. +RUNTIME=$(xcrun simctl list runtimes available -j | jq -r '[.runtimes[] | select(.platform == "iOS")] | sort_by(.version | split(".") | map(tonumber)) | last | .identifier') +UDID=$(xcrun simctl list devices available -j | jq -r --arg rt "$RUNTIME" --arg re "$PATTERN" ' + [.devices[$rt][] | select(.name | test($re))] | last | .udid // empty') +if [ -z "$UDID" ]; then + xcrun simctl list devices available + echo "::error::No $DEVICE simulator on $RUNTIME" + exit 1 +fi +NAME=$(xcrun simctl list devices -j | jq -r --arg u "$UDID" '.devices[][] | select(.udid == $u) | .name') +echo "Using $NAME ($UDID) on $RUNTIME" + +xcrun simctl boot "$UDID" +xcrun simctl bootstatus "$UDID" -b + +PREFS=com.apple.keyboard.preferences +xcrun simctl spawn "$UDID" defaults write $PREFS DidShowContinuousPathIntroduction -bool true +xcrun simctl spawn "$UDID" defaults write $PREFS KeyboardContinuousPathEnabled -bool false +xcrun simctl spawn "$UDID" defaults write $PREFS KeyboardPrediction -bool false +xcrun simctl spawn "$UDID" defaults write $PREFS KeyboardAutocorrection -bool false +xcrun simctl spawn "$UDID" defaults write $PREFS KeyboardAutocapitalization -bool false +xcrun simctl spawn "$UDID" defaults write $PREFS KeyboardShowPredictionBar -bool false +xcrun simctl spawn "$UDID" defaults write com.apple.WebUI AutoFillPasswords -bool false || true + +xcrun simctl ui "$UDID" appearance "$APPEARANCE" +xcrun simctl install "$UDID" "$APP" + +{ + echo "UDID=$UDID" + echo "SIM_NAME=$NAME" +} >> "${GITHUB_ENV:-/dev/null}" diff --git a/scripts/e2e/failed-steps.mjs b/scripts/e2e/failed-steps.mjs new file mode 100644 index 00000000..f4fdf605 --- /dev/null +++ b/scripts/e2e/failed-steps.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node +// Summarise Maestro's debug output (~/.maestro/tests) without its secrets. +// +// The JSON there records every command with its arguments, so typed +// passwords are in it; it is never uploaded. This prints, per flow, the +// command types and their status (no arguments), and for a failed command +// the element tree Maestro saw (labels, ids and bounds), which is what a +// failing selector needs. run-flows.sh scrubs the secret values from the +// output as well. +import fs from 'node:fs'; +import path from 'node:path'; + +const root = process.argv[2]; +const out = []; + +function walkFiles(dir) { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const p = path.join(dir, e.name); + return e.isDirectory() ? walkFiles(p) : [p]; + }); +} + +function tree(node, depth, lines) { + if (!node || typeof node !== 'object') return; + const a = node.attributes || {}; + const label = [a.accessibilityText, a.text, a.hintText, a['resource-id'] && `#${a['resource-id']}`] + .filter(Boolean) + .filter((v, i, all) => all.indexOf(v) === i) + .join(' | '); + if (label) lines.push(`${' '.repeat(Math.min(depth, 12))}${label} ${a.bounds || ''}${a.focused === 'true' ? ' [focused]' : ''}`); + for (const c of node.children || []) tree(c, depth + 1, lines); +} + +function findHierarchy(obj, seen = new Set()) { + if (!obj || typeof obj !== 'object' || seen.has(obj)) return null; + seen.add(obj); + if (obj.attributes && Array.isArray(obj.children)) return obj; + for (const v of Object.values(obj)) { + const h = findHierarchy(v, seen); + if (h) return h; + } + return null; +} + +for (const file of walkFiles(root).filter((f) => /commands-.*\.json$/.test(f))) { + let data; + try { + data = JSON.parse(fs.readFileSync(file, 'utf8')); + } catch { + continue; + } + out.push(`== ${path.basename(file)}`); + for (const entry of Array.isArray(data) ? data : []) { + const command = entry.command || {}; + const type = Object.keys(command).find((k) => command[k]) || '?'; + const status = entry.metadata?.status || '?'; + out.push(` ${status.padEnd(9)} ${type}`); + if (status === 'FAILED') { + const err = entry.metadata?.error?.message; + if (err) out.push(` error: ${String(err).slice(0, 300)}`); + const h = findHierarchy(entry.metadata); + if (h) { + const lines = []; + tree(h, 0, lines); + out.push(...lines.slice(0, 250).map((l) => ` ${l}`)); + } + } + } +} +process.stdout.write(out.join('\n') + '\n'); diff --git a/scripts/e2e/helper.mjs b/scripts/e2e/helper.mjs new file mode 100644 index 00000000..9c23c068 --- /dev/null +++ b/scripts/e2e/helper.mjs @@ -0,0 +1,384 @@ +#!/usr/bin/env node +// Test helper for the iOS simulator flows (.maestro/). Runs on the CI runner +// next to the simulator; Maestro flows call it from runScript over +// http://127.0.0.1:8787 for things a flow can't do itself: +// +// GET /totp current TOTP code of the e2e-totp-* accounts +// GET /pair?user=&deliver= sign in to the test webmail with Playwright, +// create a fresh "Link mobile app" code and hand +// its bulwarkmail://pair link to the app +// (deliver=openurl: simctl openurl, as a link +// from another app; pbcopy: simctl pbcopy for +// "Paste sign-in link"; none: keep it only) +// GET /redeliver?user=&deliver= hand the last link for that user over again +// GET /pair-later?user=&deliver=&delay= create a code now, deliver it after +// seconds (expired-code test) +// GET /send?to=&subject=&attach=1&html=1 userb sends a message over JMAP +// GET /find?subject=&as=userb&timeout= wait until a message with that subject +// arrives; answers {found, attachments} +// GET /openurl?url= simctl openurl for other links (mailto:, ...) +// GET /clipboard?text= simctl pbcopy of a plain value +// +// Secrets come from the environment and never appear in a response: the +// pairing link is delivered to the simulator directly, not returned. The +// answers are JSON. Logs go to stdout without secrets or links. +// +// Env: E2E_SERVER (JMAP server), E2E_PASSWORD (shared by the e2e accounts), +// E2E_USERB_PASSWORD, E2E_TOTP_SECRET (base32), E2E_WEBMAIL (test webmail base URL), UDID +// (booted simulator), PLAYWRIGHT_DIR (node_modules folder with playwright). + +import http from 'node:http'; +import crypto from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { createRequire } from 'node:module'; +import path from 'node:path'; + +const PORT = Number(process.env.HELPER_PORT || 8787); +const SERVER = (process.env.E2E_SERVER || '').replace(/\/+$/, ''); +const WEBMAIL = (process.env.E2E_WEBMAIL || 'https://bulwark.internal.bulwarkmail.org').replace(/\/+$/, ''); +const PASSWORD = process.env.E2E_PASSWORD || ''; +const TOTP_SECRET = process.env.E2E_TOTP_SECRET || ''; +const UDID = process.env.UDID || 'booted'; +const DOMAIN = 'example.org'; + +const log = (...a) => console.log(new Date().toISOString().slice(11, 19), ...a); + +// ── TOTP (RFC 6238, SHA-1, 6 digits, 30 s) ────────────────────────────────── + +function base32Decode(s) { + const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; + let bits = 0; + let value = 0; + const out = []; + for (const ch of s.replace(/=+$/, '').toUpperCase()) { + const idx = alphabet.indexOf(ch); + if (idx < 0) continue; + value = (value << 5) | idx; + bits += 5; + if (bits >= 8) { + out.push((value >>> (bits - 8)) & 0xff); + bits -= 8; + } + } + return Buffer.from(out); +} + +function totpAt(secret, time) { + const counter = Math.floor(time / 30); + const buf = Buffer.alloc(8); + buf.writeBigUInt64BE(BigInt(counter)); + const mac = crypto.createHmac('sha1', base32Decode(secret)).update(buf).digest(); + const offset = mac[mac.length - 1] & 0xf; + const code = (mac.readUInt32BE(offset) & 0x7fffffff) % 1_000_000; + return String(code).padStart(6, '0'); +} + +// A code with at least `minLeft` seconds to live, waiting for the next window +// when the current one is about to close. +async function freshTotp(minLeft = 10) { + if (!TOTP_SECRET) throw new Error('E2E_TOTP_SECRET is not set'); + const now = Date.now() / 1000; + const left = 30 - (now % 30); + if (left < minLeft) await sleep((left + 0.5) * 1000); + return totpAt(TOTP_SECRET, Date.now() / 1000); +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// ── simulator ─────────────────────────────────────────────────────────────── + +function simctl(args, input) { + return new Promise((resolve, reject) => { + const child = execFile('xcrun', ['simctl', ...args], { timeout: 30_000 }, (err, stdout, stderr) => { + if (err) reject(new Error(`simctl ${args[0]} failed: ${stderr || err.message}`)); + else resolve(stdout); + }); + if (input !== undefined) { + child.stdin.end(input); + } + }); +} + +async function deliver(link, how) { + if (how === 'openurl') await simctl(['openurl', UDID, link]); + else if (how === 'pbcopy') await simctl(['pbcopy', UDID], link); + else if (how !== 'none') throw new Error(`unknown deliver=${how}`); +} + +// ── webmail pairing (Playwright) ──────────────────────────────────────────── + +let chromium = null; +function loadPlaywright() { + if (chromium) return chromium; + const dir = process.env.PLAYWRIGHT_DIR; + const req = createRequire(dir ? path.join(dir, 'noop.js') : import.meta.url); + ({ chromium } = req('playwright')); + return chromium; +} + +const lastLinks = new Map(); + +async function createPairingLink(user) { + const browser = await loadPlaywright().launch(); + const started = Date.now(); + let page = null; + try { + const context = await browser.newContext({ locale: 'en-US', viewport: { width: 1280, height: 900 } }); + page = await context.newPage(); + await page.goto(`${WEBMAIL}/en/login`, { waitUntil: 'domcontentloaded' }); + await page.locator('#username').fill(`${user}@${DOMAIN}`); + await page.locator('#password').fill(PASSWORD); + // Keep the session across page loads (a Basic session without "remember + // me" is signed out by the next navigation). + const remember = page.getByRole('checkbox', { name: /remember/i }); + if (await remember.count()) await remember.first().check().catch(() => undefined); + const totpField = page.locator('#totp'); + if (await totpField.isVisible().catch(() => false)) await totpField.fill(await freshTotp()); + await page.locator('button[type=submit]').first().click(); + + // A TOTP account: the webmail asks for the code after the password. + const signedIn = page.waitForURL((u) => !/\/login/.test(u.pathname), { timeout: 30_000 }).then(() => 'in'); + const askedCode = totpField.waitFor({ state: 'visible', timeout: 30_000 }).then(() => 'totp'); + const first = await Promise.race([signedIn, askedCode]); + if (first === 'totp' && /\/login/.test(new URL(page.url()).pathname)) { + await totpField.fill(await freshTotp()); + await page.locator('button[type=submit]').first().click(); + await page.waitForURL((u) => !/\/login/.test(u.pathname), { timeout: 30_000 }); + } + + await page.goto(`${WEBMAIL}/en/settings/security`, { waitUntil: 'domcontentloaded' }); + const show = page.getByRole('button', { name: /Show QR code/i }); + await show.waitFor({ timeout: 30_000 }); + await show.click(); + + // Step-up: the password again (and a code for a TOTP account). + const linkInput = page.locator('input[readonly][value^="bulwarkmail://"]'); + // The Security page has other password forms; use the link-device one. + const form = page.locator('form', { hasText: /confirm it's you/i }); + const pw = form.locator('input[type=password]').first(); + await Promise.race([ + linkInput.waitFor({ timeout: 30_000 }), + pw.waitFor({ timeout: 30_000 }), + ]); + if (!(await linkInput.count())) { + await pw.fill(PASSWORD); + // For a TOTP account the code field is there from the start. + const codeField = form.locator('input:not([type=password])'); + if (await codeField.count()) await codeField.last().fill(await freshTotp()); + await form.getByRole('button', { name: /^Continue$/ }).click(); + const codeAsked = page.getByText(/code from your authenticator app/i); + await Promise.race([ + linkInput.waitFor({ timeout: 30_000 }), + codeAsked.waitFor({ timeout: 30_000 }), + ]); + if (!(await linkInput.count())) { + if (!(await pw.inputValue())) await pw.fill(PASSWORD); + await form.locator('input:not([type=password])').last().fill(await freshTotp()); + await form.getByRole('button', { name: /^Continue$/ }).click(); + await linkInput.waitFor({ timeout: 30_000 }); + } + } + const link = await linkInput.inputValue(); + if (!link.startsWith('bulwarkmail://pair?')) throw new Error('no pairing link on the page'); + log(`pair: code for ${user} after ${Date.now() - started} ms`); + return link; + } catch (err) { + // What the webmail showed, for the run's artifacts (passwords are masked + // on the page; a TOTP code is dead in 30 s). + if (page && process.env.HELPER_SHOTS) { + await page.screenshot({ path: path.join(process.env.HELPER_SHOTS, `pair-${user}-${Date.now()}.png`), fullPage: true }).catch(() => undefined); + } + throw err; + } finally { + await browser.close(); + } +} + +// ── JMAP as userb (or any e2e account) ───────────────────────────────────── + +const sessions = new Map(); +async function jmapSession(user) { + if (sessions.has(user)) return sessions.get(user); + const pass = user === 'userb' && process.env.E2E_USERB_PASSWORD ? process.env.E2E_USERB_PASSWORD : PASSWORD; + const auth = 'Basic ' + Buffer.from(`${user}@${DOMAIN}:${pass}`).toString('base64'); + const res = await fetch(`${SERVER}/.well-known/jmap`, { headers: { Authorization: auth } }); + if (!res.ok) throw new Error(`session for ${user}: HTTP ${res.status}`); + const session = await res.json(); + const s = { + auth, + apiUrl: session.apiUrl, + uploadUrl: session.uploadUrl, + accountId: session.primaryAccounts['urn:ietf:params:jmap:mail'], + }; + sessions.set(user, s); + return s; +} + +async function jmap(user, calls) { + const s = await jmapSession(user); + const res = await fetch(s.apiUrl, { + method: 'POST', + headers: { Authorization: s.auth, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + using: ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:mail', 'urn:ietf:params:jmap:submission'], + methodCalls: calls, + }), + }); + if (!res.ok) throw new Error(`JMAP as ${user}: HTTP ${res.status}`); + return (await res.json()).methodResponses; +} + +async function upload(user, data, type) { + const s = await jmapSession(user); + const res = await fetch(s.uploadUrl.replace('{accountId}', s.accountId), { + method: 'POST', + headers: { Authorization: s.auth, 'Content-Type': type }, + body: data, + }); + if (!res.ok) throw new Error(`upload as ${user}: HTTP ${res.status}`); + return (await res.json()).blobId; +} + +async function sendMail({ from = 'userb', to, subject, text, html, attach }) { + const s = await jmapSession(from); + const [[, boxes], [, ids]] = await jmap(from, [ + ['Mailbox/get', { accountId: s.accountId, properties: ['role'] }, '0'], + ['Identity/get', { accountId: s.accountId }, '1'], + ]); + const drafts = boxes.list.find((m) => m.role === 'drafts')?.id; + const sent = boxes.list.find((m) => m.role === 'sent')?.id; + const identity = ids.list.find((i) => i.email === `${from}@${DOMAIN}`) ?? ids.list[0]; + const email = { + mailboxIds: { [drafts]: true }, + keywords: { $draft: true }, + from: [{ name: identity.name || from, email: `${from}@${DOMAIN}` }], + to: [{ email: to.includes('@') ? to : `${to}@${DOMAIN}` }], + subject, + bodyValues: { t: { value: text || `Test message "${subject}" from the iOS simulator tests.\n` } }, + textBody: [{ partId: 't', type: 'text/plain' }], + }; + if (html) { + email.bodyValues.h = { value: `

${subject}

An HTML test message with a link.

cell 1cell 2
` }; + email.htmlBody = [{ partId: 'h', type: 'text/html' }]; + } + if (attach) { + const pdf = Buffer.from( + '%PDF-1.4\n1 0 obj<>endobj 2 0 obj<>endobj ' + + '3 0 obj<>>>>>endobj ' + + '4 0 obj<>stream\nBT /F1 18 Tf 20 70 Td (E2E attachment) Tj ET\nendstream endobj ' + + '5 0 obj<>endobj\ntrailer<>\n%%EOF\n', + ); + const blobId = await upload(from, pdf, 'application/pdf'); + email.attachments = [{ blobId, type: 'application/pdf', name: 'e2e-attachment.pdf', disposition: 'attachment' }]; + } + const resp = await jmap(from, [ + ['Email/set', { accountId: s.accountId, create: { m: email } }, '0'], + ['EmailSubmission/set', { + accountId: s.accountId, + create: { s: { identityId: identity.id, emailId: '#m' } }, + onSuccessUpdateEmail: { '#s': { [`mailboxIds/${drafts}`]: null, [`mailboxIds/${sent}`]: true, 'keywords/$draft': null } }, + }, '1'], + ]); + const created = resp.find(([name, args]) => name === 'EmailSubmission/set' && args.created?.s); + if (!created) throw new Error(`send failed: ${JSON.stringify(resp).slice(0, 400)}`); + log(`send: ${from} -> ${to} "${subject}"`); +} + +async function findMail({ as = 'userb', subject, timeout = 30 }) { + const s = await jmapSession(as); + const deadline = Date.now() + timeout * 1000; + for (;;) { + const [[, q], [, got]] = await jmap(as, [ + ['Email/query', { accountId: s.accountId, filter: { subject }, sort: [{ property: 'receivedAt', isAscending: false }], limit: 5 }, '0'], + ['Email/get', { accountId: s.accountId, '#ids': { resultOf: '0', name: 'Email/query', path: '/ids' }, properties: ['subject', 'attachments', 'mailboxIds', 'from'] }, '1'], + ]); + const hit = got.list.find((m) => m.subject === subject); + if (hit) { + return { + found: true, + attachments: (hit.attachments || []).map((a) => a.name), + from: hit.from?.[0]?.email ?? null, + }; + } + if (Date.now() > deadline) return { found: false, total: q.ids.length }; + await sleep(2000); + } +} + +// ── server ────────────────────────────────────────────────────────────────── + +const routes = { + async '/health'() { + return { ok: true }; + }, + async '/totp'(q) { + return { code: await freshTotp(Number(q.get('minLeft') || 10)) }; + }, + async '/pair'(q) { + const user = q.get('user'); + const how = q.get('deliver') || 'openurl'; + const link = await createPairingLink(user); + lastLinks.set(user, link); + await deliver(link, how); + return { ok: true }; + }, + async '/redeliver'(q) { + const link = lastLinks.get(q.get('user')); + if (!link) throw new Error('no link for that user yet'); + await deliver(link, q.get('deliver') || 'openurl'); + return { ok: true }; + }, + async '/pair-later'(q) { + const user = q.get('user'); + const how = q.get('deliver') || 'openurl'; + const delay = Number(q.get('delay') || 130); + const link = await createPairingLink(user); + lastLinks.set(user, link); + setTimeout(() => { + deliver(link, how).then(() => log(`pair-later: delivered after ${delay}s`), (e) => log('pair-later:', e.message)); + }, delay * 1000); + return { ok: true, deliverIn: delay }; + }, + async '/send'(q) { + await sendMail({ + from: q.get('from') || 'userb', + to: q.get('to'), + subject: q.get('subject'), + text: q.get('text') || undefined, + html: q.get('html') === '1', + attach: q.get('attach') === '1', + }); + return { ok: true }; + }, + async '/find'(q) { + return findMail({ as: q.get('as') || 'userb', subject: q.get('subject'), timeout: Number(q.get('timeout') || 30) }); + }, + async '/openurl'(q) { + await simctl(['openurl', UDID, q.get('url')]); + return { ok: true }; + }, + async '/clipboard'(q) { + await simctl(['pbcopy', UDID], q.get('text') || ''); + return { ok: true }; + }, +}; + +http.createServer(async (req, res) => { + const url = new URL(req.url, 'http://localhost'); + const route = routes[url.pathname]; + res.setHeader('Content-Type', 'application/json'); + if (!route) { + res.statusCode = 404; + res.end(JSON.stringify({ error: 'not found' })); + return; + } + try { + const body = await route(url.searchParams); + log(`${url.pathname} ok`); + res.end(JSON.stringify(body)); + } catch (err) { + log(`${url.pathname} failed: ${err.message}`); + res.statusCode = 500; + res.end(JSON.stringify({ error: err.message })); + } +}).listen(PORT, '127.0.0.1', () => log(`e2e helper on :${PORT}`)); diff --git a/scripts/e2e/run-flows.sh b/scripts/e2e/run-flows.sh new file mode 100644 index 00000000..d11dddad --- /dev/null +++ b/scripts/e2e/run-flows.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Run one suite of Maestro flows against the booted simulator ($UDID). +# +# scripts/e2e/run-flows.sh [flow names, comma separated] +# +# Needs in the environment: UDID, OUT (output folder), and for the flows the +# E2E_* values (see .github/workflows/e2e-ios.yml). Output: $OUT/report.xml, +# $OUT/screenshots, $OUT/run.mp4, $OUT/app.log. +# +# Secrets: Maestro's debug output and the JSON under ~/.maestro/tests hold +# every typed text, passwords included, and artifacts of this public repo are +# public. Only the warm-up (which types a made-up address) prints a debug +# log; afterwards only PNGs are copied, and every text file in $OUT is +# scrubbed of the secret values before upload. +set -uo pipefail +SUITE=$1 +ONLY=${2:-} +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +mkdir -p "$OUT/screenshots" + +maestro --version + +xcrun simctl io "$UDID" recordVideo --codec=h264 --force "$OUT/run.mp4" & +RECORDER=$! + +# Warm-up: the XCTest driver takes minutes to start on a cold runner. +touch "$RUNNER_TEMP/flows-started" +(cd "$OUT/screenshots" && maestro --device "$UDID" test \ + --debug-output "$RUNNER_TEMP/maestro-warmup" "$ROOT/.maestro/launch.yaml") +STATUS=$? +if [ $STATUS -ne 0 ]; then + echo "::group::Maestro warm-up log" + find "$RUNNER_TEMP/maestro-warmup" -name 'maestro.log' -exec tail -n 300 {} \; + echo "::endgroup::" + maestro --device "$UDID" hierarchy > "$OUT/hierarchy-warmup.json" 2>/dev/null || true +fi + +if [ $STATUS -eq 0 ]; then + ARGS=( + -e "SERVER=$E2E_SERVER" + -e "PASSWORD=$E2E_PASSWORD" + -e "ACCOUNT=${E2E_ACCOUNT:-}" + -e "ACCOUNT2=${E2E_ACCOUNT2:-}" + -e "TOTP_ACCOUNT=${E2E_TOTP_ACCOUNT:-}" + -e "WEBMAIL=${E2E_WEBMAIL:-}" + -e "HELPER=http://127.0.0.1:${HELPER_PORT:-8787}" + -e "ORIENTATION=${ORIENTATION:-portrait}" + -e "DEVICE=${DEVICE:-phone}" + -e "APPEARANCE=${APPEARANCE:-light}" + ) + TARGETS=() + if [ -n "$ONLY" ]; then + IFS=',' read -ra NAMES <<< "$ONLY" + for n in "${NAMES[@]}"; do + f="$ROOT/.maestro/$SUITE/${n%.yaml}.yaml" + [ -f "$f" ] && TARGETS+=("$f") || echo "::warning::no flow $SUITE/$n" + done + else + TARGETS=("$ROOT/.maestro/$SUITE") + fi + if [ ${#TARGETS[@]} -gt 0 ]; then + (cd "$OUT/screenshots" && maestro --device "$UDID" test \ + --format junit --output "$OUT/report.xml" \ + "${ARGS[@]}" "${TARGETS[@]}") + STATUS=$? + fi +fi + +kill -INT "$RECORDER" 2>/dev/null || true +wait "$RECORDER" 2>/dev/null || true + +# Screenshots of failed steps land under ~/.maestro/tests. Images only, plus +# a summary of the steps without their arguments (see failed-steps.mjs). +find "$HOME/.maestro/tests" -name '*.png' -exec cp {} "$OUT/screenshots/" \; 2>/dev/null || true +node "$ROOT/scripts/e2e/failed-steps.mjs" "$HOME/.maestro/tests" > "$OUT/steps.txt" 2>&1 || true + +xcrun simctl spawn "$UDID" log show --last 60m --style compact \ + --predicate 'process == "BulwarkMobile"' > "$OUT/app.log" 2>&1 || true + +# Scrub secrets from every text file that gets uploaded. +for f in $(find "$OUT" -type f \( -name '*.xml' -o -name '*.log' -o -name '*.json' -o -name '*.txt' \)); do + node -e ' + const fs = require("fs"); + const f = process.argv[1]; + let s = fs.readFileSync(f, "utf8"); + for (const k of ["E2E_PASSWORD", "E2E_USERB_PASSWORD", "E2E_TOTP_SECRET"]) { + const v = process.env[k]; + if (v && v.length >= 6) s = s.split(v).join("***"); + } + fs.writeFileSync(f, s); + ' "$f" +done + +ls -la "$OUT/screenshots" | tail -n +2 | wc -l | xargs echo "screenshots:" +exit $STATUS