diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml index 715cc5f5..8c269702 100644 --- a/.github/workflows/e2e-ios.yml +++ b/.github/workflows/e2e-ios.yml @@ -24,13 +24,22 @@ on: type: string required: false legs: - description: "Legs to run: all, signin, features, or a comma list (signin-phone, features-tablet-dark, ...)" + description: "Legs to run: all, signin, features, perf, or a comma list (signin-phone, features-tablet-dark, ...)" type: string default: all flows: description: "Only these flows (comma list of file names without .yaml); empty runs all" type: string required: false + perf_base_run_id: + description: "perf leg: compare against the simulator app of this run (run id)" + type: string + required: false + perf_rounds: + description: "perf leg: rounds (each: 3 cold starts and the interaction flow per app)" + type: string + default: "5" + required: false release: types: [published] @@ -226,3 +235,95 @@ jobs: flows: ${{ inputs.flows }} playwright: "false" name: ${{ matrix.name }} + + # Part 2 of docs/ios-test-2026-10.md: timing marks from the app + # (src/lib/perf-marks.ts), read by scripts/e2e/perf.mjs. Compares the app + # of this run (or app_run_id) with the app of perf_base_run_id, alternating + # them on one simulator so both sides share the runner. + perf: + needs: [plan, build] + if: >- + always() && !cancelled() && + (needs.build.result == 'success' || needs.build.result == 'skipped') && + contains(inputs.legs, 'perf') + runs-on: macos-latest + timeout-minutes: 120 + permissions: + contents: read + actions: read + env: + OUT: ${{ github.workspace }}/e2e-output + DEVICE: phone + APPEARANCE: light + ORIENTATION: portrait + E2E_SERVER: ${{ vars.E2E_SERVER }} + E2E_ACCOUNT: e2e-bulk + E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} + steps: + - uses: actions/checkout@v4 + + - name: Download the app under test (this run) + if: inputs.app_run_id == '' + uses: actions/download-artifact@v4 + with: + name: simulator-app + path: ${{ runner.temp }}/cand + - name: Download the app under test (earlier run) + if: inputs.app_run_id != '' + uses: actions/download-artifact@v4 + with: + name: simulator-app + run-id: ${{ inputs.app_run_id }} + github-token: ${{ github.token }} + path: ${{ runner.temp }}/cand + - name: Download the baseline app + if: inputs.perf_base_run_id != '' + uses: actions/download-artifact@v4 + with: + name: simulator-app + run-id: ${{ inputs.perf_base_run_id }} + github-token: ${{ github.token }} + path: ${{ runner.temp }}/base + + - name: Unpack + run: | + for side in cand base; do + [ -f "$RUNNER_TEMP/$side/simulator-app.tar.gz" ] || continue + mkdir -p "$RUNNER_TEMP/$side/app" + tar -C "$RUNNER_TEMP/$side/app" -xzf "$RUNNER_TEMP/$side/simulator-app.tar.gz" + done + echo "APP=$RUNNER_TEMP/cand/app/BulwarkMobile.app" >> "$GITHUB_ENV" + + - name: Boot a simulator + run: bash scripts/e2e/boot-sim.sh "$DEVICE" "$APPEARANCE" "$APP" + + - name: Install Maestro + run: | + curl -fsSL "https://get.maestro.mobile.dev" | bash + echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" + + - name: Sign in once + run: | + mkdir -p "$OUT/screenshots" + cd "$OUT/screenshots" + # Warm-up for the driver, then the sign-in (keychain survives reinstalls). + maestro --device "$UDID" test "$GITHUB_WORKSPACE/.maestro/launch.yaml" + maestro --device "$UDID" test -e "SERVER=$E2E_SERVER" -e "PASSWORD=$E2E_PASSWORD" \ + -e "ACCOUNT=$E2E_ACCOUNT" -e "ORIENTATION=portrait" "$GITHUB_WORKSPACE/.maestro/perf/sign-in.yaml" + + - name: Measure + run: | + APPS=(--app "cand=$RUNNER_TEMP/cand/app/BulwarkMobile.app") + if [ -d "$RUNNER_TEMP/base/app/BulwarkMobile.app" ]; then + APPS=(--app "base=$RUNNER_TEMP/base/app/BulwarkMobile.app" "${APPS[@]}") + fi + node scripts/e2e/perf.mjs "${APPS[@]}" --rounds "${{ inputs.perf_rounds || 5 }}" --cold 3 \ + --flow "$GITHUB_WORKSPACE/.maestro/perf/interactions.yaml" --out "$OUT" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-ios-perf + path: e2e-output/ + if-no-files-found: ignore diff --git a/.maestro/perf/interactions.yaml b/.maestro/perf/interactions.yaml new file mode 100644 index 00000000..eb7cf957 --- /dev/null +++ b/.maestro/perf/interactions.yaml @@ -0,0 +1,96 @@ +# Performance runs: the interactions the harness times (scripts/e2e/perf.mjs). +# The app is already running with -perfMarks YES; no launchApp here, it would +# start it without the launch argument. +appId: org.bulwarkmail.mobile +name: Perf interactions +--- +- extendedWaitUntil: + visible: + id: email-row + timeout: 60000 +- waitForAnimationToEnd: + timeout: 5000 + +# Open a message and go back, five times. +- repeat: + times: 5 + commands: + - tapOn: + id: email-row + index: 1 + - extendedWaitUntil: + visible: "Reply All" + timeout: 20000 + - waitForAnimationToEnd: + timeout: 3000 + - runFlow: + when: + visible: "Back" + commands: + - tapOn: "Back" + - waitForAnimationToEnd: + timeout: 3000 + +# Switch through the tabs, three times. +- repeat: + times: 3 + commands: + - tapOn: + id: tab-calendar + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-contacts + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-files + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-settings + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-mail + - waitForAnimationToEnd: + timeout: 3000 + +# Open the composer and close it untouched, five times. +- repeat: + times: 5 + commands: + - tapOn: "Compose" + - extendedWaitUntil: + visible: "New Message" + timeout: 20000 + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: "Close" + - waitForAnimationToEnd: + timeout: 3000 + - runFlow: + when: + visible: "Discard" + commands: + - tapOn: "Discard" + +# Fling through the long list, then back up. +- repeat: + times: 8 + commands: + - swipe: + start: 50%,80% + end: 50%,20% + duration: 250 + - waitForAnimationToEnd: + timeout: 4000 +- repeat: + times: 8 + commands: + - swipe: + start: 50%,20% + end: 50%,80% + duration: 250 + - waitForAnimationToEnd: + timeout: 4000 diff --git a/.maestro/perf/sign-in.yaml b/.maestro/perf/sign-in.yaml new file mode 100644 index 00000000..99b41936 --- /dev/null +++ b/.maestro/perf/sign-in.yaml @@ -0,0 +1,11 @@ +# Performance runs: sign in once (ACCOUNT, the bulk account with thousands of +# messages). Later installs of other builds keep the data and the keychain. +appId: org.bulwarkmail.mobile +name: Perf sign-in +--- +- runFlow: ../subflows/start.yaml +- runFlow: + file: ../subflows/password-sign-in.yaml + env: + LOGIN: ${ACCOUNT}@example.org + INBOX_TEXT: ".*Bulk.*" diff --git a/scripts/e2e/perf.mjs b/scripts/e2e/perf.mjs new file mode 100644 index 00000000..2e6dc21d --- /dev/null +++ b/scripts/e2e/perf.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node +// Performance harness for the iOS simulator (Part 2 of docs/ios-test-2026-10.md). +// +// node scripts/e2e/perf.mjs --app base= --app cand= \ +// --rounds 5 --cold 3 --flow .maestro/perf/interactions.yaml --out +// +// The app logs `[perf] {...}` lines when launched with `-perfMarks YES` +// (src/lib/perf-marks.ts); they reach the simulator's system log at info +// level. This script streams that log, and per round and app: +// 1. installs the app over the previous one (data and keychain stay, so it +// stays signed in), +// 2. cold-starts it `--cold` times: terminate, launch, wait for +// `inbox-rendered`; cold start = that mark's time minus the launch time +// taken on this machine (the simulator shares the host clock), +// 3. runs the Maestro interaction flow once (open messages, switch tabs, +// open compose, scroll), which produces the other marks. +// Apps alternate inside every round, so both sides of a comparison see the +// same runner, the same simulator and the same server load. +// +// Output: /perf.json (every sample) and /perf.md (medians). + +import { spawn, execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +const args = process.argv.slice(2); +const opt = (name, fallback) => { + const i = args.indexOf(`--${name}`); + return i >= 0 ? args[i + 1] : fallback; +}; +const apps = []; +for (let i = 0; i < args.length; i++) { + if (args[i] === '--app') { + const [label, p] = args[i + 1].split('='); + apps.push({ label, path: p }); + } +} +const UDID = process.env.UDID; +const BUNDLE = 'org.bulwarkmail.mobile'; +const rounds = Number(opt('rounds', 5)); +const coldRuns = Number(opt('cold', 3)); +const flow = opt('flow', '.maestro/perf/interactions.yaml'); +const out = opt('out', 'perf-output'); +const maestroEnv = (opt('env', '') || '').split(',').filter(Boolean); +fs.mkdirSync(out, { recursive: true }); + +if (!UDID || apps.length === 0) { + console.error('need UDID and at least one --app label=path'); + process.exit(2); +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const simctl = (...a) => execFileSync('xcrun', ['simctl', ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + +// ── log stream ────────────────────────────────────────────────────────────── +const marks = []; // { n, t, ...detail, label } +let currentLabel = null; +const waiters = []; +const stream = spawn('xcrun', [ + 'simctl', 'spawn', UDID, 'log', 'stream', '--level', 'info', '--style', 'ndjson', + '--predicate', 'subsystem == "com.facebook.react.log" AND eventMessage CONTAINS "[perf]"', +]); +let buf = ''; +stream.stdout.on('data', (chunk) => { + buf += chunk; + let nl; + while ((nl = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + let ev; + try { ev = JSON.parse(line); } catch { continue; } + const msg = String(ev.eventMessage || ''); + const at = msg.indexOf('[perf] '); + if (at < 0) continue; + // The message is the console line, possibly quoted by RCTLog. + const json = msg.slice(at + 7).replace(/^'|'$/g, ''); + let mark; + try { mark = JSON.parse(json); } catch { continue; } + mark.label = currentLabel; + marks.push(mark); + for (const w of [...waiters]) { + if (w.name === mark.n) { + waiters.splice(waiters.indexOf(w), 1); + w.resolve(mark); + } + } + } +}); + +function waitForMark(name, timeoutMs) { + return new Promise((resolve) => { + const w = { name, resolve }; + waiters.push(w); + setTimeout(() => { + const i = waiters.indexOf(w); + if (i >= 0) { waiters.splice(i, 1); resolve(null); } + }, timeoutMs); + }); +} + +// ── steps ─────────────────────────────────────────────────────────────────── +const samples = []; // { label, metric, ms, round } + +async function coldStart(label, round) { + try { simctl('terminate', UDID, BUNDLE); } catch { /* not running */ } + await sleep(1500); + const pending = waitForMark('inbox-rendered', 120_000); + const bundlePending = waitForMark('bundle-loaded', 120_000); + const t0 = Date.now(); + simctl('launch', UDID, BUNDLE, '-perfMarks', 'YES'); + const [mark, bundle] = await Promise.all([pending, bundlePending]); + if (!mark) { + console.log(` ${label} cold start: no inbox-rendered mark within 120 s`); + return; + } + samples.push({ label, round, metric: 'cold-start-to-inbox', ms: mark.t - t0 }); + if (bundle) samples.push({ label, round, metric: 'cold-start-to-bundle', ms: bundle.t - t0 }); + console.log(` ${label} cold start ${mark.t - t0} ms (bundle ${bundle ? bundle.t - t0 : '?'} ms, rows ${mark.rows})`); + await sleep(3000); +} + +function runFlow(label) { + const before = marks.length; + const env = maestroEnv.flatMap((kv) => ['-e', kv]); + const res = spawn('maestro', ['--device', UDID, 'test', ...env, flow], { stdio: 'inherit' }); + return new Promise((resolve) => { + res.on('exit', (code) => { + const got = marks.slice(before).filter((m) => typeof m.ms === 'number'); + console.log(` ${label} interactions: exit ${code}, ${got.length} timed marks`); + resolve(got); + }); + }); +} + +(async () => { + await sleep(2000); // let the log stream attach + for (let round = 1; round <= rounds; round++) { + console.log(`round ${round}/${rounds}`); + // Alternate the order too, so neither app always goes first. + const order = round % 2 ? apps : [...apps].reverse(); + for (const app of order) { + currentLabel = app.label; + try { simctl('terminate', UDID, BUNDLE); } catch { /* not running */ } + simctl('install', UDID, app.path); + for (let i = 0; i < coldRuns; i++) await coldStart(app.label, round); + const got = await runFlow(app.label); + for (const m of got) { + const metric = m.n === 'tab-switch' && m.to ? `tab-switch:${m.to}` : m.n; + samples.push({ label: app.label, round, metric, ms: m.ms, ...(m.long !== undefined ? { long: m.long, frames: m.frames, worst: m.worst } : {}) }); + } + } + } + stream.kill(); + + const median = (xs) => { + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : Math.round((s[m - 1] + s[m]) / 2); + }; + const metrics = [...new Set(samples.map((s) => s.metric))].sort(); + const labels = apps.map((a) => a.label); + const lines = [ + `| metric | ${labels.map((l) => `${l} median (n)`).join(' | ')} |${labels.length === 2 ? ' change |' : ''}`, + `|---|${labels.map(() => '---').join('|')}|${labels.length === 2 ? '---|' : ''}`, + ]; + for (const metric of metrics) { + const cells = labels.map((l) => { + const xs = samples.filter((s) => s.label === l && s.metric === metric).map((s) => s.ms); + return xs.length ? { med: median(xs), n: xs.length } : null; + }); + let change = ''; + if (labels.length === 2 && cells[0] && cells[1]) { + change = ` ${Math.round(((cells[1].med - cells[0].med) / cells[0].med) * 100)}% |`; + } else if (labels.length === 2) change = ' |'; + lines.push(`| ${metric} | ${cells.map((c) => (c ? `${c.med} ms (${c.n})` : '-')).join(' | ')} |${change}`); + } + const scroll = labels.map((l) => { + const xs = samples.filter((s) => s.label === l && s.metric === 'list-scroll'); + if (!xs.length) return null; + return `${l}: median ${median(xs.map((s) => s.long))} long frames of ${median(xs.map((s) => s.frames))}, worst ${median(xs.map((s) => s.worst))} ms`; + }).filter(Boolean); + const md = [...lines, '', ...scroll.map((s) => `- list-scroll ${s}`), ''].join('\n'); + fs.writeFileSync(path.join(out, 'perf.json'), JSON.stringify({ apps, rounds, coldRuns, samples }, null, 1)); + fs.writeFileSync(path.join(out, 'perf.md'), md); + console.log(md); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Performance\n\n${md}\n`); + process.exit(0); +})();