diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml index 8c269702..ea11f22a 100644 --- a/.github/workflows/e2e-ios.yml +++ b/.github/workflows/e2e-ios.yml @@ -318,7 +318,7 @@ jobs: APPS=(--app "base=$RUNNER_TEMP/base/app/BulwarkMobile.app" "${APPS[@]}") fi node scripts/e2e/perf.mjs "${APPS[@]}" --rounds "${{ inputs.perf_rounds || 5 }}" --cold 3 \ - --flow "$GITHUB_WORKSPACE/.maestro/perf/interactions.yaml" --out "$OUT" + --flows "$(ls "$GITHUB_WORKSPACE"/.maestro/perf/[0-9]-*.yaml | paste -sd, -)" --out "$OUT" - name: Upload results if: always() diff --git a/.maestro/perf/1-open-message.yaml b/.maestro/perf/1-open-message.yaml new file mode 100644 index 00000000..d89b3c38 --- /dev/null +++ b/.maestro/perf/1-open-message.yaml @@ -0,0 +1,27 @@ +# Perf: open a message and go back, five times. The app is already running +# with -perfMarks YES (no launchApp here: it would drop the launch argument). +appId: org.bulwarkmail.mobile +name: Perf open message +--- +- extendedWaitUntil: + visible: + id: email-row + timeout: 60000 +- repeat: + times: 5 + commands: + - tapOn: + id: email-row + index: 1 + - extendedWaitUntil: + visible: "Reply All" + timeout: 30000 + - waitForAnimationToEnd: + timeout: 3000 + - runFlow: + when: + visible: "Back" + commands: + - tapOn: "Back" + - waitForAnimationToEnd: + timeout: 3000 diff --git a/.maestro/perf/2-tabs.yaml b/.maestro/perf/2-tabs.yaml new file mode 100644 index 00000000..07be4454 --- /dev/null +++ b/.maestro/perf/2-tabs.yaml @@ -0,0 +1,33 @@ +# Perf: switch through the tabs, three times. +appId: org.bulwarkmail.mobile +name: Perf tab switches +--- +- repeat: + times: 3 + commands: + - tapOn: + id: tab-calendar + - runFlow: + when: + visible: ".*Would Like to Send You Notifications.*" + commands: + - tapOn: "Allow" + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-contacts + - waitForAnimationToEnd: + timeout: 3000 + - takeScreenshot: perf-contacts + - tapOn: + id: tab-files + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-settings + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: + id: tab-mail + - waitForAnimationToEnd: + timeout: 3000 diff --git a/.maestro/perf/3-compose.yaml b/.maestro/perf/3-compose.yaml new file mode 100644 index 00000000..49651559 --- /dev/null +++ b/.maestro/perf/3-compose.yaml @@ -0,0 +1,23 @@ +# Perf: open the composer and close it untouched, five times. +appId: org.bulwarkmail.mobile +name: Perf compose open +--- +- tapOn: + id: tab-mail +- repeat: + times: 5 + commands: + - tapOn: "Compose" + - extendedWaitUntil: + visible: "New Message" + timeout: 20000 + - waitForAnimationToEnd: + timeout: 3000 + - tapOn: "Close" + - waitForAnimationToEnd: + timeout: 3000 + - runFlow: + when: + visible: "Discard" + commands: + - tapOn: "Discard" diff --git a/.maestro/perf/4-scroll.yaml b/.maestro/perf/4-scroll.yaml new file mode 100644 index 00000000..422b81a9 --- /dev/null +++ b/.maestro/perf/4-scroll.yaml @@ -0,0 +1,28 @@ +# Perf: fling through the long inbox (thousands of messages), then back up. +appId: org.bulwarkmail.mobile +name: Perf list scroll +--- +- tapOn: + id: tab-mail +- extendedWaitUntil: + visible: + id: email-row + timeout: 30000 +- repeat: + times: 8 + commands: + - swipe: + start: 50%,80% + end: 50%,20% + duration: 250 + - waitForAnimationToEnd: + timeout: 4000 +- repeat: + times: 8 + commands: + - swipe: + start: 50%,20% + end: 50%,80% + duration: 250 + - waitForAnimationToEnd: + timeout: 4000 diff --git a/.maestro/perf/interactions.yaml b/.maestro/perf/interactions.yaml deleted file mode 100644 index eb7cf957..00000000 --- a/.maestro/perf/interactions.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Performance runs: the interactions the harness times (scripts/e2e/perf.mjs). -# The app is already running with -perfMarks YES; no launchApp here, it would -# start it without the launch argument. -appId: org.bulwarkmail.mobile -name: Perf interactions ---- -- extendedWaitUntil: - visible: - id: email-row - timeout: 60000 -- waitForAnimationToEnd: - timeout: 5000 - -# Open a message and go back, five times. -- repeat: - times: 5 - commands: - - tapOn: - id: email-row - index: 1 - - extendedWaitUntil: - visible: "Reply All" - timeout: 20000 - - waitForAnimationToEnd: - timeout: 3000 - - runFlow: - when: - visible: "Back" - commands: - - tapOn: "Back" - - waitForAnimationToEnd: - timeout: 3000 - -# Switch through the tabs, three times. -- repeat: - times: 3 - commands: - - tapOn: - id: tab-calendar - - waitForAnimationToEnd: - timeout: 3000 - - tapOn: - id: tab-contacts - - waitForAnimationToEnd: - timeout: 3000 - - tapOn: - id: tab-files - - waitForAnimationToEnd: - timeout: 3000 - - tapOn: - id: tab-settings - - waitForAnimationToEnd: - timeout: 3000 - - tapOn: - id: tab-mail - - waitForAnimationToEnd: - timeout: 3000 - -# Open the composer and close it untouched, five times. -- repeat: - times: 5 - commands: - - tapOn: "Compose" - - extendedWaitUntil: - visible: "New Message" - timeout: 20000 - - waitForAnimationToEnd: - timeout: 3000 - - tapOn: "Close" - - waitForAnimationToEnd: - timeout: 3000 - - runFlow: - when: - visible: "Discard" - commands: - - tapOn: "Discard" - -# Fling through the long list, then back up. -- repeat: - times: 8 - commands: - - swipe: - start: 50%,80% - end: 50%,20% - duration: 250 - - waitForAnimationToEnd: - timeout: 4000 -- repeat: - times: 8 - commands: - - swipe: - start: 50%,20% - end: 50%,80% - duration: 250 - - waitForAnimationToEnd: - timeout: 4000 diff --git a/scripts/e2e/perf.mjs b/scripts/e2e/perf.mjs index 2e6dc21d..fcf958e1 100644 --- a/scripts/e2e/perf.mjs +++ b/scripts/e2e/perf.mjs @@ -2,7 +2,7 @@ // Performance harness for the iOS simulator (Part 2 of docs/ios-test-2026-10.md). // // node scripts/e2e/perf.mjs --app base= --app cand= \ -// --rounds 5 --cold 3 --flow .maestro/perf/interactions.yaml --out +// --rounds 5 --cold 3 --flows .maestro/perf/1-open-message.yaml,... --out // // The app logs `[perf] {...}` lines when launched with `-perfMarks YES` // (src/lib/perf-marks.ts); they reach the simulator's system log at info @@ -12,8 +12,9 @@ // 2. cold-starts it `--cold` times: terminate, launch, wait for // `inbox-rendered`; cold start = that mark's time minus the launch time // taken on this machine (the simulator shares the host clock), -// 3. runs the Maestro interaction flow once (open messages, switch tabs, -// open compose, scroll), which produces the other marks. +// 3. runs the Maestro interaction flows once each (open messages, switch +// tabs, open compose, scroll), which produce the other marks. Each flow +// is its own run, so one that fails costs only its own metric. // Apps alternate inside every round, so both sides of a comparison see the // same runner, the same simulator and the same server load. // @@ -39,7 +40,7 @@ const UDID = process.env.UDID; const BUNDLE = 'org.bulwarkmail.mobile'; const rounds = Number(opt('rounds', 5)); const coldRuns = Number(opt('cold', 3)); -const flow = opt('flow', '.maestro/perf/interactions.yaml'); +const flows = (opt('flows', '') || '').split(',').filter(Boolean); const out = opt('out', 'perf-output'); const maestroEnv = (opt('env', '') || '').split(',').filter(Boolean); fs.mkdirSync(out, { recursive: true }); @@ -119,14 +120,14 @@ async function coldStart(label, round) { await sleep(3000); } -function runFlow(label) { +function runFlow(label, flow) { const before = marks.length; const env = maestroEnv.flatMap((kv) => ['-e', kv]); const res = spawn('maestro', ['--device', UDID, 'test', ...env, flow], { stdio: 'inherit' }); return new Promise((resolve) => { res.on('exit', (code) => { const got = marks.slice(before).filter((m) => typeof m.ms === 'number'); - console.log(` ${label} interactions: exit ${code}, ${got.length} timed marks`); + console.log(` ${label} ${path.basename(flow)}: exit ${code}, ${got.length} timed marks`); resolve(got); }); }); @@ -143,7 +144,8 @@ function runFlow(label) { try { simctl('terminate', UDID, BUNDLE); } catch { /* not running */ } simctl('install', UDID, app.path); for (let i = 0; i < coldRuns; i++) await coldStart(app.label, round); - const got = await runFlow(app.label); + const got = []; + for (const flow of flows) got.push(...(await runFlow(app.label, flow))); for (const m of got) { const metric = m.n === 'tab-switch' && m.to ? `tab-switch:${m.to}` : m.n; samples.push({ label: app.label, round, metric, ms: m.ms, ...(m.long !== undefined ? { long: m.long, frames: m.frames, worst: m.worst } : {}) });