mirror of
https://github.com/bulwarkmail/native.git
synced 2026-10-31 00:06:17 +00:00
ci: add an ios performance leg that compares two simulator builds
This commit is contained in:
Linus Rath
@@ -24,13 +24,22 @@ on:
|
||||
type: string
|
||||
required: false
|
||||
legs:
|
||||
description: "Legs to run: all, signin, features, or a comma list (signin-phone, features-tablet-dark, ...)"
|
||||
description: "Legs to run: all, signin, features, perf, or a comma list (signin-phone, features-tablet-dark, ...)"
|
||||
type: string
|
||||
default: all
|
||||
flows:
|
||||
description: "Only these flows (comma list of file names without .yaml); empty runs all"
|
||||
type: string
|
||||
required: false
|
||||
perf_base_run_id:
|
||||
description: "perf leg: compare against the simulator app of this run (run id)"
|
||||
type: string
|
||||
required: false
|
||||
perf_rounds:
|
||||
description: "perf leg: rounds (each: 3 cold starts and the interaction flow per app)"
|
||||
type: string
|
||||
default: "5"
|
||||
required: false
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
@@ -226,3 +235,95 @@ jobs:
|
||||
flows: ${{ inputs.flows }}
|
||||
playwright: "false"
|
||||
name: ${{ matrix.name }}
|
||||
|
||||
# Part 2 of docs/ios-test-2026-10.md: timing marks from the app
|
||||
# (src/lib/perf-marks.ts), read by scripts/e2e/perf.mjs. Compares the app
|
||||
# of this run (or app_run_id) with the app of perf_base_run_id, alternating
|
||||
# them on one simulator so both sides share the runner.
|
||||
perf:
|
||||
needs: [plan, build]
|
||||
if: >-
|
||||
always() && !cancelled() &&
|
||||
(needs.build.result == 'success' || needs.build.result == 'skipped') &&
|
||||
contains(inputs.legs, 'perf')
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
OUT: ${{ github.workspace }}/e2e-output
|
||||
DEVICE: phone
|
||||
APPEARANCE: light
|
||||
ORIENTATION: portrait
|
||||
E2E_SERVER: ${{ vars.E2E_SERVER }}
|
||||
E2E_ACCOUNT: e2e-bulk
|
||||
E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download the app under test (this run)
|
||||
if: inputs.app_run_id == ''
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: simulator-app
|
||||
path: ${{ runner.temp }}/cand
|
||||
- name: Download the app under test (earlier run)
|
||||
if: inputs.app_run_id != ''
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: simulator-app
|
||||
run-id: ${{ inputs.app_run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
path: ${{ runner.temp }}/cand
|
||||
- name: Download the baseline app
|
||||
if: inputs.perf_base_run_id != ''
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: simulator-app
|
||||
run-id: ${{ inputs.perf_base_run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
path: ${{ runner.temp }}/base
|
||||
|
||||
- name: Unpack
|
||||
run: |
|
||||
for side in cand base; do
|
||||
[ -f "$RUNNER_TEMP/$side/simulator-app.tar.gz" ] || continue
|
||||
mkdir -p "$RUNNER_TEMP/$side/app"
|
||||
tar -C "$RUNNER_TEMP/$side/app" -xzf "$RUNNER_TEMP/$side/simulator-app.tar.gz"
|
||||
done
|
||||
echo "APP=$RUNNER_TEMP/cand/app/BulwarkMobile.app" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Boot a simulator
|
||||
run: bash scripts/e2e/boot-sim.sh "$DEVICE" "$APPEARANCE" "$APP"
|
||||
|
||||
- name: Install Maestro
|
||||
run: |
|
||||
curl -fsSL "https://get.maestro.mobile.dev" | bash
|
||||
echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Sign in once
|
||||
run: |
|
||||
mkdir -p "$OUT/screenshots"
|
||||
cd "$OUT/screenshots"
|
||||
# Warm-up for the driver, then the sign-in (keychain survives reinstalls).
|
||||
maestro --device "$UDID" test "$GITHUB_WORKSPACE/.maestro/launch.yaml"
|
||||
maestro --device "$UDID" test -e "SERVER=$E2E_SERVER" -e "PASSWORD=$E2E_PASSWORD" \
|
||||
-e "ACCOUNT=$E2E_ACCOUNT" -e "ORIENTATION=portrait" "$GITHUB_WORKSPACE/.maestro/perf/sign-in.yaml"
|
||||
|
||||
- name: Measure
|
||||
run: |
|
||||
APPS=(--app "cand=$RUNNER_TEMP/cand/app/BulwarkMobile.app")
|
||||
if [ -d "$RUNNER_TEMP/base/app/BulwarkMobile.app" ]; then
|
||||
APPS=(--app "base=$RUNNER_TEMP/base/app/BulwarkMobile.app" "${APPS[@]}")
|
||||
fi
|
||||
node scripts/e2e/perf.mjs "${APPS[@]}" --rounds "${{ inputs.perf_rounds || 5 }}" --cold 3 \
|
||||
--flow "$GITHUB_WORKSPACE/.maestro/perf/interactions.yaml" --out "$OUT"
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-ios-perf
|
||||
path: e2e-output/
|
||||
if-no-files-found: ignore
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# Performance runs: the interactions the harness times (scripts/e2e/perf.mjs).
|
||||
# The app is already running with -perfMarks YES; no launchApp here, it would
|
||||
# start it without the launch argument.
|
||||
appId: org.bulwarkmail.mobile
|
||||
name: Perf interactions
|
||||
---
|
||||
- extendedWaitUntil:
|
||||
visible:
|
||||
id: email-row
|
||||
timeout: 60000
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 5000
|
||||
|
||||
# Open a message and go back, five times.
|
||||
- repeat:
|
||||
times: 5
|
||||
commands:
|
||||
- tapOn:
|
||||
id: email-row
|
||||
index: 1
|
||||
- extendedWaitUntil:
|
||||
visible: "Reply All"
|
||||
timeout: 20000
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- runFlow:
|
||||
when:
|
||||
visible: "Back"
|
||||
commands:
|
||||
- tapOn: "Back"
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
|
||||
# Switch through the tabs, three times.
|
||||
- repeat:
|
||||
times: 3
|
||||
commands:
|
||||
- tapOn:
|
||||
id: tab-calendar
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- tapOn:
|
||||
id: tab-contacts
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- tapOn:
|
||||
id: tab-files
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- tapOn:
|
||||
id: tab-settings
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- tapOn:
|
||||
id: tab-mail
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
|
||||
# Open the composer and close it untouched, five times.
|
||||
- repeat:
|
||||
times: 5
|
||||
commands:
|
||||
- tapOn: "Compose"
|
||||
- extendedWaitUntil:
|
||||
visible: "New Message"
|
||||
timeout: 20000
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- tapOn: "Close"
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 3000
|
||||
- runFlow:
|
||||
when:
|
||||
visible: "Discard"
|
||||
commands:
|
||||
- tapOn: "Discard"
|
||||
|
||||
# Fling through the long list, then back up.
|
||||
- repeat:
|
||||
times: 8
|
||||
commands:
|
||||
- swipe:
|
||||
start: 50%,80%
|
||||
end: 50%,20%
|
||||
duration: 250
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 4000
|
||||
- repeat:
|
||||
times: 8
|
||||
commands:
|
||||
- swipe:
|
||||
start: 50%,20%
|
||||
end: 50%,80%
|
||||
duration: 250
|
||||
- waitForAnimationToEnd:
|
||||
timeout: 4000
|
||||
@@ -0,0 +1,11 @@
|
||||
# Performance runs: sign in once (ACCOUNT, the bulk account with thousands of
|
||||
# messages). Later installs of other builds keep the data and the keychain.
|
||||
appId: org.bulwarkmail.mobile
|
||||
name: Perf sign-in
|
||||
---
|
||||
- runFlow: ../subflows/start.yaml
|
||||
- runFlow:
|
||||
file: ../subflows/password-sign-in.yaml
|
||||
env:
|
||||
LOGIN: ${ACCOUNT}@example.org
|
||||
INBOX_TEXT: ".*Bulk.*"
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env node
|
||||
// Performance harness for the iOS simulator (Part 2 of docs/ios-test-2026-10.md).
|
||||
//
|
||||
// node scripts/e2e/perf.mjs --app base=<path.app> --app cand=<path.app> \
|
||||
// --rounds 5 --cold 3 --flow .maestro/perf/interactions.yaml --out <dir>
|
||||
//
|
||||
// The app logs `[perf] {...}` lines when launched with `-perfMarks YES`
|
||||
// (src/lib/perf-marks.ts); they reach the simulator's system log at info
|
||||
// level. This script streams that log, and per round and app:
|
||||
// 1. installs the app over the previous one (data and keychain stay, so it
|
||||
// stays signed in),
|
||||
// 2. cold-starts it `--cold` times: terminate, launch, wait for
|
||||
// `inbox-rendered`; cold start = that mark's time minus the launch time
|
||||
// taken on this machine (the simulator shares the host clock),
|
||||
// 3. runs the Maestro interaction flow once (open messages, switch tabs,
|
||||
// open compose, scroll), which produces the other marks.
|
||||
// Apps alternate inside every round, so both sides of a comparison see the
|
||||
// same runner, the same simulator and the same server load.
|
||||
//
|
||||
// Output: <out>/perf.json (every sample) and <out>/perf.md (medians).
|
||||
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const opt = (name, fallback) => {
|
||||
const i = args.indexOf(`--${name}`);
|
||||
return i >= 0 ? args[i + 1] : fallback;
|
||||
};
|
||||
const apps = [];
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--app') {
|
||||
const [label, p] = args[i + 1].split('=');
|
||||
apps.push({ label, path: p });
|
||||
}
|
||||
}
|
||||
const UDID = process.env.UDID;
|
||||
const BUNDLE = 'org.bulwarkmail.mobile';
|
||||
const rounds = Number(opt('rounds', 5));
|
||||
const coldRuns = Number(opt('cold', 3));
|
||||
const flow = opt('flow', '.maestro/perf/interactions.yaml');
|
||||
const out = opt('out', 'perf-output');
|
||||
const maestroEnv = (opt('env', '') || '').split(',').filter(Boolean);
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
|
||||
if (!UDID || apps.length === 0) {
|
||||
console.error('need UDID and at least one --app label=path');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const simctl = (...a) => execFileSync('xcrun', ['simctl', ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
|
||||
// ── log stream ──────────────────────────────────────────────────────────────
|
||||
const marks = []; // { n, t, ...detail, label }
|
||||
let currentLabel = null;
|
||||
const waiters = [];
|
||||
const stream = spawn('xcrun', [
|
||||
'simctl', 'spawn', UDID, 'log', 'stream', '--level', 'info', '--style', 'ndjson',
|
||||
'--predicate', 'subsystem == "com.facebook.react.log" AND eventMessage CONTAINS "[perf]"',
|
||||
]);
|
||||
let buf = '';
|
||||
stream.stdout.on('data', (chunk) => {
|
||||
buf += chunk;
|
||||
let nl;
|
||||
while ((nl = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, nl);
|
||||
buf = buf.slice(nl + 1);
|
||||
let ev;
|
||||
try { ev = JSON.parse(line); } catch { continue; }
|
||||
const msg = String(ev.eventMessage || '');
|
||||
const at = msg.indexOf('[perf] ');
|
||||
if (at < 0) continue;
|
||||
// The message is the console line, possibly quoted by RCTLog.
|
||||
const json = msg.slice(at + 7).replace(/^'|'$/g, '');
|
||||
let mark;
|
||||
try { mark = JSON.parse(json); } catch { continue; }
|
||||
mark.label = currentLabel;
|
||||
marks.push(mark);
|
||||
for (const w of [...waiters]) {
|
||||
if (w.name === mark.n) {
|
||||
waiters.splice(waiters.indexOf(w), 1);
|
||||
w.resolve(mark);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function waitForMark(name, timeoutMs) {
|
||||
return new Promise((resolve) => {
|
||||
const w = { name, resolve };
|
||||
waiters.push(w);
|
||||
setTimeout(() => {
|
||||
const i = waiters.indexOf(w);
|
||||
if (i >= 0) { waiters.splice(i, 1); resolve(null); }
|
||||
}, timeoutMs);
|
||||
});
|
||||
}
|
||||
|
||||
// ── steps ───────────────────────────────────────────────────────────────────
|
||||
const samples = []; // { label, metric, ms, round }
|
||||
|
||||
async function coldStart(label, round) {
|
||||
try { simctl('terminate', UDID, BUNDLE); } catch { /* not running */ }
|
||||
await sleep(1500);
|
||||
const pending = waitForMark('inbox-rendered', 120_000);
|
||||
const bundlePending = waitForMark('bundle-loaded', 120_000);
|
||||
const t0 = Date.now();
|
||||
simctl('launch', UDID, BUNDLE, '-perfMarks', 'YES');
|
||||
const [mark, bundle] = await Promise.all([pending, bundlePending]);
|
||||
if (!mark) {
|
||||
console.log(` ${label} cold start: no inbox-rendered mark within 120 s`);
|
||||
return;
|
||||
}
|
||||
samples.push({ label, round, metric: 'cold-start-to-inbox', ms: mark.t - t0 });
|
||||
if (bundle) samples.push({ label, round, metric: 'cold-start-to-bundle', ms: bundle.t - t0 });
|
||||
console.log(` ${label} cold start ${mark.t - t0} ms (bundle ${bundle ? bundle.t - t0 : '?'} ms, rows ${mark.rows})`);
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
function runFlow(label) {
|
||||
const before = marks.length;
|
||||
const env = maestroEnv.flatMap((kv) => ['-e', kv]);
|
||||
const res = spawn('maestro', ['--device', UDID, 'test', ...env, flow], { stdio: 'inherit' });
|
||||
return new Promise((resolve) => {
|
||||
res.on('exit', (code) => {
|
||||
const got = marks.slice(before).filter((m) => typeof m.ms === 'number');
|
||||
console.log(` ${label} interactions: exit ${code}, ${got.length} timed marks`);
|
||||
resolve(got);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await sleep(2000); // let the log stream attach
|
||||
for (let round = 1; round <= rounds; round++) {
|
||||
console.log(`round ${round}/${rounds}`);
|
||||
// Alternate the order too, so neither app always goes first.
|
||||
const order = round % 2 ? apps : [...apps].reverse();
|
||||
for (const app of order) {
|
||||
currentLabel = app.label;
|
||||
try { simctl('terminate', UDID, BUNDLE); } catch { /* not running */ }
|
||||
simctl('install', UDID, app.path);
|
||||
for (let i = 0; i < coldRuns; i++) await coldStart(app.label, round);
|
||||
const got = await runFlow(app.label);
|
||||
for (const m of got) {
|
||||
const metric = m.n === 'tab-switch' && m.to ? `tab-switch:${m.to}` : m.n;
|
||||
samples.push({ label: app.label, round, metric, ms: m.ms, ...(m.long !== undefined ? { long: m.long, frames: m.frames, worst: m.worst } : {}) });
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.kill();
|
||||
|
||||
const median = (xs) => {
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
const m = Math.floor(s.length / 2);
|
||||
return s.length % 2 ? s[m] : Math.round((s[m - 1] + s[m]) / 2);
|
||||
};
|
||||
const metrics = [...new Set(samples.map((s) => s.metric))].sort();
|
||||
const labels = apps.map((a) => a.label);
|
||||
const lines = [
|
||||
`| metric | ${labels.map((l) => `${l} median (n)`).join(' | ')} |${labels.length === 2 ? ' change |' : ''}`,
|
||||
`|---|${labels.map(() => '---').join('|')}|${labels.length === 2 ? '---|' : ''}`,
|
||||
];
|
||||
for (const metric of metrics) {
|
||||
const cells = labels.map((l) => {
|
||||
const xs = samples.filter((s) => s.label === l && s.metric === metric).map((s) => s.ms);
|
||||
return xs.length ? { med: median(xs), n: xs.length } : null;
|
||||
});
|
||||
let change = '';
|
||||
if (labels.length === 2 && cells[0] && cells[1]) {
|
||||
change = ` ${Math.round(((cells[1].med - cells[0].med) / cells[0].med) * 100)}% |`;
|
||||
} else if (labels.length === 2) change = ' |';
|
||||
lines.push(`| ${metric} | ${cells.map((c) => (c ? `${c.med} ms (${c.n})` : '-')).join(' | ')} |${change}`);
|
||||
}
|
||||
const scroll = labels.map((l) => {
|
||||
const xs = samples.filter((s) => s.label === l && s.metric === 'list-scroll');
|
||||
if (!xs.length) return null;
|
||||
return `${l}: median ${median(xs.map((s) => s.long))} long frames of ${median(xs.map((s) => s.frames))}, worst ${median(xs.map((s) => s.worst))} ms`;
|
||||
}).filter(Boolean);
|
||||
const md = [...lines, '', ...scroll.map((s) => `- list-scroll ${s}`), ''].join('\n');
|
||||
fs.writeFileSync(path.join(out, 'perf.json'), JSON.stringify({ apps, rounds, coldRuns, samples }, null, 1));
|
||||
fs.writeFileSync(path.join(out, 'perf.md'), md);
|
||||
console.log(md);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Performance\n\n${md}\n`);
|
||||
process.exit(0);
|
||||
})();
|
||||
Reference in New Issue
Block a user