starred/native
1
0
Fork 0
mirror of https://github.com/bulwarkmail/native.git synced 2026-10-31 00:06:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: run each ios perf measurement as its own flow

Browse Source
This commit is contained in:
Linus Rath
2026-10-22 08:29:55 +02:00
parent 8533bd4f99
commit 680d4745ed
7 changed files with 121 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e-ios.yml
+1 -1
View File
@@ -318,7 +318,7 @@ jobs:
APPS=(--app "base=$RUNNER_TEMP/base/app/BulwarkMobile.app" "${APPS[@]}")
fi
node scripts/e2e/perf.mjs "${APPS[@]}" --rounds "${{ inputs.perf_rounds || 5 }}" --cold 3 \
--flow "$GITHUB_WORKSPACE/.maestro/perf/interactions.yaml" --out "$OUT"
--flows "$(ls "$GITHUB_WORKSPACE"/.maestro/perf/[0-9]-*.yaml | paste -sd, -)" --out "$OUT"
- name: Upload results
if: always()
.maestro/perf/1-open-message.yaml
+27
View File
@@ -0,0 +1,27 @@
# Perf: open a message and go back, five times. The app is already running
# with -perfMarks YES (no launchApp here: it would drop the launch argument).
appId: org.bulwarkmail.mobile
name: Perf open message
---
- extendedWaitUntil:
visible:
id: email-row
timeout: 60000
- repeat:
times: 5
commands:
- tapOn:
id: email-row
index: 1
- extendedWaitUntil:
visible: "Reply All"
timeout: 30000
- waitForAnimationToEnd:
timeout: 3000
- runFlow:
when:
visible: "Back"
commands:
- tapOn: "Back"
- waitForAnimationToEnd:
timeout: 3000
.maestro/perf/2-tabs.yaml
+33
View File
@@ -0,0 +1,33 @@
# Perf: switch through the tabs, three times.
appId: org.bulwarkmail.mobile
name: Perf tab switches
---
- repeat:
times: 3
commands:
- tapOn:
id: tab-calendar
- runFlow:
when:
visible: ".*Would Like to Send You Notifications.*"
commands:
- tapOn: "Allow"
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-contacts
- waitForAnimationToEnd:
timeout: 3000
- takeScreenshot: perf-contacts
- tapOn:
id: tab-files
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-settings
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-mail
- waitForAnimationToEnd:
timeout: 3000
.maestro/perf/3-compose.yaml
+23
View File
@@ -0,0 +1,23 @@
# Perf: open the composer and close it untouched, five times.
appId: org.bulwarkmail.mobile
name: Perf compose open
---
- tapOn:
id: tab-mail
- repeat:
times: 5
commands:
- tapOn: "Compose"
- extendedWaitUntil:
visible: "New Message"
timeout: 20000
- waitForAnimationToEnd:
timeout: 3000
- tapOn: "Close"
- waitForAnimationToEnd:
timeout: 3000
- runFlow:
when:
visible: "Discard"
commands:
- tapOn: "Discard"
.maestro/perf/4-scroll.yaml
+28
View File
@@ -0,0 +1,28 @@
# Perf: fling through the long inbox (thousands of messages), then back up.
appId: org.bulwarkmail.mobile
name: Perf list scroll
---
- tapOn:
id: tab-mail
- extendedWaitUntil:
visible:
id: email-row
timeout: 30000
- repeat:
times: 8
commands:
- swipe:
start: 50%,80%
end: 50%,20%
duration: 250
- waitForAnimationToEnd:
timeout: 4000
- repeat:
times: 8
commands:
- swipe:
start: 50%,20%
end: 50%,80%
duration: 250
- waitForAnimationToEnd:
timeout: 4000
.maestro/perf/interactions.yaml
-96
View File
@@ -1,96 +0,0 @@
# Performance runs: the interactions the harness times (scripts/e2e/perf.mjs).
# The app is already running with -perfMarks YES; no launchApp here, it would
# start it without the launch argument.
appId: org.bulwarkmail.mobile
name: Perf interactions
---
- extendedWaitUntil:
visible:
id: email-row
timeout: 60000
- waitForAnimationToEnd:
timeout: 5000
# Open a message and go back, five times.
- repeat:
times: 5
commands:
- tapOn:
id: email-row
index: 1
- extendedWaitUntil:
visible: "Reply All"
timeout: 20000
- waitForAnimationToEnd:
timeout: 3000
- runFlow:
when:
visible: "Back"
commands:
- tapOn: "Back"
- waitForAnimationToEnd:
timeout: 3000
# Switch through the tabs, three times.
- repeat:
times: 3
commands:
- tapOn:
id: tab-calendar
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-contacts
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-files
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-settings
- waitForAnimationToEnd:
timeout: 3000
- tapOn:
id: tab-mail
- waitForAnimationToEnd:
timeout: 3000
# Open the composer and close it untouched, five times.
- repeat:
times: 5
commands:
- tapOn: "Compose"
- extendedWaitUntil:
visible: "New Message"
timeout: 20000
- waitForAnimationToEnd:
timeout: 3000
- tapOn: "Close"
- waitForAnimationToEnd:
timeout: 3000
- runFlow:
when:
visible: "Discard"
commands:
- tapOn: "Discard"
# Fling through the long list, then back up.
- repeat:
times: 8
commands:
- swipe:
start: 50%,80%
end: 50%,20%
duration: 250
- waitForAnimationToEnd:
timeout: 4000
- repeat:
times: 8
commands:
- swipe:
start: 50%,20%
end: 50%,80%
duration: 250
- waitForAnimationToEnd:
timeout: 4000
scripts/e2e/perf.mjs
+9 -7
View File
@@ -2,7 +2,7 @@
// Performance harness for the iOS simulator (Part 2 of docs/ios-test-2026-10.md).
//
// node scripts/e2e/perf.mjs --app base=<path.app> --app cand=<path.app> \
// --rounds 5 --cold 3 --flow .maestro/perf/interactions.yaml --out <dir>
// --rounds 5 --cold 3 --flows .maestro/perf/1-open-message.yaml,... --out <dir>
//
// The app logs `[perf] {...}` lines when launched with `-perfMarks YES`
// (src/lib/perf-marks.ts); they reach the simulator's system log at info
@@ -12,8 +12,9 @@
// 2. cold-starts it `--cold` times: terminate, launch, wait for
// `inbox-rendered`; cold start = that mark's time minus the launch time
// taken on this machine (the simulator shares the host clock),
// 3. runs the Maestro interaction flow once (open messages, switch tabs,
// open compose, scroll), which produces the other marks.
// 3. runs the Maestro interaction flows once each (open messages, switch
// tabs, open compose, scroll), which produce the other marks. Each flow
// is its own run, so one that fails costs only its own metric.
// Apps alternate inside every round, so both sides of a comparison see the
// same runner, the same simulator and the same server load.
//
@@ -39,7 +40,7 @@ const UDID = process.env.UDID;
const BUNDLE = 'org.bulwarkmail.mobile';
const rounds = Number(opt('rounds', 5));
const coldRuns = Number(opt('cold', 3));
const flow = opt('flow', '.maestro/perf/interactions.yaml');
const flows = (opt('flows', '') || '').split(',').filter(Boolean);
const out = opt('out', 'perf-output');
const maestroEnv = (opt('env', '') || '').split(',').filter(Boolean);
fs.mkdirSync(out, { recursive: true });
@@ -119,14 +120,14 @@ async function coldStart(label, round) {
await sleep(3000);
}
function runFlow(label) {
function runFlow(label, flow) {
const before = marks.length;
const env = maestroEnv.flatMap((kv) => ['-e', kv]);
const res = spawn('maestro', ['--device', UDID, 'test', ...env, flow], { stdio: 'inherit' });
return new Promise((resolve) => {
res.on('exit', (code) => {
const got = marks.slice(before).filter((m) => typeof m.ms === 'number');
console.log(` ${label} interactions: exit ${code}, ${got.length} timed marks`);
console.log(` ${label} ${path.basename(flow)}: exit ${code}, ${got.length} timed marks`);
resolve(got);
});
});
@@ -143,7 +144,8 @@ function runFlow(label) {
try { simctl('terminate', UDID, BUNDLE); } catch { /* not running */ }
simctl('install', UDID, app.path);
for (let i = 0; i < coldRuns; i++) await coldStart(app.label, round);
const got = await runFlow(app.label);
const got = [];
for (const flow of flows) got.push(...(await runFlow(app.label, flow)));
for (const m of got) {
const metric = m.n === 'tab-switch' && m.to ? `tab-switch:${m.to}` : m.n;
samples.push({ label: app.label, round, metric, ms: m.ms, ...(m.long !== undefined ? { long: m.long, frames: m.frames, worst: m.worst } : {}) });