From 3db98b70f74487d93f19482825946f6715ee71d7 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 2 Sep 2026 00:32:06 -0400 Subject: [PATCH 1/5] fix(mobile-e2e): hold the simulator long press across two gesture commands One `orca emulator gesture` call delivers its whole point list as a single batch, so the 62-point begin/hold/end sequence landed as a tap: the native Source Control baseline long-pressed the workspace row, navigated into the session instead, and failed on a missing "Source Control" control. Split the press across two commands, hold, poll for the settled control while the touch is down, and always lift. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .../hosted-ios-emulator-long-press.mjs | 49 ++++++++--- ...ios-host-origin-source-control-journey.mjs | 2 +- ...osted-ios-source-control-review-parity.mjs | 8 +- .../hosted-ios-emulator-long-press.test.ts | 87 ++++++++++++++----- ...host-origin-source-control-journey.test.ts | 8 +- 5 files changed, 121 insertions(+), 33 deletions(-) diff --git a/mobile/scripts/hosted-ios-emulator-long-press.mjs b/mobile/scripts/hosted-ios-emulator-long-press.mjs index ab01772800d..7f389ed5704 100644 --- a/mobile/scripts/hosted-ios-emulator-long-press.mjs +++ b/mobile/scripts/hosted-ios-emulator-long-press.mjs @@ -1,35 +1,64 @@ import { runHostedIosEmulatorCommand, + waitForHostedIosAccessibilityControl, waitForHostedIosAccessibilityControlByLabelPrefix } from './hosted-ios-emulator-accessibility.mjs' -const LONG_PRESS_MOVE_FRAMES = 60 +// Why: a single `gesture` command delivers its whole point list as one batch, so +// begin/hold-frames/end lands as a tap however many hold frames it carries. The +// touch only stays down across separate commands, so the press is split in two. +const LONG_PRESS_HOLD_MS = 900 +const LONG_PRESS_SETTLE_TIMEOUT_MS = 8_000 export async function longPressHostedIosAccessibilityControlByLabelPrefix( args, labelPrefix, timeoutMs, - runCommand = runHostedIosEmulatorCommand + runCommand = runHostedIosEmulatorCommand, + settledLabel = null ) { + const deadline = Date.now() + timeoutMs const point = await waitForHostedIosAccessibilityControlByLabelPrefix( args, labelPrefix, timeoutMs, runCommand ) - await longPressHostedIosPoint(args, point, runCommand) + await longPressHostedIosPoint(args, point, runCommand, { deadline, settledLabel }) return point } export async function longPressHostedIosPoint( args, point, - runCommand = runHostedIosEmulatorCommand + runCommand = runHostedIosEmulatorCommand, + { deadline = Date.now() + LONG_PRESS_SETTLE_TIMEOUT_MS, settledLabel = null } = {} ) { - const holdFrames = Array.from({ length: LONG_PRESS_MOVE_FRAMES }, () => ({ - type: 'move', - ...point - })) - const gesture = [{ type: 'begin', ...point }, ...holdFrames, { type: 'end', ...point }] - await runCommand(args, ['gesture', JSON.stringify(gesture)]) + const sendGesture = (points) => runCommand(args, ['gesture', JSON.stringify(points)]) + // The move frame flushes the touch down; a lone begin never reaches the app. + await sendGesture([ + { type: 'begin', ...point }, + { type: 'move', ...point } + ]) + try { + await delay(LONG_PRESS_HOLD_MS) + if (settledLabel) { + await waitForHostedIosAccessibilityControl( + args, + settledLabel, + Math.min(LONG_PRESS_SETTLE_TIMEOUT_MS, Math.max(1_000, deadline - Date.now())), + runCommand + ) + } + } finally { + // Always lift: a stuck touch poisons every later step on the device. + await sendGesture([ + { type: 'move', ...point }, + { type: 'end', ...point } + ]) + } +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) } diff --git a/mobile/scripts/hosted-ios-host-origin-source-control-journey.mjs b/mobile/scripts/hosted-ios-host-origin-source-control-journey.mjs index c72bbc4ce43..84bf559e946 100644 --- a/mobile/scripts/hosted-ios-host-origin-source-control-journey.mjs +++ b/mobile/scripts/hosted-ios-host-origin-source-control-journey.mjs @@ -23,7 +23,7 @@ export async function verifyHostedHostOriginSourceControlJourney({ const waitForDocument = operations.waitForDocument ?? waitForVisibleHostedWebView const readState = operations.readState ?? readHostedWebViewState - await longPress(emulator, workspaceName, timeoutMs) + await longPress(emulator, workspaceName, timeoutMs, undefined, 'Source Control') await tapNative(emulator, 'Source Control', timeoutMs) const sourceControl = await waitForDocument({ discoveryUrl, diff --git a/mobile/scripts/hosted-ios-source-control-review-parity.mjs b/mobile/scripts/hosted-ios-source-control-review-parity.mjs index b8e7f0b9c12..6d6c367d3c2 100644 --- a/mobile/scripts/hosted-ios-source-control-review-parity.mjs +++ b/mobile/scripts/hosted-ios-source-control-review-parity.mjs @@ -24,7 +24,13 @@ export async function captureNativeSourceControlReviewBaselines({ timeoutMs }) { await dismissEmulatorDeveloperMenuIfPresent(emulator) - await longPressHostedIosAccessibilityControlByLabelPrefix(emulator, expectedWorkspace, timeoutMs) + await longPressHostedIosAccessibilityControlByLabelPrefix( + emulator, + expectedWorkspace, + timeoutMs, + undefined, + 'Source Control' + ) await tapHostedIosAccessibilityControl(emulator, 'Source Control', timeoutMs) const changedFileControl = await waitForHostedIosAccessibilityControlMatching( emulator, diff --git a/mobile/src/mobile-web/hosted-ios-emulator-long-press.test.ts b/mobile/src/mobile-web/hosted-ios-emulator-long-press.test.ts index 40023ff8268..1f4c6905610 100644 --- a/mobile/src/mobile-web/hosted-ios-emulator-long-press.test.ts +++ b/mobile/src/mobile-web/hosted-ios-emulator-long-press.test.ts @@ -8,38 +8,85 @@ const emulator = { worktree: '/repo' } +const rowTree = { + ok: true, + result: [ + { + label: 'mobile-rearch, mobile-rearch, 1', + enabled: true, + frame: { x: 0, y: 0.2, width: 1, height: 0.1 } + } + ] +} + +const sheetTree = { + ok: true, + result: [ + { label: 'Source Control', enabled: true, frame: { x: 0, y: 0.7, width: 1, height: 0.05 } } + ] +} + +function axResponse(tree: unknown) { + return { stderr: '', stdout: JSON.stringify(tree) } +} + +function gesturePoints(call: [unknown, string[]] | undefined) { + return JSON.parse(call?.[1][1] as string) +} + describe('hosted iOS emulator long press', () => { - it('holds a composite workspace row beyond the React Native threshold', async () => { + it('holds the touch down across two gesture commands', async () => { const runCommand = vi .fn() - .mockResolvedValueOnce({ - stderr: '', - stdout: JSON.stringify({ - ok: true, - result: [ - { - label: 'mobile-rearch, mobile-rearch, 1', - enabled: true, - frame: { x: 0, y: 0.2, width: 1, height: 0.1 } - } - ] - }) - }) + .mockResolvedValueOnce(axResponse(rowTree)) + .mockResolvedValueOnce({ stderr: '', stdout: JSON.stringify({ ok: true }) }) + .mockResolvedValueOnce(axResponse(sheetTree)) .mockResolvedValueOnce({ stderr: '', stdout: JSON.stringify({ ok: true }) }) await expect( longPressHostedIosAccessibilityControlByLabelPrefix( emulator, 'mobile-rearch', - 1_000, - runCommand + 20_000, + runCommand, + 'Source Control' ) ).resolves.toEqual({ x: 0.5, y: 0.25 }) - const gesture = JSON.parse(runCommand.mock.calls[1]?.[1][1] as string) + // Why: a single command batches its points, so the press must span two commands. expect(runCommand.mock.calls[1]?.[1][0]).toBe('gesture') - expect(gesture).toHaveLength(62) - expect(gesture[0]).toEqual({ type: 'begin', x: 0.5, y: 0.25 }) - expect(gesture.at(-1)).toEqual({ type: 'end', x: 0.5, y: 0.25 }) + expect(gesturePoints(runCommand.mock.calls[1])).toEqual([ + { type: 'begin', x: 0.5, y: 0.25 }, + { type: 'move', x: 0.5, y: 0.25 } + ]) + expect(runCommand.mock.calls[3]?.[1][0]).toBe('gesture') + expect(gesturePoints(runCommand.mock.calls[3])).toEqual([ + { type: 'move', x: 0.5, y: 0.25 }, + { type: 'end', x: 0.5, y: 0.25 } + ]) + }) + + it('releases the touch when the press never settles', async () => { + const runCommand = vi.fn(async (_args: unknown, command: string[]) => { + if (command[0] === 'ax') { + return axResponse(rowTree) + } + return { stderr: '', stdout: JSON.stringify({ ok: true }) } + }) + + await expect( + longPressHostedIosAccessibilityControlByLabelPrefix( + emulator, + 'mobile-rearch', + 2_000, + runCommand, + 'Source Control' + ) + ).rejects.toThrow('Source Control was not accessible') + + const endGestures = runCommand.mock.calls.filter( + (call) => call[1][0] === 'gesture' && gesturePoints(call).at(-1).type === 'end' + ) + expect(endGestures).toHaveLength(1) }) }) diff --git a/mobile/src/mobile-web/hosted-ios-host-origin-source-control-journey.test.ts b/mobile/src/mobile-web/hosted-ios-host-origin-source-control-journey.test.ts index 0d442c142b6..adc03744509 100644 --- a/mobile/src/mobile-web/hosted-ios-host-origin-source-control-journey.test.ts +++ b/mobile/src/mobile-web/hosted-ios-host-origin-source-control-journey.test.ts @@ -44,7 +44,13 @@ describe('hosted iOS host-origin Source Control journey', () => { operations: { activate, longPress, readState, tapNative, waitForDocument } }) - expect(longPress).toHaveBeenCalledWith({ udid: 'SIMULATOR-1' }, 'mobile-rearch', 30_000) + expect(longPress).toHaveBeenCalledWith( + { udid: 'SIMULATOR-1' }, + 'mobile-rearch', + 30_000, + undefined, + 'Source Control' + ) expect(tapNative).toHaveBeenCalledWith({ udid: 'SIMULATOR-1' }, 'Source Control', 30_000) expect(activate.mock.calls.map((call) => call[1])).toEqual([ { kind: 'label', value: 'Open changed file mobile/src/mobile-web/bridge.ts' }, From 8e668a75314c45fa6ac8b3c4b645ff8af97d5233 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 2 Sep 2026 00:42:59 -0400 Subject: [PATCH 2/5] test(mobile-e2e): drop the retired synthetic-resume gate and add a native A/B capture The Agent History journeys still asserted "Failed to resume session." for a page-synthesized resume. mr-gesture-delete removed that gate on purpose, so the synthetic click now resumes and the wait timed out on the resumed session route. Both journeys keep the native-touch resume. Adds a native journey capture that walks the six baseline journeys recording ordered accessibility labels plus a screenshot per stop, a diff over two captures, and an ORCA_E2E_MOBILE_METRO_DIR override so an old client checkout can serve Metro while the paired desktop runtime stays on this one. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- ...diff-hosted-ios-native-journey-capture.mjs | 63 ++++++ .../hosted-android-agent-history-journey.mjs | 12 -- .../hosted-ios-agent-history-journey.mjs | 12 -- .../hosted-ios-emulator-accessibility.mjs | 7 + mobile/scripts/hosted-ios-mobile-launcher.mjs | 6 +- .../hosted-ios-native-journey-labels.mjs | 203 ++++++++++++++++++ .../run-hosted-ios-native-journey-capture.mjs | 119 ++++++++++ mobile/scripts/start-emulator.mjs | 3 + ...sted-android-agent-history-journey.test.ts | 3 +- 9 files changed, 401 insertions(+), 27 deletions(-) create mode 100644 mobile/scripts/diff-hosted-ios-native-journey-capture.mjs create mode 100644 mobile/scripts/hosted-ios-native-journey-labels.mjs create mode 100644 mobile/scripts/run-hosted-ios-native-journey-capture.mjs diff --git a/mobile/scripts/diff-hosted-ios-native-journey-capture.mjs b/mobile/scripts/diff-hosted-ios-native-journey-capture.mjs new file mode 100644 index 00000000000..6dfb88c5f25 --- /dev/null +++ b/mobile/scripts/diff-hosted-ios-native-journey-capture.mjs @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +import { readFileSync } from 'node:fs' +import process from 'node:process' + +// Why: the status bar changes between runs, so it cannot distinguish a client +// difference from a second passing on the clock. +const VOLATILE_LABEL = + /^(\d{1,2}:\d{2}\s?(AM|PM)|Dynamic Island.*|Cellular|SSID.*|\d+% battery power|Not charging|Charging|No signal|\d of \d Wi-Fi bars)$/ + +function readCapture(file) { + const capture = JSON.parse(readFileSync(file, 'utf8')) + return { capture, stops: new Map(capture.stops.map((stop) => [stop.name, stop])) } +} + +function normalize(labels, workspace) { + return labels + .filter((label) => typeof label === 'string' && label.length > 0) + .filter((label) => !VOLATILE_LABEL.test(label)) + .map((label) => label.replaceAll(workspace, '')) +} + +function diffStop(name, left, right, leftWorkspace, rightWorkspace) { + if (!left || !right) { + return [`${name}: only captured on ${left ? 'the first' : 'the second'} client`] + } + const a = normalize(left.labels, leftWorkspace) + const b = normalize(right.labels, rightWorkspace) + const lines = [] + for (const label of new Set(a.filter((label) => !b.includes(label)))) { + lines.push(`${name}: only on the first client: ${label}`) + } + for (const label of new Set(b.filter((label) => !a.includes(label)))) { + lines.push(`${name}: only on the second client: ${label}`) + } + if (lines.length === 0 && a.join(' ') !== b.join(' ')) { + lines.push(`${name}: same label set in a different order`) + } + return lines +} + +const [leftFile, rightFile] = process.argv.slice(2) +if (!leftFile || !rightFile) { + throw new Error( + 'Usage: node scripts/diff-hosted-ios-native-journey-capture.mjs ' + ) +} +const left = readCapture(leftFile) +const right = readCapture(rightFile) +const names = [...new Set([...left.stops.keys(), ...right.stops.keys()])] +const differences = names.flatMap((name) => + diffStop( + name, + left.stops.get(name), + right.stops.get(name), + left.capture.expectedWorkspace, + right.capture.expectedWorkspace + ) +) +console.log( + JSON.stringify({ stops: names, differenceCount: differences.length, differences }, null, 2) +) +process.exitCode = differences.length === 0 ? 0 : 1 diff --git a/mobile/scripts/hosted-android-agent-history-journey.mjs b/mobile/scripts/hosted-android-agent-history-journey.mjs index 63be700e76d..360580e78f7 100644 --- a/mobile/scripts/hosted-android-agent-history-journey.mjs +++ b/mobile/scripts/hosted-android-agent-history-journey.mjs @@ -83,17 +83,6 @@ export async function verifyHostedAndroidAgentHistoryJourney({ timeoutMs }) } - await evidenceStep('reject untrusted synthetic resume', () => - activateHostedWebViewControl(historyDocument, { - kind: 'label', - value: 'Resume agent session' - }) - ) - historyDocument = await waitForVisibleHostedWebView({ - discoveryUrl, - expectedText: 'Failed to resume session.', - timeoutMs - }) const nativeTouchPoint = await evidenceStep('resume Agent History from native touch', () => tapAndroidLabel(emulator, historyDocument, 'Resume agent session') ) @@ -110,7 +99,6 @@ export async function verifyHostedAndroidAgentHistoryJourney({ row: EMULATOR_AGENT_HISTORY_TITLE, preview: EMULATOR_AGENT_HISTORY_PREVIEW_MARKER, resume: { - synthetic: 'gesture rejected', native: 'queued', nativeTouchPoint, resumedRoute: resumedSessionDocument.href diff --git a/mobile/scripts/hosted-ios-agent-history-journey.mjs b/mobile/scripts/hosted-ios-agent-history-journey.mjs index 71a91ff3a45..2063dbba6d0 100644 --- a/mobile/scripts/hosted-ios-agent-history-journey.mjs +++ b/mobile/scripts/hosted-ios-agent-history-journey.mjs @@ -208,17 +208,6 @@ export async function verifyHostedAgentHistoryJourney({ expectedText: EMULATOR_AGENT_HISTORY_TITLE, timeoutMs }) - await evidenceStep('reject untrusted synthetic resume', () => - activateHostedWebViewControl(activeHistoryDocument, { - kind: 'label', - value: 'Resume agent session' - }) - ) - activeHistoryDocument = await waitForVisibleHostedWebView({ - discoveryUrl, - expectedText: 'Failed to resume session.', - timeoutMs - }) const nativeResumePoint = await evidenceStep('resume Agent History from native touch', () => tapHostedIosAccessibilityControl(emulator, 'Resume agent session', timeoutMs) ) @@ -283,7 +272,6 @@ export async function verifyHostedAgentHistoryJourney({ }, reconnect, resume: { - synthetic: 'gesture rejected', native: 'queued', nativeTouchPoint: nativeResumePoint, resumedRoute: resumedSessionDocument.href diff --git a/mobile/scripts/hosted-ios-emulator-accessibility.mjs b/mobile/scripts/hosted-ios-emulator-accessibility.mjs index a94e93763b0..d5e9cf2b266 100644 --- a/mobile/scripts/hosted-ios-emulator-accessibility.mjs +++ b/mobile/scripts/hosted-ios-emulator-accessibility.mjs @@ -278,6 +278,13 @@ export async function waitForHostedIosAccessibilityLabel( throw new Error(`${label} was not accessible in the visible viewport`) } +export async function readHostedIosAccessibilityLabels( + args, + runCommand = runHostedIosEmulatorCommand +) { + return accessibilityLabels(await readAccessibilityNodes(args, runCommand)) +} + export async function rotateHostedIosEmulator( args, orientation, diff --git a/mobile/scripts/hosted-ios-mobile-launcher.mjs b/mobile/scripts/hosted-ios-mobile-launcher.mjs index ba953b56cf1..2e1dec536a6 100644 --- a/mobile/scripts/hosted-ios-mobile-launcher.mjs +++ b/mobile/scripts/hosted-ios-mobile-launcher.mjs @@ -6,6 +6,7 @@ export function startHostedIosMobileLauncher({ deviceUdid, emulatorControlUserDataPath, environment, + metroDirectory, orcaCli, runtimeDirectory, worktree @@ -31,7 +32,10 @@ export function startHostedIosMobileLauncher({ ORCA_E2E_MOBILE_AGENT_HISTORY_FIXTURE: '1', ORCA_E2E_MOBILE_RUN_DIRECTORY: path.join(runtimeDirectory, 'paired-host'), ORCA_E2E_MOBILE_RESTART_HOLD_MS: '2000', - ORCA_E2E_MOBILE_EMULATOR_CONTROL_USER_DATA_PATH: emulatorControlUserDataPath + ORCA_E2E_MOBILE_EMULATOR_CONTROL_USER_DATA_PATH: emulatorControlUserDataPath, + // Why: an old-client A/B serves Metro from another checkout while the paired + // desktop runtime and the registered workspace stay on this one. + ...(metroDirectory ? { ORCA_E2E_MOBILE_METRO_DIR: metroDirectory } : {}) }, stdio: ['ignore', 'pipe', 'pipe'] } diff --git a/mobile/scripts/hosted-ios-native-journey-labels.mjs b/mobile/scripts/hosted-ios-native-journey-labels.mjs new file mode 100644 index 00000000000..0f77d6579bf --- /dev/null +++ b/mobile/scripts/hosted-ios-native-journey-labels.mjs @@ -0,0 +1,203 @@ +import { execFile } from 'node:child_process' +import path from 'node:path' +import { promisify } from 'node:util' +import { EMULATOR_AGENT_HISTORY_TITLE } from './emulator-agent-history-fixture.mjs' +import { dismissEmulatorDeveloperMenuIfPresent } from './emulator-developer-menu-dismissal.mjs' +import { + readHostedIosAccessibilityLabels, + rotateHostedIosEmulator, + tapHostedIosAccessibilityControl, + tapHostedIosAccessibilityControlByLabelPrefix, + tapHostedIosPoint, + typeHostedIosText, + waitForHostedIosAccessibilityControl, + waitForHostedIosAccessibilityControlByLabelPrefix, + waitForHostedIosAccessibilityControlEndingWith, + waitForHostedIosAccessibilityControlMatching, + waitForHostedIosAccessibilityLabel +} from './hosted-ios-emulator-accessibility.mjs' +import { longPressHostedIosAccessibilityControlByLabelPrefix } from './hosted-ios-emulator-long-press.mjs' + +const execFileAsync = promisify(execFile) +const ACCOUNTS_STABLE_TEXT = 'Add or re-authenticate accounts from desktop Settings → Accounts.' +const ACCOUNTS_TOOLBAR_X = 0.8 +const TASKS_TOOLBAR_X = 0.87 +const AGENT_HISTORY_SEARCH_PLACEHOLDER = 'Search sessions, repo:, path:' +const AGENT_HISTORY_SEARCH_QUERY = 'hybrid agent history fixture' +const CHANGED_FILE_LABEL_PREFIX = 'Open changed file ' +const FILES_STABLE_LABEL = 'Open folder Casks' +const PREVIEW_FILE_LABEL = 'Preview file orca.rb' +const PREVIEW_STABLE_LABEL = 'File preview' + +// Why: the six native journeys the hosted e2e baselines drive, captured as ordered +// accessibility labels plus a screenshot so two client builds can be diffed. +export async function captureHostedIosNativeJourneyLabels(args) { + const stops = [] + const capture = async (name) => { + stops.push({ name, ...(await captureStop(args, name)) }) + } + await dismissEmulatorDeveloperMenuIfPresent(args.emulator) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) + await capture('workspaces') + await walkAccounts(args, capture) + await walkTasks(args, capture) + await walkSessionAndFiles(args, capture) + await walkSourceControl(args, capture) + await walkAgentHistory(args, capture) + return stops +} + +async function walkAccounts(args, capture) { + const filterPoint = await waitForHostedIosAccessibilityControl( + args.emulator, + 'Filter', + args.timeoutMs + ) + await tapHostedIosPoint(args.emulator, { x: ACCOUNTS_TOOLBAR_X, y: filterPoint.y }) + const title = await waitForHostedIosAccessibilityControl( + args.emulator, + 'Accounts', + args.timeoutMs + ) + await waitForHostedIosAccessibilityControl(args.emulator, ACCOUNTS_STABLE_TEXT, args.timeoutMs) + await capture('accounts') + await tapHostedIosPoint(args.emulator, { x: 0.075, y: title.y }) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) +} + +async function walkTasks(args, capture) { + const filterPoint = await waitForHostedIosAccessibilityControl( + args.emulator, + 'Filter', + args.timeoutMs + ) + await tapHostedIosPoint(args.emulator, { x: TASKS_TOOLBAR_X, y: filterPoint.y }) + const title = await waitForHostedIosAccessibilityControl(args.emulator, 'Tasks', args.timeoutMs) + await capture('tasks') + await tapHostedIosPoint(args.emulator, { x: 0.075, y: title.y }) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) +} + +async function walkSessionAndFiles(args, capture) { + await tapHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + 'Mobile Emulator', + args.timeoutMs + ) + await capture('session') + await tapHostedIosAccessibilityControl(args.emulator, 'Open file explorer', args.timeoutMs) + await waitForHostedIosAccessibilityControl(args.emulator, FILES_STABLE_LABEL, args.timeoutMs) + await capture('files') + await tapHostedIosAccessibilityControl(args.emulator, FILES_STABLE_LABEL, args.timeoutMs) + await tapHostedIosAccessibilityControl(args.emulator, PREVIEW_FILE_LABEL, args.timeoutMs) + await waitForHostedIosAccessibilityLabel(args.emulator, PREVIEW_STABLE_LABEL, args.timeoutMs) + await capture('file-preview') + await tapHostedIosAccessibilityControl(args.emulator, 'Back to files', args.timeoutMs) + await waitForHostedIosAccessibilityControl(args.emulator, FILES_STABLE_LABEL, args.timeoutMs) + await tapHostedIosAccessibilityControl(args.emulator, 'Back to session', args.timeoutMs) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + 'Mobile Emulator', + args.timeoutMs + ) + await tapHostedIosAccessibilityControl(args.emulator, 'Back to worktrees', args.timeoutMs) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) +} + +async function walkSourceControl(args, capture) { + await longPressHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs, + undefined, + 'Source Control' + ) + await capture('workspace-actions') + await tapHostedIosAccessibilityControl(args.emulator, 'Source Control', args.timeoutMs) + const changedFile = await waitForHostedIosAccessibilityControlMatching( + args.emulator, + (node) => + node.label?.startsWith(CHANGED_FILE_LABEL_PREFIX) || + node.value?.startsWith(CHANGED_FILE_LABEL_PREFIX), + args.timeoutMs + ) + await waitForHostedIosAccessibilityControlEndingWith(args.emulator, ' on branch', args.timeoutMs) + await capture('source-control') + const changedFileLabel = [changedFile.label, changedFile.value].find((value) => + value?.startsWith(CHANGED_FILE_LABEL_PREFIX) + ) + await tapHostedIosAccessibilityControl(args.emulator, changedFileLabel, args.timeoutMs) + await waitForHostedIosAccessibilityControl(args.emulator, 'Open review actions', args.timeoutMs) + await capture('review') + await tapHostedIosAccessibilityControl(args.emulator, 'Back', args.timeoutMs) + await waitForHostedIosAccessibilityControl(args.emulator, 'Source Control', args.timeoutMs) + await tapHostedIosAccessibilityControl(args.emulator, 'Back to session', args.timeoutMs) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) +} + +async function walkAgentHistory(args, capture) { + await rotateHostedIosEmulator(args.emulator, 'portrait') + await tapHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + args.expectedWorkspace, + args.timeoutMs + ) + await tapHostedIosAccessibilityControl(args.emulator, 'More session actions', args.timeoutMs) + await capture('session-actions') + await tapHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + 'Agent History', + args.timeoutMs + ) + await tapHostedIosAccessibilityControl( + args.emulator, + AGENT_HISTORY_SEARCH_PLACEHOLDER, + args.timeoutMs + ) + await typeHostedIosText(args.emulator, AGENT_HISTORY_SEARCH_QUERY) + await waitForHostedIosAccessibilityControlByLabelPrefix( + args.emulator, + EMULATOR_AGENT_HISTORY_TITLE, + args.timeoutMs + ) + await capture('agent-history-search') + await tapHostedIosAccessibilityControl(args.emulator, 'Agent Session History', args.timeoutMs) + await capture('agent-history') +} + +async function captureStop(args, name) { + await delay(600) + const labels = await readHostedIosAccessibilityLabels(args.emulator) + const screenshot = path.join(args.outputDirectory, `${name}.png`) + await execFileAsync('xcrun', ['simctl', 'io', args.deviceUdid, 'screenshot', screenshot]) + return { labels, screenshot } +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/mobile/scripts/run-hosted-ios-native-journey-capture.mjs b/mobile/scripts/run-hosted-ios-native-journey-capture.mjs new file mode 100644 index 00000000000..b1633aa814e --- /dev/null +++ b/mobile/scripts/run-hosted-ios-native-journey-capture.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node + +import { mkdirSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import process from 'node:process' +import { resolveEmulatorOrcaCli } from './emulator-orca-cli-selection.mjs' +import { startHostedIosEmulatorController } from './hosted-ios-emulator-controller.mjs' +import { captureHostedIosNativeJourneyLabels } from './hosted-ios-native-journey-labels.mjs' +import { completeHostedIosNativeOnboarding } from './hosted-ios-native-onboarding.mjs' +import { + startHostedIosMobileLauncher, + waitForHostedIosMobileLauncher +} from './hosted-ios-mobile-launcher.mjs' +import { installHostedIosSimulatorApp } from './hosted-ios-simulator-app-build.mjs' +import { + bootHostedIosSimulator, + resolveHostedIosSimulatorUdid +} from './hosted-ios-simulator-device.mjs' +import { stopHostedChildProcess } from './hosted-child-process-shutdown.mjs' + +const usage = + 'Usage: node scripts/run-hosted-ios-native-journey-capture.mjs --app --out [--metro-dir ] [--device ] [--timeout-ms ]' + +function parseOptions(argv) { + const options = { + appPath: null, + device: 'iPhone 17 Pro', + metroDirectory: null, + outputDirectory: null, + timeoutMs: 180_000 + } + for (let index = 0; index < argv.length; index += 1) { + if (argv[index] === '--app' && argv[index + 1]) { + options.appPath = path.resolve(argv[++index]) + } else if (argv[index] === '--out' && argv[index + 1]) { + options.outputDirectory = path.resolve(argv[++index]) + } else if (argv[index] === '--metro-dir' && argv[index + 1]) { + options.metroDirectory = path.resolve(argv[++index]) + } else if (argv[index] === '--device' && argv[index + 1]) { + options.device = argv[++index] + } else if (argv[index] === '--timeout-ms' && argv[index + 1]) { + options.timeoutMs = Number(argv[++index]) + } else { + throw new Error(`Unknown argument: ${argv[index]}\n${usage}`) + } + } + if (!options.appPath || !options.outputDirectory) { + throw new Error(usage) + } + return options +} + +const worktree = path.resolve(import.meta.dirname, '../..') +const options = parseOptions(process.argv.slice(2)) +const runtimeDirectory = path.join(options.outputDirectory, 'runtime') + +async function main() { + mkdirSync(runtimeDirectory, { recursive: true, mode: 0o700 }) + const deviceUdid = await resolveHostedIosSimulatorUdid(options.device) + await bootHostedIosSimulator(deviceUdid) + await installHostedIosSimulatorApp({ deviceUdid, appPath: options.appPath }) + const orcaSelection = resolveEmulatorOrcaCli({ + explicitCommand: process.env.ORCA_CLI, + managedCommand: process.env.ORCA_CLI_COMMAND, + devRepoRoot: process.env.ORCA_DEV_REPO_ROOT, + worktree, + cwd: worktree + }) + const controller = await startHostedIosEmulatorController({ + orcaCli: orcaSelection.command, + runtimeDirectory, + worktree + }) + const launcher = startHostedIosMobileLauncher({ + deviceUdid, + emulatorControlUserDataPath: controller.userData, + metroDirectory: options.metroDirectory, + orcaCli: orcaSelection.command, + runtimeDirectory, + worktree + }) + try { + await waitForHostedIosMobileLauncher(launcher, options.timeoutMs) + const emulator = { + deviceUdid, + orcaCli: orcaSelection.command, + userDataDir: controller.userData, + worktree + } + const expectedWorkspace = path.basename(worktree) + const onboarding = await completeHostedIosNativeOnboarding( + emulator, + expectedWorkspace, + options.timeoutMs + ) + const stops = await captureHostedIosNativeJourneyLabels({ + deviceUdid, + emulator, + expectedWorkspace, + outputDirectory: options.outputDirectory, + timeoutMs: options.timeoutMs + }) + const evidence = { + appPath: options.appPath, + expectedWorkspace, + metroDirectory: options.metroDirectory ?? path.join(worktree, 'mobile'), + onboarding, + stops + } + const evidencePath = path.join(options.outputDirectory, 'native-journey-capture.json') + writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`) + console.log(evidencePath) + } finally { + await stopHostedChildProcess(launcher).catch(() => undefined) + await controller.stop().catch(() => undefined) + } +} + +await main() diff --git a/mobile/scripts/start-emulator.mjs b/mobile/scripts/start-emulator.mjs index 58df550e2e8..9863d432ace 100755 --- a/mobile/scripts/start-emulator.mjs +++ b/mobile/scripts/start-emulator.mjs @@ -167,6 +167,9 @@ async function getWorktree() { } function getMobileDir(worktree) { + if (process.env.ORCA_E2E_MOBILE_METRO_DIR) { + return path.resolve(process.env.ORCA_E2E_MOBILE_METRO_DIR) + } return !options.worktree && path.basename(process.cwd()) === 'mobile' ? process.cwd() : path.join(worktree, 'mobile') diff --git a/mobile/src/mobile-web/hosted-android-agent-history-journey.test.ts b/mobile/src/mobile-web/hosted-android-agent-history-journey.test.ts index 0a1bf4bda71..275604c6a92 100644 --- a/mobile/src/mobile-web/hosted-android-agent-history-journey.test.ts +++ b/mobile/src/mobile-web/hosted-android-agent-history-journey.test.ts @@ -58,7 +58,7 @@ describe('hosted Android Agent History journey', () => { })) }) - it('covers the existing scopes, preview, search, trusted resume, and Back flow', async () => { + it('covers the existing scopes, preview, search, native resume, and Back flow', async () => { const sessionDocument = { href: 'orca-mobile-web://build/h/host/session/workspace' } @@ -78,7 +78,6 @@ describe('hosted Android Agent History journey', () => { row: 'Hybrid Agent History Fixture', preview: 'E2E_AGENT_HISTORY_PREVIEW_MARKER', resume: { - synthetic: 'gesture rejected', native: 'queued', nativeTouchPoint: { x: 0.4, y: 0.3 }, resumedRoute: 'orca-mobile-web://build/h/host/session/workspace' From e60475d3a09de900d07ebcc718b2f58abf6423aa Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 2 Sep 2026 00:44:28 -0400 Subject: [PATCH 3/5] test(mobile-e2e): pin the settled-label argument in the parity call census Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .../hosted-ios-source-control-review-parity.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile/src/mobile-web/hosted-ios-source-control-review-parity.test.ts b/mobile/src/mobile-web/hosted-ios-source-control-review-parity.test.ts index 877ff6b8d07..3d587b1a89f 100644 --- a/mobile/src/mobile-web/hosted-ios-source-control-review-parity.test.ts +++ b/mobile/src/mobile-web/hosted-ios-source-control-review-parity.test.ts @@ -81,7 +81,9 @@ describe('hosted iOS Source Control and Review parity', () => { expect(mocks.longPressControlByPrefix).toHaveBeenCalledWith( { deviceUdid: 'simulator' }, 'mobile-rearch', - 30_000 + 30_000, + undefined, + 'Source Control' ) expect(mocks.tapControl).toHaveBeenCalledWith( { deviceUdid: 'simulator' }, From 909edbb68ad4862f8746877f748f118be0161c6a Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 2 Sep 2026 01:15:12 -0400 Subject: [PATCH 4/5] fix(mobile-e2e): read the native landmark after the route settles The host-origin Source Control route animates in, so captureNativeRoute read its landmark before the 500ms settle and screenshotted after. The parity check then failed on landmarkDelta.y 0.0475 against a 0.005 budget while the pixels agreed at 0.0121 against 0.03. Measured on a live hybrid session: the DOM point and the accessibility point agree to 0.0000064, so the coordinate spaces were never the problem. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- mobile/scripts/hosted-ios-source-control-review-parity.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mobile/scripts/hosted-ios-source-control-review-parity.mjs b/mobile/scripts/hosted-ios-source-control-review-parity.mjs index 6d6c367d3c2..50e6da7b72b 100644 --- a/mobile/scripts/hosted-ios-source-control-review-parity.mjs +++ b/mobile/scripts/hosted-ios-source-control-review-parity.mjs @@ -149,8 +149,11 @@ async function captureNativeRoute({ title, timeoutMs }) { - const screenTitlePoint = await waitForHostedIosAccessibilityControl(emulator, title, timeoutMs) + await waitForHostedIosAccessibilityControl(emulator, title, timeoutMs) await delay(500) + // Why: the host-origin route animates in, so the first frame's landmark is a + // position the screenshot below never shows. Re-read it once the screen settles. + const screenTitlePoint = await waitForHostedIosAccessibilityControl(emulator, title, timeoutMs) const screenshot = path.join(runtimeDirectory, screenshotName) await captureSimulatorScreenshot(deviceUdid, screenshot) return { screenTitlePoint, screenshot } From 4dc2229283ac2b4d099c045715468ae40948dadb Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 2 Sep 2026 01:35:30 -0400 Subject: [PATCH 5/5] test(mobile-e2e): capture a native journey stop only once its tree repeats A single accessibility read caught the Tasks list mid-load on one client and a second, transient xterm helper node on the other, and both read as client differences. Poll until two consecutive volatile-free reads match, then screenshot. Re-running the A/B with this in place left the twelve stops identical except for live GitHub issue content. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .../hosted-ios-native-journey-labels.mjs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/mobile/scripts/hosted-ios-native-journey-labels.mjs b/mobile/scripts/hosted-ios-native-journey-labels.mjs index 0f77d6579bf..c22660afa0f 100644 --- a/mobile/scripts/hosted-ios-native-journey-labels.mjs +++ b/mobile/scripts/hosted-ios-native-journey-labels.mjs @@ -28,6 +28,10 @@ const CHANGED_FILE_LABEL_PREFIX = 'Open changed file ' const FILES_STABLE_LABEL = 'Open folder Casks' const PREVIEW_FILE_LABEL = 'Preview file orca.rb' const PREVIEW_STABLE_LABEL = 'File preview' +const SETTLE_INTERVAL_MS = 900 +const SETTLE_TIMEOUT_MS = 20_000 +const VOLATILE_LABEL = + /^(\d{1,2}:\d{2}\s?(AM|PM)|Dynamic Island.*|Cellular|SSID.*|\d+% battery power|Not charging|Charging|No signal|\d of \d Wi-Fi bars)$/ // Why: the six native journeys the hosted e2e baselines drive, captured as ordered // accessibility labels plus a screenshot so two client builds can be diffed. @@ -191,13 +195,34 @@ async function walkAgentHistory(args, capture) { } async function captureStop(args, name) { - await delay(600) - const labels = await readHostedIosAccessibilityLabels(args.emulator) + const labels = await readSettledAccessibilityLabels(args) const screenshot = path.join(args.outputDirectory, `${name}.png`) await execFileAsync('xcrun', ['simctl', 'io', args.deviceUdid, 'screenshot', screenshot]) return { labels, screenshot } } +// Why: a screen still loading its rows, or a WebView still swapping its +// accessibility subtree, reads as a client difference. Only a tree that repeats +// itself is evidence. +async function readSettledAccessibilityLabels(args) { + const deadline = Date.now() + Math.min(args.timeoutMs, SETTLE_TIMEOUT_MS) + let previous = null + while (Date.now() < deadline) { + await delay(SETTLE_INTERVAL_MS) + const labels = await readHostedIosAccessibilityLabels(args.emulator) + const comparable = volatileFreeLabels(labels).join('\u0000') + if (previous !== null && previous === comparable) { + return labels + } + previous = comparable + } + return readHostedIosAccessibilityLabels(args.emulator) +} + +function volatileFreeLabels(labels) { + return labels.filter((label) => !VOLATILE_LABEL.test(label)) +} + function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)) }