diff --git a/src/cli/specs/emulator.ts b/src/cli/specs/emulator.ts index ef83090a8e2..69b3a6412bb 100644 --- a/src/cli/specs/emulator.ts +++ b/src/cli/specs/emulator.ts @@ -108,7 +108,7 @@ export const EMULATOR_COMMAND_SPECS: CommandSpec[] = [ }, { path: ['emulator', 'ax'], - summary: 'Dump the Android accessibility (uiautomator) tree', + summary: 'Dump the target emulator accessibility tree', usage: 'orca emulator ax [--device ] [--worktree ] [--json]', allowedFlags: [...GLOBAL_FLAGS, 'device', 'emulator', 'worktree'] }, diff --git a/src/main/emulator/backends/emulator-backend.ts b/src/main/emulator/backends/emulator-backend.ts index 9e5ccf39df0..6186a493498 100644 --- a/src/main/emulator/backends/emulator-backend.ts +++ b/src/main/emulator/backends/emulator-backend.ts @@ -80,7 +80,7 @@ export type EmulatorBackend = { rotate(deviceId: string, orientation: string): Promise exec(deviceId: string, command: string): Promise - // Capability-gated verbs (Android today). The router checks `capabilities` + // Capability-gated verbs. The router checks `capabilities` // before calling these and rejects unsupported backends with emulator_unsupported. installApp?(deviceId: string, apkPath: string, options?: { reinstall?: boolean }): Promise launchApp?(deviceId: string, packageName: string, activity?: string): Promise @@ -90,7 +90,7 @@ export type EmulatorBackend = { packageName: string, permission?: string ): Promise - accessibilityTree?(deviceId: string): Promise + accessibilityTree?(deviceId: string, axUrl?: string): Promise logcat?( deviceId: string, options?: { lines?: number; filters?: readonly string[] } diff --git a/src/main/emulator/backends/ios-emulator-backend.test.ts b/src/main/emulator/backends/ios-emulator-backend.test.ts index fd7df588f19..c2fba2fa409 100644 --- a/src/main/emulator/backends/ios-emulator-backend.test.ts +++ b/src/main/emulator/backends/ios-emulator-backend.test.ts @@ -11,7 +11,8 @@ const { listServeSimHelperProcessesForDeviceMock, shutdownSimulatorDeviceMock, sendEmulatorGestureSequenceMock, - parseServeSimDetachedSessionMock + parseServeSimDetachedSessionMock, + netFetchMock } = vi.hoisted(() => ({ ensureSimulatorBootedMock: vi.fn(async () => {}), execServeSimCommandMock: vi.fn(async (_executable?: unknown, _args?: string[]) => ({})), @@ -21,9 +22,12 @@ const { listServeSimHelperProcessesForDeviceMock: vi.fn(async (): Promise => []), shutdownSimulatorDeviceMock: vi.fn(async () => {}), sendEmulatorGestureSequenceMock: vi.fn(async () => {}), - parseServeSimDetachedSessionMock: vi.fn() + parseServeSimDetachedSessionMock: vi.fn(), + netFetchMock: vi.fn() })) +vi.mock('electron', () => ({ net: { fetch: netFetchMock } })) + vi.mock('../serve-sim-execution', () => ({ execServeSimCommand: execServeSimCommandMock, parseServeSimCommandArgs: vi.fn((input: string) => input.split(' ').filter(Boolean)), @@ -81,9 +85,10 @@ describe('IosEmulatorBackend', () => { sendEmulatorGestureSequenceMock.mockReset() sendEmulatorGestureSequenceMock.mockImplementation(async () => {}) parseServeSimDetachedSessionMock.mockReset() + netFetchMock.mockReset() }) - it('declares ios kind, mjpeg codec, and no explicit-verb capabilities', () => { + it('advertises the iOS accessibility tree capability', () => { const backend = new IosEmulatorBackend() expect(backend.kind).toBe('ios') expect(backend.streamCodec).toBe('mjpeg') @@ -91,11 +96,37 @@ describe('IosEmulatorBackend', () => { install: false, launch: false, permissions: false, - accessibilityTree: false, + accessibilityTree: true, logcat: false }) }) + it('returns the raw serve-sim accessibility tree', async () => { + const tree = [{ type: 'Application', children: [{ AXLabel: 'Continue' }] }] + netFetchMock.mockResolvedValue(new Response(JSON.stringify(tree), { status: 200 })) + const backend = new IosEmulatorBackend() + + await expect( + backend.accessibilityTree('device-1', 'http://127.0.0.1:3100/ax') + ).resolves.toEqual(tree) + expect(netFetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:3100/ax', + expect.objectContaining({ signal: expect.any(AbortSignal) }) + ) + }) + + it('reports missing sessions and temporarily unavailable AX endpoints', async () => { + const backend = new IosEmulatorBackend() + await expect(backend.accessibilityTree('device-1')).rejects.toMatchObject({ + code: 'emulator_no_active' + }) + + netFetchMock.mockResolvedValue(new Response('{"error":"ax_unavailable"}', { status: 503 })) + await expect( + backend.accessibilityTree('device-1', 'http://127.0.0.1:3100/ax') + ).rejects.toMatchObject({ code: 'emulator_helper_failed' }) + }) + it('taps via serve-sim with the resolved device', async () => { const backend = new IosEmulatorBackend() await backend.tap('iPhone 16 Pro', 0.5, 0.7) diff --git a/src/main/emulator/backends/ios-emulator-backend.ts b/src/main/emulator/backends/ios-emulator-backend.ts index a0828c75898..0cfe9a39f32 100644 --- a/src/main/emulator/backends/ios-emulator-backend.ts +++ b/src/main/emulator/backends/ios-emulator-backend.ts @@ -23,6 +23,7 @@ import { import type { EmulatorBridgeOptions } from '../emulator-bridge-types' import { sendEmulatorGestureSequence, type EmulatorGesturePoint } from '../emulator-gesture-sender' import { parseServeSimDetachedSession } from '../serve-sim-detached-session' +import { requestServeSimAccessibilityTree } from '../serve-sim-accessibility-tree' import { hideNativeSimulatorApp } from '../simulator-app-visibility' import type { BackendAvailability, @@ -37,12 +38,11 @@ import type { export class IosEmulatorBackend implements EmulatorBackend { readonly kind = 'ios' as const readonly streamCodec = 'mjpeg' as const - // iOS exposes ax/permissions/etc. via `exec`; explicit verbs are Android-only for v1. readonly capabilities: EmulatorBackendCapabilities = { install: false, launch: false, permissions: false, - accessibilityTree: false, + accessibilityTree: true, logcat: false } @@ -176,6 +176,16 @@ export class IosEmulatorBackend implements EmulatorBackend { return this.execServeSim([...rawArgs, '-d', udid], { json: true }) } + async accessibilityTree(_deviceId: string, axUrl?: string): Promise { + if (!axUrl) { + throw new EmulatorError( + 'emulator_no_active', + 'No active iOS emulator AX endpoint — attach the simulator first.' + ) + } + return requestServeSimAccessibilityTree(axUrl) + } + async startSession(deviceId: string): Promise { const udid = await this.resolveDeviceId(deviceId) await ensureSimulatorBooted(udid) diff --git a/src/main/emulator/emulator-bridge.test.ts b/src/main/emulator/emulator-bridge.test.ts index 662df30f800..3cb355bf9ee 100644 --- a/src/main/emulator/emulator-bridge.test.ts +++ b/src/main/emulator/emulator-bridge.test.ts @@ -9,16 +9,20 @@ const { killServeSimHelperProcessesForDeviceMock, listSimulatorDevicesMock, listServeSimHelperProcessesForDeviceMock, - shutdownSimulatorDeviceMock + shutdownSimulatorDeviceMock, + netFetchMock } = vi.hoisted(() => ({ execServeSimCommandMock: vi.fn(async () => ({})), hideNativeSimulatorAppMock: vi.fn(async () => {}), killServeSimHelperProcessesForDeviceMock: vi.fn(async () => {}), listSimulatorDevicesMock: vi.fn(async (): Promise => []), listServeSimHelperProcessesForDeviceMock: vi.fn(async (): Promise => []), - shutdownSimulatorDeviceMock: vi.fn(async () => {}) + shutdownSimulatorDeviceMock: vi.fn(async () => {}), + netFetchMock: vi.fn() })) +vi.mock('electron', () => ({ net: { fetch: netFetchMock } })) + vi.mock('./serve-sim-execution', () => ({ execServeSimCommand: execServeSimCommandMock, parseServeSimCommandArgs: vi.fn(() => []), @@ -62,6 +66,7 @@ function session(deviceUdid: string): EmulatorSessionInfo { deviceUdid, streamUrl: `http://127.0.0.1:3100/${deviceUdid}`, wsUrl: `ws://127.0.0.1:3100/${deviceUdid}`, + axUrl: `http://127.0.0.1:3100/${deviceUdid}/ax`, helperPid: 1234, // iOS serve-sim sessions round-trip through the registry as mjpeg. streamCodec: 'mjpeg' @@ -84,6 +89,7 @@ describe('EmulatorBridge helper ownership', () => { hideNativeSimulatorAppMock.mockImplementation(async () => {}) shutdownSimulatorDeviceMock.mockReset() shutdownSimulatorDeviceMock.mockImplementation(async () => {}) + netFetchMock.mockReset() }) it('stops the previous Orca-managed helper when a worktree switches devices', async () => { @@ -352,6 +358,59 @@ describe('RuntimeEmulatorCommands attach lifecycle', () => { hideNativeSimulatorAppMock.mockImplementation(async () => {}) shutdownSimulatorDeviceMock.mockReset() shutdownSimulatorDeviceMock.mockImplementation(async () => {}) + netFetchMock.mockReset() + }) + + it('reads iOS accessibility from the active worktree session', async () => { + const tree = [{ type: 'Application', children: [] }] + netFetchMock.mockResolvedValue(new Response(JSON.stringify(tree), { status: 200 })) + const bridge = new EmulatorBridge() + bridge.registerActiveEmulator('wt-1', session('device-1'), { managed: true }) + const commands = new RuntimeEmulatorCommands({ + getEmulatorBridge: () => bridge, + resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })), + getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never, + getSettings: () => ({ + mobileEmulatorEnabled: true, + mobileEmulatorDefaultDeviceUdid: null + }) + }) + + await expect(commands.emulatorAx({ worktree: 'wt-1' })).resolves.toEqual(tree) + expect(netFetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:3100/device-1/ax', + expect.any(Object) + ) + }) + + it('reports when the requested iOS device differs from the active session', async () => { + listSimulatorDevicesMock.mockResolvedValue([ + { + name: 'iPhone requested', + udid: 'device-requested', + state: 'Booted', + runtime: 'iOS 26.0' + } + ]) + const bridge = new EmulatorBridge() + bridge.registerActiveEmulator('wt-1', session('device-active'), { managed: true }) + const commands = new RuntimeEmulatorCommands({ + getEmulatorBridge: () => bridge, + resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })), + getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never, + getSettings: () => ({ + mobileEmulatorEnabled: true, + mobileEmulatorDefaultDeviceUdid: null + }) + }) + + await expect( + commands.emulatorAx({ device: 'device-requested', worktree: 'wt-1' }) + ).rejects.toMatchObject({ + code: 'emulator_no_active', + message: expect.stringContaining('active: device-active') + }) + expect(netFetchMock).not.toHaveBeenCalled() }) it('reconnects to an existing active helper instead of replacing it', async () => { diff --git a/src/main/emulator/emulator-bridge.ts b/src/main/emulator/emulator-bridge.ts index 08ada439ab8..e1c1f36a299 100644 --- a/src/main/emulator/emulator-bridge.ts +++ b/src/main/emulator/emulator-bridge.ts @@ -199,6 +199,23 @@ export class EmulatorBridge { return backend.exec(device, command) } + async accessibilityTree(opts?: EmulatorTargetOpts): Promise { + return this.runCapability('accessibilityTree', opts, async (backend, device) => { + if (backend.kind !== 'ios') { + return backend.accessibilityTree!(device) + } + const udid = await backend.resolveDeviceId(device) + const active = this.getActiveForWorktree(opts?.worktreeId) + if (active && active.deviceUdid !== udid) { + throw new EmulatorError( + 'emulator_no_active', + `iOS simulator ${udid} is not active for this worktree (active: ${active.deviceUdid}); attach the requested simulator first.` + ) + } + return backend.accessibilityTree!(udid, active?.axUrl) + }) + } + // Runs a capability-gated verb against the resolved target, rejecting backends // that do not advertise the capability (e.g. install/logcat on iOS). async runCapability( diff --git a/src/main/emulator/serve-sim-accessibility-tree.ts b/src/main/emulator/serve-sim-accessibility-tree.ts new file mode 100644 index 00000000000..6e2f36613cf --- /dev/null +++ b/src/main/emulator/serve-sim-accessibility-tree.ts @@ -0,0 +1,47 @@ +import { net } from 'electron' +import { EmulatorError } from './emulator-errors' + +const AX_REQUEST_TIMEOUT_MS = 5_000 +const MAX_ERROR_BODY_LENGTH = 512 + +export async function requestServeSimAccessibilityTree(axUrl: string): Promise { + try { + const response = await net.fetch(axUrl, { + signal: AbortSignal.timeout(AX_REQUEST_TIMEOUT_MS) + }) + const body = await response.text() + if (!response.ok) { + const detail = body.slice(0, MAX_ERROR_BODY_LENGTH) || response.statusText + const retry = response.status === 503 ? ' Accessibility may still be warming up; retry.' : '' + throw new EmulatorError( + 'emulator_helper_failed', + `serve-sim AX request failed (${response.status}): ${detail}.${retry}` + ) + } + + let tree: unknown + try { + tree = JSON.parse(body) + } catch { + throw new EmulatorError('emulator_error', 'serve-sim AX returned invalid JSON.') + } + if ( + !Array.isArray(tree) || + tree.some((node) => typeof node !== 'object' || node === null || Array.isArray(node)) + ) { + throw new EmulatorError('emulator_error', 'serve-sim AX returned an invalid tree.') + } + return tree + } catch (error) { + if (error instanceof EmulatorError) { + throw error + } + const detail = + error instanceof Error && error.name === 'TimeoutError' + ? 'request timed out' + : error instanceof Error + ? error.message + : 'unknown request failure' + throw new EmulatorError('emulator_helper_failed', `Unable to read serve-sim AX: ${detail}`) + } +} diff --git a/src/main/emulator/serve-sim-detached-session.test.ts b/src/main/emulator/serve-sim-detached-session.test.ts index d3997d9848d..6b62d60cf1e 100644 --- a/src/main/emulator/serve-sim-detached-session.test.ts +++ b/src/main/emulator/serve-sim-detached-session.test.ts @@ -15,10 +15,32 @@ describe('parseServeSimDetachedSession', () => { expect(info).toMatchObject({ deviceUdid: 'device-1', streamUrl: 'http://127.0.0.1:3100/stream.mjpeg', - wsUrl: 'ws://127.0.0.1:3100/ws' + wsUrl: 'ws://127.0.0.1:3100/ws', + axUrl: 'http://127.0.0.1:3100/ax' }) }) + it('derives the device-scoped AX endpoint and preserves an explicit one', () => { + const derived = parseServeSimDetachedSession( + { + streamUrl: 'http://127.0.0.1:3200/helper/device-1/stream.mjpeg', + wsUrl: 'ws://127.0.0.1:3200/helper/device-1/ws' + }, + 'device-1' + ) + const explicit = parseServeSimDetachedSession( + { + streamUrl: 'http://127.0.0.1:3200/stream.mjpeg', + wsUrl: 'ws://127.0.0.1:3200/ws', + axUrl: 'http://127.0.0.1:3200/custom-ax' + }, + 'device-1' + ) + + expect(derived.axUrl).toBe('http://127.0.0.1:3200/helper/device-1/ax') + expect(explicit.axUrl).toBe('http://127.0.0.1:3200/custom-ax') + }) + it('derives the MJPEG stream endpoint from older serve-sim url output', () => { const info = parseServeSimDetachedSession( { diff --git a/src/main/emulator/serve-sim-detached-session.ts b/src/main/emulator/serve-sim-detached-session.ts index 40847d92923..6a67e481737 100644 --- a/src/main/emulator/serve-sim-detached-session.ts +++ b/src/main/emulator/serve-sim-detached-session.ts @@ -8,6 +8,17 @@ function streamUrlFromServeSimUrl(url: string): string { return url.endsWith('/stream.mjpeg') ? url : `${url.replace(/\/$/, '')}/stream.mjpeg` } +function axUrlFromStreamUrl(streamUrl: string | undefined): string | undefined { + if (!streamUrl) { + return undefined + } + try { + return new URL('ax', streamUrl).toString() + } catch { + return undefined + } +} + export function parseServeSimDetachedSession(raw: unknown, udid: string): EmulatorSessionInfo { if (!raw || typeof raw !== 'object') { throw new EmulatorError('emulator_helper_failed', 'serve-sim did not return stream endpoints.') @@ -24,7 +35,7 @@ export function parseServeSimDetachedSession(raw: unknown, udid: string): Emulat deviceUdid: typeof json.device === 'string' ? json.device : udid, wsUrl: wsUrl ?? '', streamUrl: streamUrl ?? '', - axUrl: typeof json.axUrl === 'string' ? json.axUrl : undefined + axUrl: typeof json.axUrl === 'string' ? json.axUrl : axUrlFromStreamUrl(streamUrl) } if (!info.streamUrl || !info.wsUrl) { throw new EmulatorError('emulator_helper_failed', 'serve-sim did not return stream endpoints.') diff --git a/src/main/runtime/orca-runtime-emulator.ts b/src/main/runtime/orca-runtime-emulator.ts index 9bda55ca899..b1bcbe5c689 100644 --- a/src/main/runtime/orca-runtime-emulator.ts +++ b/src/main/runtime/orca-runtime-emulator.ts @@ -249,11 +249,10 @@ export class RuntimeEmulatorCommands { async emulatorAx(params: EmulatorTargetParams): Promise { const worktreeId = await this.resolveWorktreeId(params.worktree) - return this.requireEmulatorBridge().runCapability( - 'accessibilityTree', - { device: params.device ?? params.emulator, worktreeId }, - (backend, device) => backend.accessibilityTree!(device) - ) + return this.requireEmulatorBridge().accessibilityTree({ + device: params.device ?? params.emulator, + worktreeId + }) } async emulatorLogcat(