Files
orca/src/main/runtime/orca-runtime-emulator.ts
T
Jinjingand5Hyeons d1265bfb3b fix(emulator): iOS ax via plain-JSON serve-sim helper (supersedes #10007) (#10029)
* Revert "Enable accessibility tree (`ax`) command on iOS emulator sessions (#10007)"

This reverts commit 43ae014a64.

* fix(emulator): expose iOS accessibility tree

* fix(emulator): support device-only iOS AX

* fix(emulator): normalize iOS ax to 0..1 and heal missing axUrl

serve-sim's helper /ax reports element frames in absolute pixels, but
tap/gesture take normalized 0..1 coords. Normalize the raw AX node tree
into a compact nested shape whose frames are 0..1 over the device screen
(first root's frame), mirroring serve-sim's own normalizeAxTree, so agents
can feed ax output straight back into input commands.

Also heal sessions that were registered without an axUrl: #9924 only
derived /ax at parse time, so already-active sessions had no endpoint.
The bridge now derives it from the session's mjpeg stream URL, guarded to
the /stream.mjpeg suffix so a non-mjpeg URL never fabricates a bogus /ax.

* docs(emulator): mark ax working on iOS with correct raw-AX-tree shape

Both skill guides and the CLI summary described iOS ax as unsupported (or,
via the reverted #10007, as a normalized "screen + elements" shape that
never matched the endpoint). ax works on both backends: Android via
uiautomator, iOS via the serve-sim helper. Document the real iOS output —
a raw AX node tree (labels, roles, nested children) with frames normalized
to 0..1 — and regenerate the bundled skill guides.

* chore(skills): regenerate skill bundle manifests

CI verify failed because generated skill artifacts were stale after version/skill revision bumps.

* fix(emulator): read ax from explicit device without active session

Fall back to udid-keyed session lookup when a worktree has no active emulator,
allowing `--device` targeting to work the same way for ax as it does for tap/type.
Also clarify in docs that AX frames are normalized 0..1 with top-left origin,
and show how to tap an element at its frame center (x+width/2, y+height/2).

* fix(emulator): cap iOS AX tree at 500 nodes

Unbounded accessibility trees can flood agent output. Enforce a 500-node limit (matching serve-sim's snapshot cap) and mark truncated parents so consumers know the tree was cut.

---------

Co-authored-by: 5Hyeons <ohs2251@naver.com>
2026-07-22 22:21:51 -07:00

342 lines
13 KiB
TypeScript

import type { BrowserWindow } from 'electron'
import type { EmulatorBridge } from '../emulator/emulator-bridge'
import { EmulatorError } from '../emulator/emulator-errors'
import {
inspectEmulatorAvailability,
type EmulatorAvailability
} from '../emulator/emulator-availability'
import { resolveDefaultAttachDevice } from '../emulator/emulator-default-attach-device'
import { setConfiguredAndroidSdkPath } from '../emulator/android/android-sdk-host-discovery'
import type { EmulatorGesturePoint } from '../emulator/emulator-gesture-sender'
import type { EmulatorSessionInfo } from '../emulator/emulator-types'
import type { SimulatorDevice } from '../emulator/simctl-simulator-devices'
import type { EmulatorDevice } from '../emulator/backends/emulator-backend'
import type { GlobalSettings } from '../../shared/types'
// Settings slice the emulator surface needs; keeps the host contract honest (no widening cast).
type EmulatorHostSettings = Pick<
GlobalSettings,
'mobileEmulatorEnabled' | 'mobileEmulatorDefaultDeviceUdid' | 'androidSdkPath'
>
// Why: dedicated file for "one surface" separation (emulator), parallel to orca-runtime-browser.ts. Keeps OrcaRuntimeService focused; emulator routing easy to scan. No max-lines disable (split further if grows; per AGENTS + plan Phase 3).
export type RuntimeEmulatorCommandHost = {
getEmulatorBridge(): EmulatorBridge | null
resolveWorktreeSelector(selector: string): Promise<{ id: string }>
getAuthoritativeWindow(): BrowserWindow
getSettings(): EmulatorHostSettings
}
type EmulatorTargetParams = { device?: string; emulator?: string; worktree?: string }
export class RuntimeEmulatorCommands {
constructor(private readonly host: RuntimeEmulatorCommandHost) {}
private requireEmulatorBridge(): EmulatorBridge {
const bridge = this.host.getEmulatorBridge()
if (!bridge) {
throw new EmulatorError('emulator_no_active', 'No emulator session is active')
}
// Honor the user's configured Android SDK path before the backend resolves it.
setConfiguredAndroidSdkPath(this.host.getSettings().androidSdkPath ?? null)
return bridge
}
// Why: RPC envelopes require a serializable `result` field; void/undefined omits it and breaks CLI schema validation.
private static readonly OK = { ok: true as const }
// High-level delegation (mirror browser* methods).
async emulatorTap(
params: EmulatorTargetParams & { x: number; y: number }
): Promise<{ ok: true }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
await bridge.tap(params.x, params.y, { device: params.device ?? params.emulator, worktreeId })
return RuntimeEmulatorCommands.OK
}
async emulatorGesture(params: {
points: EmulatorGesturePoint[]
device?: string
emulator?: string
worktree?: string
}): Promise<{ ok: true }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
await bridge.gesture(params.points, { device: params.device ?? params.emulator, worktreeId })
return RuntimeEmulatorCommands.OK
}
async emulatorType(params: {
text: string
device?: string
emulator?: string
worktree?: string
}): Promise<{ ok: true }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
await bridge.type(params.text, { device: params.device ?? params.emulator, worktreeId })
return RuntimeEmulatorCommands.OK
}
async emulatorButton(params: {
name: string
device?: string
emulator?: string
worktree?: string
}): Promise<{ ok: true }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
await bridge.button(params.name, { device: params.device ?? params.emulator, worktreeId })
return RuntimeEmulatorCommands.OK
}
async emulatorRotate(params: {
orientation: string
device?: string
emulator?: string
worktree?: string
}): Promise<{ ok: true }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
await bridge.rotate(params.orientation, {
device: params.device ?? params.emulator,
worktreeId
})
return RuntimeEmulatorCommands.OK
}
async emulatorExec(params: {
command: string
device?: string
emulator?: string
worktree?: string
}): Promise<unknown> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
return bridge.exec(params.command, {
device: params.device,
emulator: params.emulator,
worktreeId
})
}
async emulatorAttach(params: {
device?: string
worktree?: string
focus?: boolean
}): Promise<{ attached: boolean; info?: EmulatorSessionInfo }> {
const settings = this.host.getSettings()
if (settings.mobileEmulatorEnabled === false) {
throw new EmulatorError('emulator_disabled', 'Mobile Emulator is disabled in Settings.')
}
const bridge = this.requireEmulatorBridge()
let device = params.device ?? settings.mobileEmulatorDefaultDeviceUdid ?? undefined
if (!device) {
device = await resolveDefaultAttachDevice(bridge)
}
if (!device) {
throw new EmulatorError(
'emulator_device_not_found',
'No emulator device specified. Choose a default device in Settings > Mobile Emulator or pass a device.'
)
}
const worktreeId = await this.resolveWorktreeId(params.worktree)
if (worktreeId) {
const reusable = await bridge.getReusableActiveForWorktree(worktreeId, device)
if (reusable) {
// Why: renderer remounts should reconnect to the existing stream, not
// kill it and create the stream-disconnected reload loop users see.
this.notifyRendererEmulatorAutoAttach(worktreeId, reusable)
if (params.focus) {
this.notifyRendererEmulatorPaneFocus(worktreeId)
}
return { attached: true, info: reusable }
}
// A different requested device is an explicit switch; the bridge keeps a
// slow-to-boot Android emulator alive for instant switch-back.
await bridge.stopActiveForSwitch(worktreeId)
}
const info = await bridge.startHelperForDevice(device)
if (worktreeId) {
bridge.registerActiveEmulator(worktreeId, info, { managed: true })
this.notifyRendererEmulatorAutoAttach(worktreeId, info)
if (params.focus) {
this.notifyRendererEmulatorPaneFocus(worktreeId)
}
}
// Default: no auto steal (mirror browser tab create/switch). --focus sends emulator:pane-focus only when requested.
return { attached: true, info }
}
async emulatorList(_params: { worktree?: string } = {}): Promise<unknown> {
const bridge = this.requireEmulatorBridge()
return bridge.listRunningHelpers()
}
async emulatorUnregisterActive(params: { worktree?: string }): Promise<{ ok: true }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
if (worktreeId) {
bridge.unregisterActiveEmulator(worktreeId)
}
return RuntimeEmulatorCommands.OK
}
async emulatorListSimulators(_params: { worktree?: string } = {}): Promise<SimulatorDevice[]> {
// Why: exposed for the EmulatorPane auto-flow on "New Mobile Emulator" tab creation.
// Returns the full simctl list (including Shutdown devices) so the pane can choose a default and
// rely on startHelperForDevice + ensureDeviceBooted to boot if needed. Worktree param ignored
// (simulators are host-local, not per-worktree).
const bridge = this.requireEmulatorBridge()
return bridge.listSimulators()
}
async emulatorAvailability(_params: { worktree?: string } = {}): Promise<EmulatorAvailability> {
return inspectEmulatorAvailability(this.requireEmulatorBridge())
}
// Why: unified device inventory across backends (iOS simulators + Android
// devices/AVDs) for the cross-platform `orca emulator devices` command.
async emulatorListDevices(_params: { worktree?: string } = {}): Promise<EmulatorDevice[]> {
return this.requireEmulatorBridge().listAllDevices()
}
private async resolveWorktreeId(worktree?: string): Promise<string | undefined> {
return worktree ? (await this.host.resolveWorktreeSelector(worktree)).id : undefined
}
async emulatorInstall(
params: EmulatorTargetParams & { path: string; reinstall?: boolean }
): Promise<{ ok: true }> {
const worktreeId = await this.resolveWorktreeId(params.worktree)
await this.requireEmulatorBridge().runCapability(
'install',
{ device: params.device ?? params.emulator, worktreeId },
(backend, device) => backend.installApp!(device, params.path, { reinstall: params.reinstall })
)
return RuntimeEmulatorCommands.OK
}
async emulatorLaunch(
params: EmulatorTargetParams & { package: string; activity?: string }
): Promise<{ ok: true }> {
const worktreeId = await this.resolveWorktreeId(params.worktree)
await this.requireEmulatorBridge().runCapability(
'launch',
{ device: params.device ?? params.emulator, worktreeId },
(backend, device) => backend.launchApp!(device, params.package, params.activity)
)
return RuntimeEmulatorCommands.OK
}
async emulatorPermissions(
params: EmulatorTargetParams & {
op: 'grant' | 'revoke' | 'reset'
package?: string
permission?: string
}
): Promise<{ ok: true }> {
const worktreeId = await this.resolveWorktreeId(params.worktree)
await this.requireEmulatorBridge().runCapability(
'permissions',
{ device: params.device ?? params.emulator, worktreeId },
(backend, device) =>
backend.setPermission!(device, params.op, params.package ?? '', params.permission)
)
return RuntimeEmulatorCommands.OK
}
async emulatorAx(params: EmulatorTargetParams): Promise<unknown> {
const worktreeId = await this.resolveWorktreeId(params.worktree)
return this.requireEmulatorBridge().accessibilityTree({
device: params.device ?? params.emulator,
worktreeId
})
}
async emulatorLogcat(
params: EmulatorTargetParams & { lines?: number; filters?: string[] }
): Promise<unknown> {
const worktreeId = await this.resolveWorktreeId(params.worktree)
return this.requireEmulatorBridge().runCapability(
'logcat',
{ device: params.device ?? params.emulator, worktreeId },
(backend, device) => backend.logcat!(device, { lines: params.lines, filters: params.filters })
)
}
async emulatorKill(params: {
device?: string
emulator?: string
worktree?: string
}): Promise<{ ok: true; deviceUdid: string }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
const killedUdid = await bridge.kill(params.device ?? params.emulator, worktreeId)
return { ok: true, deviceUdid: killedUdid }
}
async emulatorShutdown(params: {
device?: string
emulator?: string
worktree?: string
managedOnly?: boolean
}): Promise<{ ok: true; deviceUdid?: string }> {
const bridge = this.requireEmulatorBridge()
const worktreeId = await this.resolveWorktreeId(params.worktree)
if (params.managedOnly && worktreeId && !params.device && !params.emulator) {
const shutdownUdid = await bridge.shutdownActiveManagedForWorktree(worktreeId)
return { ok: true, deviceUdid: shutdownUdid ?? undefined }
}
const shutdownUdid = await bridge.shutdown(params.device ?? params.emulator, worktreeId)
return { ok: true, deviceUdid: shutdownUdid }
}
// Window may not exist during shutdown, so sends are best-effort.
private sendToRenderer(channel: string, payload: unknown): void {
try {
this.host.getAuthoritativeWindow().webContents.send(channel, payload)
} catch {
// Window may not exist during shutdown
}
}
// Why: mirror browser:pane-focus — scoped per worktree, no cross-worktree yank unless user is already there.
private notifyRendererEmulatorPaneFocus(worktreeId: string): void {
this.sendToRenderer('emulator:pane-focus', { worktreeId })
}
private notifyRendererEmulatorAutoAttach(worktreeId: string, info: EmulatorSessionInfo): void {
this.sendToRenderer('ui:emulatorAutoAttach', { worktreeId, info })
}
// Raw for extensibility.
async emulatorExecRaw(params: {
command: string
device?: string
emulator?: string
worktree?: string
}): Promise<unknown> {
return this.emulatorExec(params)
}
}
// Singleton accessor pattern (mirror requireAgentBrowserBridge).
let emulatorBridgeInstance: EmulatorBridge | null = null
export function setEmulatorBridge(bridge: EmulatorBridge | null): void {
emulatorBridgeInstance = bridge
}
export function getEmulatorBridge(): EmulatorBridge | null {
return emulatorBridgeInstance
}
export function requireEmulatorBridge(): EmulatorBridge {
if (!emulatorBridgeInstance) {
throw new EmulatorError('emulator_no_active', 'Emulator bridge not initialized')
}
return emulatorBridgeInstance
}