fix(emulator): cap iOS AX tree at 500 nodes

Unbounded accessibility trees can flood agent output. Enforce a 500-node limit (matching serve-sim's snapshot cap) and mark truncated parents so consumers know the tree was cut.
This commit is contained in:
Jinjing
2026-07-22 21:11:32 -07:00
parent b13dba4088
commit fad13ae5b4
4 changed files with 56 additions and 7 deletions
+1 -1
View File
@@ -99,7 +99,7 @@ Use `--json` for agent-friendly output. Commands are workspace-scoped by default
| Rotate device | `ORCA emulator rotate landscape_left` | Remembers orientation for subsequent gestures. |
| Camera injection | `ORCA emulator camera com.acme.App --webcam` | Or --file, placeholder. Hot-swap with switch. May (re)launch app. |
| Permissions | `ORCA emulator permissions grant camera com.acme.App` | grant/revoke/reset/list. See full subcommand help. |
| Accessibility tree | `ORCA emulator ax [--device <id>]` | Raw serve-sim AX node tree (labels, roles, nested children; frames normalized 0..1 with top-left origin — tap an element at its frame center: x+width/2, y+height/2). Needs an active session. |
| Accessibility tree | `ORCA emulator ax [--device <id>]` | Raw serve-sim AX node tree (labels, roles, nested children, capped at 500 nodes; frames normalized 0..1 with top-left origin — tap an element at its frame center: x+width/2, y+height/2). Needs an active session. |
| Raw / advanced | `ORCA emulator exec --command "tap 0.5 0.7"` | Or "ca-debug blended on", "memory-warning", full serve-sim subcommands (no "serve-sim" prefix needed in the command string). Bridge injects active device context. |
| Stop | `ORCA emulator kill [--device <id>]` | Or let pane close / Orca quit clean up. |
File diff suppressed because one or more lines are too long
@@ -86,6 +86,28 @@ describe('normalizeServeSimAxTree', () => {
})
})
it('caps the tree at 500 nodes and marks the parent whose children were cut', () => {
const child = (label: string) => ({
type: 'StaticText',
AXLabel: label,
frame: { x: 0, y: 0, width: 10, height: 10 },
children: []
})
const raw = [
{
type: 'Application',
frame: { x: 0, y: 0, width: 400, height: 800 },
children: Array.from({ length: 600 }, (_, i) => child(`row-${i}`))
}
]
const [root] = normalizeServeSimAxTree(raw)
// Root consumes one slot of the 500-node budget.
expect(root.children).toHaveLength(499)
expect(root.truncated).toBe(true)
expect(root.children[0]!.truncated).toBeUndefined()
})
it('falls back to a unit screen for malformed roots instead of dividing by zero', () => {
const raw = [{ type: 'Application', children: [] }]
expect(normalizeServeSimAxTree(raw)).toEqual([
@@ -7,6 +7,9 @@
export type NormalizedAxFrame = { x: number; y: number; width: number; height: number }
// Matches serve-sim's own snapshot cap; an unbounded tree can flood agent output.
const MAX_AX_NODES = 500
// One accessibility element, position normalized, children nested (raw tree shape).
export type NormalizedAxNode = {
role: string
@@ -17,6 +20,8 @@ export type NormalizedAxNode = {
id?: string
frame: NormalizedAxFrame
children: NormalizedAxNode[]
// Present when children were dropped by the node cap.
truncated?: true
}
function asRecord(value: unknown): Record<string, unknown> {
@@ -60,8 +65,21 @@ function normalizeFrame(frame: NormalizedAxFrame, screen: NormalizedAxFrame): No
}
}
function normalizeNode(raw: unknown, screen: NormalizedAxFrame): NormalizedAxNode {
function normalizeNode(
raw: unknown,
screen: NormalizedAxFrame,
budget: { remaining: number }
): NormalizedAxNode {
budget.remaining -= 1
const node = asRecord(raw)
const rawChildren = Array.isArray(node.children) ? node.children : []
const children: NormalizedAxNode[] = []
for (const child of rawChildren) {
if (budget.remaining <= 0) {
break
}
children.push(normalizeNode(child, screen, budget))
}
const normalized: NormalizedAxNode = {
role: asString(node.role_description),
type: asString(node.type),
@@ -69,19 +87,28 @@ function normalizeNode(raw: unknown, screen: NormalizedAxFrame): NormalizedAxNod
value: asString(node.AXValue),
enabled: node.enabled !== false,
frame: normalizeFrame(readFrame(node.frame), screen),
children: Array.isArray(node.children)
? node.children.map((child) => normalizeNode(child, screen))
: []
children
}
// AXUniqueId is often null; only surface it when the helper provides one.
const uniqueId = asString(node.AXUniqueId)
if (uniqueId) {
normalized.id = uniqueId
}
if (children.length < rawChildren.length) {
normalized.truncated = true
}
return normalized
}
export function normalizeServeSimAxTree(roots: unknown[]): NormalizedAxNode[] {
const screen = screenFrame(roots)
return roots.map((root) => normalizeNode(root, screen))
const budget = { remaining: MAX_AX_NODES }
const normalized: NormalizedAxNode[] = []
for (const root of roots) {
if (budget.remaining <= 0) {
break
}
normalized.push(normalizeNode(root, screen, budget))
}
return normalized
}