mirror of
https://github.com/stablyai/orca.git
synced 2026-10-18 16:03:17 +00:00
fix(emulator): cap iOS AX tree at 500 nodes
Unbounded accessibility trees can flood agent output. Enforce a 500-node limit (matching serve-sim's snapshot cap) and mark truncated parents so consumers know the tree was cut.
This commit is contained in:
@@ -99,7 +99,7 @@ Use `--json` for agent-friendly output. Commands are workspace-scoped by default
|
||||
| Rotate device | `ORCA emulator rotate landscape_left` | Remembers orientation for subsequent gestures. |
|
||||
| Camera injection | `ORCA emulator camera com.acme.App --webcam` | Or --file, placeholder. Hot-swap with switch. May (re)launch app. |
|
||||
| Permissions | `ORCA emulator permissions grant camera com.acme.App` | grant/revoke/reset/list. See full subcommand help. |
|
||||
| Accessibility tree | `ORCA emulator ax [--device <id>]` | Raw serve-sim AX node tree (labels, roles, nested children; frames normalized 0..1 with top-left origin — tap an element at its frame center: x+width/2, y+height/2). Needs an active session. |
|
||||
| Accessibility tree | `ORCA emulator ax [--device <id>]` | Raw serve-sim AX node tree (labels, roles, nested children, capped at 500 nodes; frames normalized 0..1 with top-left origin — tap an element at its frame center: x+width/2, y+height/2). Needs an active session. |
|
||||
| Raw / advanced | `ORCA emulator exec --command "tap 0.5 0.7"` | Or "ca-debug blended on", "memory-warning", full serve-sim subcommands (no "serve-sim" prefix needed in the command string). Bridge injects active device context. |
|
||||
| Stop | `ORCA emulator kill [--device <id>]` | Or let pane close / Orca quit clean up. |
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -86,6 +86,28 @@ describe('normalizeServeSimAxTree', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('caps the tree at 500 nodes and marks the parent whose children were cut', () => {
|
||||
const child = (label: string) => ({
|
||||
type: 'StaticText',
|
||||
AXLabel: label,
|
||||
frame: { x: 0, y: 0, width: 10, height: 10 },
|
||||
children: []
|
||||
})
|
||||
const raw = [
|
||||
{
|
||||
type: 'Application',
|
||||
frame: { x: 0, y: 0, width: 400, height: 800 },
|
||||
children: Array.from({ length: 600 }, (_, i) => child(`row-${i}`))
|
||||
}
|
||||
]
|
||||
|
||||
const [root] = normalizeServeSimAxTree(raw)
|
||||
// Root consumes one slot of the 500-node budget.
|
||||
expect(root.children).toHaveLength(499)
|
||||
expect(root.truncated).toBe(true)
|
||||
expect(root.children[0]!.truncated).toBeUndefined()
|
||||
})
|
||||
|
||||
it('falls back to a unit screen for malformed roots instead of dividing by zero', () => {
|
||||
const raw = [{ type: 'Application', children: [] }]
|
||||
expect(normalizeServeSimAxTree(raw)).toEqual([
|
||||
|
||||
@@ -7,6 +7,9 @@
|
||||
|
||||
export type NormalizedAxFrame = { x: number; y: number; width: number; height: number }
|
||||
|
||||
// Matches serve-sim's own snapshot cap; an unbounded tree can flood agent output.
|
||||
const MAX_AX_NODES = 500
|
||||
|
||||
// One accessibility element, position normalized, children nested (raw tree shape).
|
||||
export type NormalizedAxNode = {
|
||||
role: string
|
||||
@@ -17,6 +20,8 @@ export type NormalizedAxNode = {
|
||||
id?: string
|
||||
frame: NormalizedAxFrame
|
||||
children: NormalizedAxNode[]
|
||||
// Present when children were dropped by the node cap.
|
||||
truncated?: true
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
@@ -60,8 +65,21 @@ function normalizeFrame(frame: NormalizedAxFrame, screen: NormalizedAxFrame): No
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeNode(raw: unknown, screen: NormalizedAxFrame): NormalizedAxNode {
|
||||
function normalizeNode(
|
||||
raw: unknown,
|
||||
screen: NormalizedAxFrame,
|
||||
budget: { remaining: number }
|
||||
): NormalizedAxNode {
|
||||
budget.remaining -= 1
|
||||
const node = asRecord(raw)
|
||||
const rawChildren = Array.isArray(node.children) ? node.children : []
|
||||
const children: NormalizedAxNode[] = []
|
||||
for (const child of rawChildren) {
|
||||
if (budget.remaining <= 0) {
|
||||
break
|
||||
}
|
||||
children.push(normalizeNode(child, screen, budget))
|
||||
}
|
||||
const normalized: NormalizedAxNode = {
|
||||
role: asString(node.role_description),
|
||||
type: asString(node.type),
|
||||
@@ -69,19 +87,28 @@ function normalizeNode(raw: unknown, screen: NormalizedAxFrame): NormalizedAxNod
|
||||
value: asString(node.AXValue),
|
||||
enabled: node.enabled !== false,
|
||||
frame: normalizeFrame(readFrame(node.frame), screen),
|
||||
children: Array.isArray(node.children)
|
||||
? node.children.map((child) => normalizeNode(child, screen))
|
||||
: []
|
||||
children
|
||||
}
|
||||
// AXUniqueId is often null; only surface it when the helper provides one.
|
||||
const uniqueId = asString(node.AXUniqueId)
|
||||
if (uniqueId) {
|
||||
normalized.id = uniqueId
|
||||
}
|
||||
if (children.length < rawChildren.length) {
|
||||
normalized.truncated = true
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
export function normalizeServeSimAxTree(roots: unknown[]): NormalizedAxNode[] {
|
||||
const screen = screenFrame(roots)
|
||||
return roots.map((root) => normalizeNode(root, screen))
|
||||
const budget = { remaining: MAX_AX_NODES }
|
||||
const normalized: NormalizedAxNode[] = []
|
||||
for (const root of roots) {
|
||||
if (budget.remaining <= 0) {
|
||||
break
|
||||
}
|
||||
normalized.push(normalizeNode(root, screen, budget))
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user