From 3586fdc055830ffc8b897866ee313af0588c18df Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:18:24 -0800 Subject: [PATCH 01/16] fix(terminal): capture replay snapshots at one output position (#41247) Capture the terminal screen, output position and unfinished parser state together on the existing output queue so restored output continues at the correct byte. Retire reconstructed partial escape state on Reset. Tracked in #34435 and STA-12466. Co-authored-by: liaoxingyu <3114147163@qq.com> --- .../output/headless-buffer-snapshot.ts | 81 ++++++++++------- .../headless-snapshot-stream-position.test.ts | 86 +++++++++++++++++++ .../output/headless-terminal-state.ts | 1 + 3 files changed, 136 insertions(+), 32 deletions(-) create mode 100644 src/main/runtime/terminal/output/headless-snapshot-stream-position.test.ts diff --git a/src/main/runtime/terminal/output/headless-buffer-snapshot.ts b/src/main/runtime/terminal/output/headless-buffer-snapshot.ts index df00b995cb2..6f49d1bc0ea 100644 --- a/src/main/runtime/terminal/output/headless-buffer-snapshot.ts +++ b/src/main/runtime/terminal/output/headless-buffer-snapshot.ts @@ -50,38 +50,55 @@ export abstract class OrcaRuntimeWithSerializeHeadlessTerminalBuffer extends Orc } await state.writeChain await state.ownership.settle() - // Why: normal history is separated from an active alternate frame, so the - // caller's scrollback policy can be honored without painting it into alt. - const scrollbackRows = opts.scrollbackRows ?? 0 - const snapshot = state.emulator.getSnapshot({ scrollbackRows }) - const terminalOwner = state.ownership.owner - const data = snapshot.rehydrateSequences + snapshot.snapshotAnsi - return data.length > 0 || opts.includeEmpty === true - ? this.preferTrackedLastTitle(ptyId, { - data, - frameRestoreAnsi: snapshot.frameRestoreAnsi, - cols: snapshot.cols, - rows: snapshot.rows, - cwd: snapshot.cwd ?? this.terminalCwdByPtyId.get(ptyId), - lastTitle: snapshot.lastTitle, - seq: state.outputSequence, - source: 'headless' as const, - oscLinks: snapshot.oscLinks, - scrollbackAnsi: snapshot.scrollbackAnsi, - // Capture flags at the image's stream position to prevent duplicate/out-of-order replay pushes and pops. - ...(parseTerminalKittyKeyboardFlags(snapshot.modes?.kittyKeyboardFlags) !== undefined - ? { kittyKeyboardFlags: snapshot.modes.kittyKeyboardFlags } - : {}), - ...(snapshot.pendingEscapeTailAnsi - ? { pendingEscapeTailAnsi: snapshot.pendingEscapeTailAnsi } - : {}), - ...(terminalOwner ? { terminalOwner } : {}), - // Alt snapshots must preserve xterm history used for TUI scroll-up after tab return. - alternateScreen: snapshot.modes?.alternateScreen ?? state.emulator.isAlternateScreen, - // Do not fold into data: the restorer's later reset would abort this partial escape. - pendingEscapeTailAnsi: snapshot.pendingEscapeTailAnsi - }) - : null + // Capture image and applied progress in the same ordered operation as parsing. + const completion = state.writeChain.then(() => { + const current = + this.headlessTerminals.get(ptyId) ?? + (this.ptysById.get(ptyId)?.connected === true + ? undefined + : this.headlessExitHolds.get(ptyId)?.exited) + if (current !== state) { + return null + } + // Why: normal history is separated from an active alternate frame, so the + // caller's scrollback policy can be honored without painting it into alt. + const scrollbackRows = opts.scrollbackRows ?? 0 + const snapshot = state.emulator.getSnapshot({ scrollbackRows }) + const terminalOwner = state.ownership.owner + const pendingEscapeTailAnsi = + state.rendererSeedSeq !== undefined + ? state.rendererSeedEscapeTail + : snapshot.pendingEscapeTailAnsi + const data = snapshot.rehydrateSequences + snapshot.snapshotAnsi + return data.length > 0 || opts.includeEmpty === true + ? this.preferTrackedLastTitle(ptyId, { + data, + frameRestoreAnsi: snapshot.frameRestoreAnsi, + cols: snapshot.cols, + rows: snapshot.rows, + cwd: snapshot.cwd ?? this.terminalCwdByPtyId.get(ptyId), + lastTitle: snapshot.lastTitle, + seq: state.outputSequence, + source: 'headless' as const, + oscLinks: snapshot.oscLinks, + scrollbackAnsi: snapshot.scrollbackAnsi, + // Capture flags at the image's stream position to prevent duplicate/out-of-order replay pushes and pops. + ...(parseTerminalKittyKeyboardFlags(snapshot.modes?.kittyKeyboardFlags) !== undefined + ? { kittyKeyboardFlags: snapshot.modes.kittyKeyboardFlags } + : {}), + ...(terminalOwner ? { terminalOwner } : {}), + // Alt snapshots must preserve xterm history used for TUI scroll-up after tab return. + alternateScreen: snapshot.modes?.alternateScreen ?? state.emulator.isAlternateScreen, + // Do not fold into data: the restorer's later reset would abort this partial escape. + pendingEscapeTailAnsi + }) + : null + }) + state.writeChain = completion.then( + () => {}, + () => {} + ) + return completion } /** Keeps the PTY's model readable through its exit until released, so a stream mid-publication can still serialize it. */ diff --git a/src/main/runtime/terminal/output/headless-snapshot-stream-position.test.ts b/src/main/runtime/terminal/output/headless-snapshot-stream-position.test.ts new file mode 100644 index 00000000000..1b3f757e453 --- /dev/null +++ b/src/main/runtime/terminal/output/headless-snapshot-stream-position.test.ts @@ -0,0 +1,86 @@ +import { assert, expect, it } from 'vitest' +import { OrcaRuntimeService } from '../../orca-runtime' +import { HeadlessEmulator } from '../../../daemon/emulation/headless-emulator' + +const grid = { cols: 80, rows: 24 } +const partial = 'prompt\r\n> \x1b[?' + +function createRuntime() { + const runtime = new OrcaRuntimeService() + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + getSize: () => grid + }) + return runtime +} + +async function replay( + snapshot: { data: string; pendingEscapeTailAnsi?: string } | null, + suffix: string +) { + assert(snapshot) + const restored = new HeadlessEmulator(grid) + try { + await restored.write(snapshot.data) + await restored.write(snapshot.pendingEscapeTailAnsi ?? '') + await restored.write(suffix) + return restored + .getVisibleLines() + .map((line) => line.trimEnd()) + .filter(Boolean) + } finally { + restored.dispose() + } +} + +it('restores a screen and live suffix without repainting a consumed escape continuation', async () => { + const runtime = createRuntime() + const continuation = '1002h\x1b[>4;2m' + try { + runtime.onPtyData('pty-snapshot', partial, 1) + const pending = runtime.serializeHiddenOutputRecoveryBuffer('pty-snapshot') + queueMicrotask(() => runtime.onPtyData('pty-snapshot', continuation, 2)) + const snapshot = await pending + expect(await replay(snapshot, (partial + continuation).slice(snapshot?.seq))).toEqual([ + 'prompt', + '>' + ]) + } finally { + await runtime.onPtyExit('pty-snapshot', 0) + } +}) + +it.each([false, true])( + 'preserves seeded parser state through replay and live output (reset=%s)', + async (reset) => { + const runtime = createRuntime() + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + getSize: () => grid, + hasRendererSerializer: () => true, + serializeBuffer: async () => ({ data: 'prompt\r\n> ', ...grid, seq: 13 }) + }) + try { + runtime.onPtyData('pty-seed', partial, 1) + let snapshot = await runtime.serializeHiddenOutputRecoveryBuffer('pty-seed') + expect(snapshot?.seq).toBe(13) + if (reset) { + await runtime.resetHeadlessTerminalInputModes('pty-seed') + snapshot = await runtime.serializeHiddenOutputRecoveryBuffer('pty-seed') + } + const continuation = reset ? 'hello' : '1002h' + const expected = reset ? ['prompt', '> hello'] : ['prompt', '>'] + expect(await replay(snapshot, continuation)).toEqual(expected) + + runtime.onPtyData('pty-seed', continuation, 2) + const live = await runtime.serializeHiddenOutputRecoveryBuffer('pty-seed') + expect(await replay(live, '')).toEqual(expected) + } finally { + await runtime.onPtyExit('pty-seed', 0) + } + } +) diff --git a/src/main/runtime/terminal/output/headless-terminal-state.ts b/src/main/runtime/terminal/output/headless-terminal-state.ts index 6df5db366d7..9ea360a80a1 100644 --- a/src/main/runtime/terminal/output/headless-terminal-state.ts +++ b/src/main/runtime/terminal/output/headless-terminal-state.ts @@ -269,6 +269,7 @@ export abstract class OrcaRuntimeWithCreatePtyHeadlessTerminalState extends Orca // Why on the chain: the ground must land after every PTY chunk already queued. const completion = state.writeChain.then(async () => { await state.emulator.write(state.ownership.groundInputModes()) + state.rendererSeedEscapeTail = undefined }) state.writeChain = completion.catch(() => {}) await completion From 8ed0d047db5fadd33b75540a605333b40105a136 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:22:18 +0000 Subject: [PATCH 02/16] feat(computer): show where snapshots were cut and read one element's subtree (#41482) * feat(computer): show where snapshots were cut and read one element's subtree * fix(computer): validate subtree targets against the fresh walk and keep scope for focused input * fix(computer): return whole-window action results after subtree reads * fix(computer): count kept records toward the macOS limit and index subtree reads against the latest cache * fix(computer): index Linux/Windows subtree reads against the snapshot they started from --- docs/site/content/docs/cli/computer-use.mdx | 2 +- native/computer-use-linux/runtime.py | 35 ++- .../computer-use-linux/runtime_render_test.py | 20 ++ .../Sources/OrcaComputerUseMacOS/main.swift | 213 ++++++++++++------ native/computer-use-windows/runtime.ps1 | 39 ++-- skill-guides/orca-computer-use.md | 6 + src/cli/computer/computer-format.ts | 2 +- src/cli/computer/computer.ts | 15 +- src/cli/discovery/command-scoped-flag-help.ts | 2 + src/cli/skills/bundled-skill-guides.ts | 6 + src/cli/specs/computer.ts | 4 +- .../desktop-script-provider-client.ts | 9 + .../desktop-script-provider-types.ts | 4 + .../desktop-script-snapshot-rendering.ts | 6 + .../desktop-script-subtree-snapshot.test.ts | 72 ++++++ .../desktop-script-subtree-snapshot.ts | 103 +++++++++ src/shared/computer-use-runtime-types.ts | 2 + src/shared/computer-use-tree-text.ts | 10 + .../rpc-contract/computer-schemas-params.ts | 5 +- 19 files changed, 455 insertions(+), 100 deletions(-) create mode 100644 src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts create mode 100644 src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index 4f6b8f31ae7..776ac5a21a2 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -37,7 +37,7 @@ orca computer click --app com.spotify.client --element-index 42 --json Element indexes are scoped to the latest `get-app-state` result and may be **sparse**. In `--json` output, read the tree from `result.snapshot.treeText`. Do not invent indexes from `elementCount`. Refresh state after navigation, focus changes, scrolling, or any app re-render before reusing an index. -Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query ` prints only matching elements and their ancestors, and `--max-elements ` caps the output. +Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query ` prints only matching elements and their ancestors, and `--max-elements ` caps the output. A window larger than the snapshot limit shows `... N more not shown (snapshot limit)` where the tree was cut; `get-app-state --element-index ` reads only that element's subtree, so pass the cut-off children's parent to see them. ## Waiting for the app diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index c81e3812534..5e5fa614185 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -487,13 +487,19 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome has_text = bool(item["name"] or item["automationId"] or item["value"]) return role in {"panel", "filler", "unknown", "section"} and not has_text and not meaningful_actions(item["actions"]) and summary is None and child_count <= 1 + def walk_children(child_items, depth, path): + for position, (child_index, child) in enumerate(child_items): + if len(records) >= MAX_NODES or depth > MAX_DEPTH: + truncation["truncated"] = True + if depth > MAX_DEPTH: + truncation["maxDepthReached"] = True + # Marks where the walk stopped, so the agent can read that parent's subtree by index. + lines.append(("\t" * depth) + f"... {len(child_items) - position} more not shown (snapshot limit)") + return + walk(child, depth, path + [child_index]) + def walk(node, depth, path): nonlocal next_index - if len(records) >= MAX_NODES or depth > MAX_DEPTH: - truncation["truncated"] = True - if depth > MAX_DEPTH: - truncation["maxDepthReached"] = True - return item = record(node, next_index, path) role_key = (item["controlType"] or "").lower() title = item["name"] or item["automationId"] or "" @@ -505,8 +511,7 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome if len(summary_values) >= 2 and is_plain_text_subtree(node): generic_summary = " ".join(summary_values) if should_elide(item, len(child_items), generic_summary): - for child_index, child in child_items: - walk(child, depth, path + [child_index]) + walk_children(child_items, depth, path) return records.append(item) next_index += 1 @@ -529,8 +534,7 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome if generic_summary or suppressed: return child_line_start = len(lines) - for child_index, child in child_items: - walk(child, depth + 1, path + [child_index]) + walk_children(child_items, depth + 1, path) if compact_browser_tabs: compact_rendered_browser_tabs(records, lines, child_line_start, depth + 1) @@ -1203,17 +1207,23 @@ def window_json(app, index, window): } -def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False): +def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False, root=None): app = find_app(query) window_index, window = choose_window(app, window_id, window_index) if restore: restore_window(app, window) window_index, window = choose_window(app, window_id, window_index) bounds = screen_rect(window) + tree_root, root_path = window, [window_index] + if root is not None: + tree_root = find_element(app, root) + if tree_root is None: + raise RuntimeError(f"stale element: element {root.get('index')} is gone; run get-app-state again and use a fresh element index") + root_path = list(root.get("runtimeId") or []) geometry_nodes = {} records, lines, truncation = render_accessibility_tree( - window, - [window_index], + tree_root, + root_path, compact_browser_tabs=is_browser_app(query, app), geometry_nodes=geometry_nodes, ) @@ -1875,6 +1885,7 @@ def run_operation(operation): operation.get("windowId"), operation.get("windowIndex"), bool(operation.get("restoreWindow")), + operation.get("root"), ), } diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py index 6e997611596..207d2de3410 100644 --- a/native/computer-use-linux/runtime_render_test.py +++ b/native/computer-use-linux/runtime_render_test.py @@ -699,6 +699,26 @@ class RuntimeRenderTest(unittest.TestCase): indexes = [item["index"] for item in snapshot["elements"]] self.assertEqual(len(indexes), len(set(indexes))) + def test_cut_tree_marks_hidden_children_and_root_reads_them(self): + items = [FakeAccessible("button", f"A{i}") for i in range(3)] + root = FakeAccessible("frame", "Main", children=[ + FakeAccessible("list", "Items", children=items), FakeAccessible("button", "After"), + ]) + with SnapshotFixture(root), patch.object(runtime, "MAX_NODES", 4): + snapshot = runtime.make_snapshot("Fixture", False) + self.assertEqual(snapshot["treeLines"], [ + "0 frame Main", + "\t1 list Items", + "\t\t2 button A0", + "\t\t3 button A1", + "\t\t... 1 more not shown (snapshot limit)", + "\t... 1 more not shown (snapshot limit)", + ]) + items_record = next(item for item in snapshot["elements"] if item["name"] == "Items") + subtree = runtime.run_operation({"tool": "get_app_state", "app": "Fixture", "noScreenshot": True, "root": items_record})["snapshot"] + self.assertEqual(subtree["treeLines"], ["0 list Items", "\t1 button A0", "\t2 button A1", "\t3 button A2"]) + self.assertEqual(subtree["elements"][-1]["runtimeId"], [0, 0, 2]) + diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 9a43b296403..0027bea1f5a 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -143,6 +143,12 @@ final class ElementRecord { } } +/// A snapshot that read one element's subtree; its elements also keep the window's earlier ones. +struct SnapshotScope { + let rootIndex: Int + let walkedIndexes: Set +} + struct Snapshot { let id: String let app: AppDescriptor @@ -161,6 +167,7 @@ struct Snapshot { let nextFreshElementIndex: Int let truncated: Bool let maxDepthReached: Bool + let scope: SnapshotScope? /// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change. let appWindowIds: [CGWindowID] @@ -187,6 +194,7 @@ struct Snapshot { nextFreshElementIndex: nextFreshElementIndex, truncated: truncated, maxDepthReached: maxDepthReached, + scope: scope, appWindowIds: appWindowIds ) } @@ -262,9 +270,10 @@ final class Provider { var action = try runAction() // Why: only --restore-window activates the app or posts to the HID tap; every other route leaves the user's desktop alone. action["delivery"] = isForegroundRequest(params) ? "foreground" : "background" - // Why: the action's currentSnapshot just cached this target's tree from before the action. + // Why: the action's currentSnapshot just cached this target's tree from before the action; + // a subtree read can't be compared with the whole-window read that follows. let before = action["path"] as? String == "accessibility" && action["verification"] == nil - ? try? cachedSnapshot(params: params) + ? (try? cachedSnapshot(params: params)).flatMap { $0.scope == nil ? $0 : nil } : nil do { let after = try observe(params: params) @@ -315,12 +324,14 @@ final class Provider { } let namespace = snapshotNamespace(params) pruneSnapshotCache() + let subtree = try requestedSubtree(params: params, app: app) let snapshot = try buildSnapshot( app: app, includeScreenshot: params["noScreenshot"]?.bool != true, - windowId: windowId, - windowIndex: windowIndex, + windowId: subtree.map(\.base.windowId) ?? windowId, + windowIndex: subtree == nil ? windowIndex : nil, restoreWindow: params["restoreWindow"]?.bool == true, + subtree: subtree, previousSnapshot: { [snapshots] resolvedWindowId in // Why: indexes carry over from this caller's own last view of exactly this window. snapshots[namespacedSnapshotKey(namespace, snapshotCanonicalWindowIdKey(resolvedWindowId))] @@ -339,6 +350,15 @@ final class Provider { return snapshot } + /// Resolves `rootElementIndex` against this caller's last snapshot of the window, whose indexes it extends. + private func requestedSubtree(params: [String: JSONValue], app: AppDescriptor) throws -> (root: ElementRecord, base: Snapshot)? { + guard let rootIndex = try optionalInteger(params, "rootElementIndex") else { return nil } + guard let base = try cachedSnapshot(params: params), base.app.pid == app.pid else { + throw ProviderError.coded("element_not_found", "--element-index needs a get-app-state snapshot of this app/window first") + } + return (try element(base, rootIndex), base) + } + private func rememberSnapshot( query: String, app: AppDescriptor, @@ -416,19 +436,30 @@ final class Provider { } } - private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot { + private func currentSnapshot(params: [String: JSONValue], needsFocus: Bool = false) throws -> Snapshot { pruneSnapshotCache() let cached = try cachedSnapshot(params: params) + let requestedIndexes = try requestedElementIndexes(params) + let observeParams = params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement } + // Why: elements read with --element-index, focused ones included, may lie past the whole-window node limit, + // so only their subtree can re-find them. + if let scope = cached?.scope, + requestedIndexes.isEmpty ? needsFocus : requestedIndexes.allSatisfy(scope.walkedIndexes.contains), + let scoped = try? observe(params: observeParams.merging(["rootElementIndex": .number(Double(scope.rootIndex))]) { _, replacement in replacement }), + !requestedIndexes.isEmpty || scoped.focusedElementId != nil { + try validateRequestedElements(cached: cached, current: scoped, requestedIndexes: requestedIndexes) + return scoped + } // Why: cached AX frames can be stale after a window move or resize, and // stale geometry can turn an intended action into a misclick. - let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) - try validateRequestedElements(cached: cached, current: snapshot, params: params) + let snapshot = try observe(params: observeParams) + try validateRequestedElements(cached: cached, current: snapshot, requestedIndexes: requestedIndexes) return snapshot } private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot { // Why: keyboard actions need fresh focused-element state, never pixels. - try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) + try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }, needsFocus: true) } private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? { @@ -473,17 +504,24 @@ final class Provider { (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key]) } - private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws { - let requestedIndexes = try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in + private func requestedElementIndexes(_ params: [String: JSONValue]) throws -> [Int] { + try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in guard params[key]?.number != nil else { return nil } return try optionalInteger(params, key) } + } + + private func validateRequestedElements(cached: Snapshot?, current: Snapshot, requestedIndexes: [Int]) throws { guard !requestedIndexes.isEmpty else { return } guard let cached else { throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window") } for index in requestedIndexes { - guard let expected = cached.elements[index], let actual = current.elements[index] else { + // Why: a subtree read also carries the window's earlier records, which prove nothing about the element now. + guard let expected = cached.elements[index], + let actual = current.elements[index], + current.scope?.walkedIndexes.contains(index) ?? true + else { throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index") } guard expected.signature == actual.signature else { @@ -658,6 +696,7 @@ final class Provider { windowId: CGWindowID?, windowIndex: Int?, restoreWindow: Bool, + subtree: (root: ElementRecord, base: Snapshot)?, previousSnapshot: (CGWindowID) -> Snapshot? ) throws -> Snapshot { guard accessibilityTrustedSettled() else { @@ -703,11 +742,29 @@ final class Provider { focused: focusedElement(appElement: appElement), compactBrowserTabs: app.isKnownBrowser ) - renderer.render(window) + if let subtree { + guard let ancestors = ancestorChain(of: subtree.root.element, in: window) else { + throw ProviderError.coded("element_not_found", "element \(subtree.root.index) is no longer in this window; run get-app-state again and use a fresh element index") + } + renderer.render(subtree.root.element, ancestors: ancestors) + } else { + renderer.render(window) + } if renderer.appStoppedResponding { throw AccessibilityMessaging.appNotResponding(app.name) } - let indexes = renderer.carryOverIndexes(from: previousSnapshot(capture.windowId)) + let previous = subtree?.base ?? previousSnapshot(capture.windowId) + let indexes = renderer.carryOverIndexes(from: previous) + var elements = renderer.records + var elementOrder = indexes.order + if let subtree { + // Why: elements outside the subtree keep resolving, so the agent can still act on what the whole-window read showed. + // Re-reading the same subtree replaces its last walk, so rows scrolled away don't pile up. + let replaced = subtree.base.scope.flatMap { $0.rootIndex == subtree.root.index ? $0.walkedIndexes : nil } ?? [] + let kept = subtree.base.elements.filter { !replaced.contains($0.key) && renderer.records[$0.key] == nil } + elements = kept.merging(renderer.records) { _, walked in walked } + elementOrder = subtree.base.elementOrder.filter { kept[$0] != nil } + indexes.order + } let screenshot = includeScreenshot ? capture.screenshotPayload() : nil let screenshotStatus: ScreenshotStatus = if screenshot != nil { .captured @@ -725,17 +782,18 @@ final class Provider { windowBounds: capture.bounds, windowId: capture.windowId, windowLayer: capture.layer, - treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary), + treeText: renderTreeText(app: app, title: title, rootIndex: subtree?.root.index, lines: renderer.lines, focused: renderer.focusedSummary), focusedElementId: renderer.focusedElementId, screenshot: screenshot, screenshotStatus: screenshotStatus, screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds), screenshotEngine: capture.image?.engine, - elements: renderer.records, - elementOrder: indexes.order, + elements: elements, + elementOrder: elementOrder, nextFreshElementIndex: indexes.nextFreshIndex, truncated: renderer.truncated, maxDepthReached: renderer.maxDepthReached, + scope: subtree.map { SnapshotScope(rootIndex: $0.root.index, walkedIndexes: Set(indexes.order)) }, appWindowIds: windowCandidates.map(\.windowId).sorted() ) } @@ -751,39 +809,43 @@ final class Provider { "scale": payload.scale, ] } - return [ - "snapshot": [ - "id": snapshot.id, - "app": [ - "name": snapshot.app.name, - "bundleId": jsonNullable(snapshot.app.bundleId), - "pid": Int(snapshot.app.pid), - ], - "window": [ - "id": Int(snapshot.windowId), - "title": snapshot.windowTitle, - "x": Int(snapshot.windowBounds.origin.x.rounded()), - "y": Int(snapshot.windowBounds.origin.y.rounded()), - "width": Int(snapshot.windowBounds.width.rounded()), - "height": Int(snapshot.windowBounds.height.rounded()), - "isMinimized": false, - "isOffscreen": false, - "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), - "platform": [ - "layer": snapshot.windowLayer, - ], - ], - "coordinateSpace": "window", - "treeText": snapshot.treeText, - "elementCount": snapshot.elements.count, - "focusedElementId": snapshot.focusedElementId as Any, - "truncation": [ - "truncated": snapshot.truncated, - "maxNodes": TreeRenderer.maxNodes, - "maxDepth": TreeRenderer.maxDepth, - "maxDepthReached": snapshot.maxDepthReached, + var snapshotPayload: [String: Any] = [ + "id": snapshot.id, + "app": [ + "name": snapshot.app.name, + "bundleId": jsonNullable(snapshot.app.bundleId), + "pid": Int(snapshot.app.pid), + ], + "window": [ + "id": Int(snapshot.windowId), + "title": snapshot.windowTitle, + "x": Int(snapshot.windowBounds.origin.x.rounded()), + "y": Int(snapshot.windowBounds.origin.y.rounded()), + "width": Int(snapshot.windowBounds.width.rounded()), + "height": Int(snapshot.windowBounds.height.rounded()), + "isMinimized": false, + "isOffscreen": false, + "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), + "platform": [ + "layer": snapshot.windowLayer, ], ], + "coordinateSpace": "window", + "treeText": snapshot.treeText, + "elementCount": snapshot.scope?.walkedIndexes.count ?? snapshot.elements.count, + "focusedElementId": snapshot.focusedElementId as Any, + "truncation": [ + "truncated": snapshot.truncated, + "maxNodes": TreeRenderer.maxNodes, + "maxDepth": TreeRenderer.maxDepth, + "maxDepthReached": snapshot.maxDepthReached, + ], + ] + if let scope = snapshot.scope { + snapshotPayload["rootElementIndex"] = scope.rootIndex + } + return [ + "snapshot": snapshotPayload, "screenshot": screenshot, "screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot), ] @@ -1462,6 +1524,18 @@ private func pointerRecipient( return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)) } +/// The element's parents from `window` down, or nil when it is no longer inside `window`. +private func ancestorChain(of element: AXUIElement, in window: AXUIElement) -> [AXUIElement]? { + var chain: [AXUIElement] = [] + var current = element + while !CFEqual(current, window) { + guard chain.count < 256, let parent = copyElement(current, kAXParentAttribute as String) else { return nil } + chain.append(parent) + current = parent + } + return chain.reversed() +} + private func containingWindow(_ element: AXUIElement) -> AXUIElement? { var current = element for _ in 0..<64 { @@ -2179,15 +2253,18 @@ private func mouseButton(_ raw: String?) throws -> MouseButtonSelection { } } -private func renderTreeText(app: AppDescriptor, title: String, bounds: CGRect, lines: [String], focused: String?) -> String { +private func renderTreeText(app: AppDescriptor, title: String, rootIndex: Int?, lines: [String], focused: String?) -> String { var output = [ "App=\(app.bundleId ?? app.name.replacingOccurrences(of: " ", with: "_")) (pid \(app.pid))", "Window: \"\(sanitize(title))\", App: \(sanitize(app.name)).", - "", ] + if let rootIndex { + output.append("Showing element \(rootIndex) and its descendants.") + } + output.append("") output.append(contentsOf: lines) output.append("") - output.append(focused.map { "The focused UI element is \($0)." } ?? "No UI element is currently focused.") + output.append(focused.map { "The focused UI element is \($0)." } ?? (rootIndex == nil ? "No UI element is currently focused." : "No UI element inside this one is focused.")) return output.joined(separator: "\n") } @@ -2231,15 +2308,6 @@ private final class TreeRenderer { func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) { guard !reader.appStoppedResponding else { return } - guard nextIndex < Self.maxNodes else { - truncated = true - return - } - guard depth < Self.maxDepth else { - truncated = true - maxDepthReached = true - return - } guard !ancestors.contains(where: { CFEqual($0, element) }) else { return } let role = reader.stringAttribute(element, kAXRoleAttribute as String) ?? "AXUnknown" @@ -2290,9 +2358,7 @@ private final class TreeRenderer { webAreaDepth: webAreaDepth ) if SnapshotRenderHeuristics.shouldElide(node) { - for child in children { - render(child, depth: depth, ancestors: ancestors + [element]) - } + renderChildren(children, depth: depth, ancestors: ancestors + [element]) return } @@ -2315,9 +2381,8 @@ private final class TreeRenderer { return } if compactBrowserTabs, let tabStripCompaction = tabStripCompaction(parent: node, children: children) { - for (childIndex, child) in children.enumerated() where tabStripCompaction.retainedIndexes.contains(childIndex) { - render(child, depth: depth + 1, ancestors: ancestors + [element]) - } + let retained = children.enumerated().filter { tabStripCompaction.retainedIndexes.contains($0.offset) }.map(\.element) + renderChildren(retained, depth: depth + 1, ancestors: ancestors + [element]) lines.append( String(repeating: "\t", count: depth + 1) + "... \(tabStripCompaction.omittedCount) inactive browser tabs omitted" @@ -2325,14 +2390,26 @@ private final class TreeRenderer { return } let childLineStart = lines.count - for child in children { - render(child, depth: depth + 1, ancestors: ancestors + [element]) - } + renderChildren(children, depth: depth + 1, ancestors: ancestors + [element]) if compactBrowserTabs { compactRenderedBrowserTabs(parent: node, startLine: childLineStart, depth: depth + 1) } } + /// Stops at the node or depth limit with a note of how many children were left, so the agent can read that subtree. + private func renderChildren(_ children: [AXUIElement], depth: Int, ancestors: [AXUIElement]) { + for (position, child) in children.enumerated() { + // Why: browser-tab compaction drops records, so the limit counts what is kept, as on Linux and Windows. + guard records.count < Self.maxNodes, depth < Self.maxDepth else { + truncated = true + maxDepthReached = maxDepthReached || depth >= Self.maxDepth + lines.append(String(repeating: "\t", count: depth) + "... \(children.count - position) more not shown (snapshot limit)") + return + } + render(child, depth: depth, ancestors: ancestors) + } + } + private func tabStripCompaction(parent: SnapshotRenderNode, children: [AXUIElement]) -> SnapshotTabStripCompaction? { let childNodes = children.map { child in let role = reader.stringAttribute(child, kAXRoleAttribute as String) ?? "AXUnknown" diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index b8d86341d86..f4e5d1fc62e 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -794,12 +794,20 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = # Tab compaction shrinks records mid-walk, so $records.Count would reuse retained indexes. $nextIndex = [ref]0 - function Visit-OrcaNode($Node, [int]$Depth) { - if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) { - $truncation.truncated = $true - if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true } - return + function Visit-OrcaChildren($Children, [int]$Depth) { + for ($i = 0; $i -lt $Children.Count; $i++) { + if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) { + $truncation.truncated = $true + if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true } + # Marks where the walk stopped, so the agent can read that parent's subtree by index. + $lines.Add(("`t" * $Depth) + "... $($Children.Count - $i) more not shown (snapshot limit)") + return + } + Visit-OrcaNode $Children.Item($i) $Depth } + } + + function Visit-OrcaNode($Node, [int]$Depth) { $identity = try { (@($Node.GetRuntimeId()) -join ".") } catch { [Guid]::NewGuid().ToString() } if (-not $seen.Add($identity)) { return } @@ -820,9 +828,7 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = } } if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) { - for ($i = 0; $i -lt $children.Count; $i++) { - Visit-OrcaNode $children.Item($i) $Depth - } + Visit-OrcaChildren $children $Depth return } @@ -847,9 +853,7 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = if (-not [string]::IsNullOrWhiteSpace($genericSummary) -or (Test-OrcaSuppressChildren $roleKey $title $record.value $genericSummary)) { return } $childLineStart = $lines.Count - for ($i = 0; $i -lt $children.Count; $i++) { - Visit-OrcaNode $children.Item($i) ($Depth + 1) - } + Visit-OrcaChildren $children ($Depth + 1) if ($CompactBrowserTabs) { Compress-OrcaRenderedBrowserTabs $records $lines $childLineStart ($Depth + 1) } @@ -1025,14 +1029,21 @@ function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame, [IntPtr]$Win } } -function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false) { +function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) { $process = Find-OrcaProcess $Query Assert-OrcaWindowTarget $process $WindowId $WindowIndex # Root first: its hang probe must run before ShowWindow, which blocks on a hung window. $root = Get-OrcaRootElement $process if ($RestoreWindow) { Restore-OrcaWindow $process } $windowFrame = Get-OrcaWindowFrame $process $root - $tree = Render-OrcaTree $root $windowFrame (Test-OrcaBrowserProcess $process) + $walkRoot = $root + if ($null -ne $TreeRoot) { + $walkRoot = Find-OrcaElement $root $TreeRoot + if ($null -eq $walkRoot) { + throw "stale element: element $($TreeRoot.index) is gone; run get-app-state again and use a fresh element index" + } + } + $tree = Render-OrcaTree $walkRoot $windowFrame (Test-OrcaBrowserProcess $process) $screenshot = Get-OrcaScreenshot $IncludeScreenshot $windowFrame ([IntPtr]$process.MainWindowHandle) [pscustomobject]@{ @@ -1452,7 +1463,7 @@ function Invoke-OrcaOperation($Operation) { return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) } } if ($Operation.tool -eq "get_app_state") { - return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) } + return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) $Operation.root } } $process = Find-OrcaProcess $Operation.app diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 4e1506d0e88..28ab7323b87 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -46,6 +46,12 @@ In a large window, look for one control with `get-app-state --query `: it ORCA computer get-app-state --app com.apple.finder --query Downloads --json ``` +A snapshot reads at most `truncation.maxNodes` elements and `maxDepth` levels. When it stops early, `truncation.truncated` is true and each cut-off place shows a `... N more not shown (snapshot limit)` line, indented like the children it stands for. To read them, pass the parent element's index (the nearest line above with less indentation) as `get-app-state --element-index `: it reads only that element and its descendants, with the full limit. Elements already shown keep their indexes and hidden ones get new ones, and you can act on them. Action results show the whole window again, so read the subtree again before acting on another hidden element. + +```text +ORCA computer get-app-state --app com.apple.Safari --element-index 312 --json +``` + ## Waiting Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout ` up to 120000). diff --git a/src/cli/computer/computer-format.ts b/src/cli/computer/computer-format.ts index a00f42f2ce7..f13b05919d8 100644 --- a/src/cli/computer/computer-format.ts +++ b/src/cli/computer/computer-format.ts @@ -31,7 +31,7 @@ export function formatGetAppState(result: ComputerSnapshotResult): string { ? '' : ` @ ${result.snapshot.window.x},${result.snapshot.window.y}` const truncation = result.snapshot.truncation?.truncated - ? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'})` + ? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'}); read cut-off children with --element-index ` : ' Truncated: no' return [ `${app.name} (pid ${app.pid}${bundle})`, diff --git a/src/cli/computer/computer.ts b/src/cli/computer/computer.ts index 7a5745c5ec1..cdd3082dfd0 100644 --- a/src/cli/computer/computer.ts +++ b/src/cli/computer/computer.ts @@ -7,6 +7,7 @@ import type { } from '../../shared/computer-use-runtime-types' import type { CommandHandler } from '../dispatch/dispatch' import { + getOptionalNonNegativeIntegerFlag, getOptionalPositiveIntegerFlag, getOptionalStringFlag, getRequiredStringFlag @@ -85,11 +86,23 @@ export const COMPUTER_HANDLERS: Record = { const observeFlags = getComputerObserveFlags(flags) const query = getOptionalStringFlag(flags, 'query') const maxElements = getOptionalPositiveIntegerFlag(flags, 'max-elements') + const rootElementIndex = getOptionalNonNegativeIntegerFlag(flags, 'element-index') const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.getAppState', { ...target, - ...observeFlags + ...observeFlags, + ...(rootElementIndex === undefined ? {} : { rootElementIndex }) }) + // Why: older runtimes drop the param and return the whole window, which would look like the subtree. + if ( + rootElementIndex !== undefined && + result.result.snapshot.rootElementIndex !== rootElementIndex + ) { + throw new RuntimeClientError( + 'unsupported_capability', + 'This Orca runtime cannot read one element subtree; update Orca, or run get-app-state without --element-index' + ) + } if (query !== undefined || maxElements !== undefined) { const { snapshot } = result.result snapshot.treeText = queryComputerTreeText(snapshot.treeText, { query, maxElements }) diff --git a/src/cli/discovery/command-scoped-flag-help.ts b/src/cli/discovery/command-scoped-flag-help.ts index b23764dd9f4..ba18dc9b3c3 100644 --- a/src/cli/discovery/command-scoped-flag-help.ts +++ b/src/cli/discovery/command-scoped-flag-help.ts @@ -19,6 +19,8 @@ const COMMAND_SCOPED_FLAG_HELP: Record> = { key: '--key Opaque key from reference list; repeat for several' }, 'computer get-app-state': { + 'element-index': + '--element-index Read only this element and its descendants, including any the window snapshot cut off', query: '--query Show only elements whose line contains this text, with their ancestors' }, diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 33aeaf25c71..6cb5a77d842 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1065,6 +1065,12 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "ORCA computer get-app-state --app com.apple.finder --query Downloads --json", "```", "", + "A snapshot reads at most `truncation.maxNodes` elements and `maxDepth` levels. When it stops early, `truncation.truncated` is true and each cut-off place shows a `... N more not shown (snapshot limit)` line, indented like the children it stands for. To read them, pass the parent element's index (the nearest line above with less indentation) as `get-app-state --element-index `: it reads only that element and its descendants, with the full limit. Elements already shown keep their indexes and hidden ones get new ones, and you can act on them. Action results show the whole window again, so read the subtree again before acting on another hidden element.", + "", + "```text", + "ORCA computer get-app-state --app com.apple.Safari --element-index 312 --json", + "```", + "", "## Waiting", "", "Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout ` up to 120000).", diff --git a/src/cli/specs/computer.ts b/src/cli/specs/computer.ts index bd4ea9d5c8d..af216111fc6 100644 --- a/src/cli/specs/computer.ts +++ b/src/cli/specs/computer.ts @@ -44,8 +44,8 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [ path: ['computer', 'get-app-state'], summary: 'Capture a compact accessibility snapshot of an app', usage: - 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--json]', - allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'query', 'max-elements'] + 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--element-index ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--json]', + allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'element-index', 'query', 'max-elements'] }, { path: ['computer', 'wait'], diff --git a/src/main/computer/desktop-script/desktop-script-provider-client.ts b/src/main/computer/desktop-script/desktop-script-provider-client.ts index 1e00ec6f6aa..cb64132de77 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-client.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-client.ts @@ -40,6 +40,7 @@ import { isRuntimeHostUnavailable } from './runtime-host/desktop-script-runtime-host' import { DesktopScriptSnapshotStore } from './snapshots/desktop-script-snapshot-store' +import { readSubtreeSnapshot } from './snapshots/desktop-script-subtree-snapshot' import { normalizeBridgeApp, renderSnapshot } from './snapshots/desktop-script-snapshot-rendering' import { normalizeComputerActionResult } from '../action-contracts/computer-action-verification-normalization' import { RuntimeClientError } from '../runtime-client-error' @@ -119,6 +120,14 @@ export class DesktopScriptProviderClient { async snapshot(params: Record): Promise { const app = stringParam(params, 'app') + const rootElementIndex = optionalNumberParam(params, 'rootElementIndex') + if (rootElementIndex !== undefined) { + return await readSubtreeSnapshot(this.snapshotStore, (request) => this.callBridge(request), { + app, + rootElementIndex, + params + }) + } const response = await this.callBridge({ tool: 'get_app_state', app, diff --git a/src/main/computer/desktop-script/desktop-script-provider-types.ts b/src/main/computer/desktop-script/desktop-script-provider-types.ts index adcdb816766..662d9230323 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-types.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-types.ts @@ -77,6 +77,8 @@ export type BridgeSnapshot = { focusedElementId?: number | null selectedText?: string | null elements?: BridgeElement[] + /** Set by the client when treeLines show only this element's subtree. */ + rootElementIndex?: number } export type BridgeWindow = { @@ -128,6 +130,8 @@ export type BridgeRequest = { requestId?: number app?: string element?: BridgeElement + /** get_app_state renders only this element's subtree. */ + root?: BridgeElement fromElement?: BridgeElement toElement?: BridgeElement x?: number diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts b/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts index f16500f2fbe..7b32866b033 100644 --- a/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts +++ b/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts @@ -49,6 +49,9 @@ export function renderSnapshot( treeText, elementCount: snapshot.elements?.length ?? 0, focusedElementId, + ...(snapshot.rootElementIndex === undefined + ? {} + : { rootElementIndex: snapshot.rootElementIndex }), truncation: { truncated: snapshot.truncation?.truncated === true, maxNodes: snapshot.truncation?.maxNodes, @@ -125,6 +128,9 @@ function renderTreeText(snapshot: BridgeSnapshot): string { const lines = [ `App=${appRef} (pid ${snapshot.app.pid})`, `Window: "${sanitize(snapshot.windowTitle ?? snapshot.app.name)}", App: ${sanitize(snapshot.app.name)}.`, + ...(snapshot.rootElementIndex === undefined + ? [] + : [`Showing element ${snapshot.rootElementIndex} and its descendants.`]), '', ...(snapshot.treeLines ?? []) ] diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts new file mode 100644 index 00000000000..319685e8c91 --- /dev/null +++ b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + createDesktopScriptProviderClient, + mockBridgeResponse, + resetDesktopScriptProviderTestHarness, + sampleBridgeSnapshot, + sampleCapabilities +} from '../desktop-script-provider-test-harness' + +function bridgeSnapshot(treeLines: string[], elements: { index: number; runtimeId: number[] }[]) { + return { ...sampleBridgeSnapshot('Mail', ''), windowIndex: 0, treeLines, elements } +} + +describe('DesktopScriptProviderClient subtree reads', () => { + afterEach(resetDesktopScriptProviderTestHarness) + + it('keeps shown indexes, numbers hidden children after them, and resolves them for actions', async () => { + const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py') + mockBridgeResponse({ + ok: true, + snapshot: bridgeSnapshot( + ['0 frame Mail', '\t1 list Inbox', '\t\t2 button A', '\t\t... 1 more not shown'], + [ + { index: 0, runtimeId: [0] }, + { index: 1, runtimeId: [0, 0] }, + { index: 2, runtimeId: [0, 0, 0] } + ] + ) + }) + await client.snapshot({ app: 'Mail' }) + + let sentRoot: unknown + mockBridgeResponse( + { + ok: true, + snapshot: bridgeSnapshot( + ['0 list Inbox', '\t1 button A', '\t2 button B'], + [ + { index: 0, runtimeId: [0, 0] }, + { index: 1, runtimeId: [0, 0, 0] }, + { index: 2, runtimeId: [0, 0, 1] } + ] + ) + }, + (operation) => { + sentRoot = operation.root + } + ) + const subtree = await client.snapshot({ app: 'Mail', rootElementIndex: 1 }) + + expect(sentRoot).toMatchObject({ index: 1, runtimeId: [0, 0] }) + expect(subtree.snapshot.rootElementIndex).toBe(1) + expect(subtree.snapshot.treeText.split('\n').slice(2, 7)).toEqual([ + 'Showing element 1 and its descendants.', + '', + '1 list Inbox', + '\t2 button A', + '\t3 button B' + ]) + + let clicked: unknown + mockBridgeResponse({ ok: true, capabilities: sampleCapabilities() }) + mockBridgeResponse( + { ok: true, snapshot: bridgeSnapshot(['0 frame Mail'], [{ index: 0, runtimeId: [0] }]) }, + (operation) => { + clicked = operation.element + } + ) + await client.action('click', { app: 'Mail', elementIndex: 3 }) + expect(clicked).toMatchObject({ runtimeId: [0, 0, 1] }) + }) +}) diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts new file mode 100644 index 00000000000..4a569d6d26b --- /dev/null +++ b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts @@ -0,0 +1,103 @@ +import type { ComputerSnapshotResult } from '../../../../shared/computer-use-runtime-types' +import { renumberComputerTreeLine } from '../../../../shared/computer-use-tree-text' +import { RuntimeClientError } from '../../runtime-client-error' +import { elementParam } from '../actions/desktop-script-action' +import { optionalNumberParam } from '../desktop-script-provider-params' +import type { + BridgeElement, + BridgeRequest, + BridgeResponse, + BridgeSnapshot +} from '../desktop-script-provider-types' +import type { DesktopScriptSnapshotStore } from './desktop-script-snapshot-store' +import { renderSnapshot } from './desktop-script-snapshot-rendering' + +/** Reads one element's subtree of the window the store last saw, the target of --element-index. */ +export async function readSubtreeSnapshot( + store: DesktopScriptSnapshotStore, + callBridge: (request: BridgeRequest) => Promise, + { + app, + rootElementIndex, + params + }: { app: string; rootElementIndex: number; params: Record } +): Promise { + const base = store.current(app, optionalNumberParam(params, 'windowId'), params) + if (!base) { + throw new RuntimeClientError( + 'element_not_found', + '--element-index needs a get-app-state snapshot of this app/window first' + ) + } + const response = await callBridge({ + tool: 'get_app_state', + app, + root: elementParam(base, rootElementIndex), + windowId: base.windowId ?? undefined, + windowIndex: base.windowIndex ?? undefined, + noScreenshot: params.noScreenshot === true, + restoreWindow: params.restoreWindow === true + }) + if (!response.snapshot) { + throw new RuntimeClientError('accessibility_error', 'desktop provider returned no snapshot') + } + const { shown, cached } = indexSubtreeSnapshot(base, response.snapshot, rootElementIndex) + store.remember(app, cached, params) + return renderSnapshot(shown, params.noScreenshot === true) +} + +/** + * Gives a subtree read the indexes the window's cached snapshot already uses: elements it showed + * keep theirs, new ones get unused indexes. Returns what to show and what to cache, where the + * cache keeps the earlier elements so their indexes still resolve for actions. + */ +function indexSubtreeSnapshot( + base: BridgeSnapshot, + subtree: BridgeSnapshot, + rootElementIndex: number +): { shown: BridgeSnapshot; cached: BridgeSnapshot } { + const baseElements = base.elements ?? [] + const baseIndexByRuntimeId = new Map() + for (const element of baseElements) { + const key = runtimeIdKey(element) + if (key !== null && !baseIndexByRuntimeId.has(key)) { + baseIndexByRuntimeId.set(key, element.index) + } + } + let nextIndex = baseElements.reduce((max, element) => Math.max(max, element.index), -1) + 1 + const indexes = new Map() + const claimed = new Set() + for (const element of subtree.elements ?? []) { + const key = runtimeIdKey(element) + const known = key === null ? undefined : baseIndexByRuntimeId.get(key) + const index = known !== undefined && !claimed.has(known) ? known : nextIndex++ + claimed.add(index) + indexes.set(element.index, index) + } + const walked = (subtree.elements ?? []).map((element) => ({ + ...element, + index: indexes.get(element.index) ?? element.index + })) + const focusedElementId = + subtree.focusedElementId === null || subtree.focusedElementId === undefined + ? subtree.focusedElementId + : (indexes.get(subtree.focusedElementId) ?? null) + const shown: BridgeSnapshot = { + ...subtree, + treeLines: subtree.treeLines?.map((line) => renumberComputerTreeLine(line, indexes)), + focusedElementId, + elements: walked, + rootElementIndex + } + const cached: BridgeSnapshot = { + ...shown, + elements: [...baseElements.filter((element) => !claimed.has(element.index)), ...walked] + } + return { shown, cached } +} + +function runtimeIdKey(element: BridgeElement): string | null { + return Array.isArray(element.runtimeId) && element.runtimeId.length > 0 + ? JSON.stringify(element.runtimeId) + : null +} diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index 42de55e7b3c..7ae199c70e8 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -52,6 +52,8 @@ export type ComputerSnapshotData = { treeText: string elementCount: number focusedElementId: number | null + // Present only when treeText is the subtree of this element, read with rootElementIndex. + rootElementIndex?: number // Present only when treeText is a diff against the previous snapshot of the same window. treeDiff?: { baseSnapshotId: string diff --git a/src/shared/computer-use-tree-text.ts b/src/shared/computer-use-tree-text.ts index 44edee2de03..2e73c73e6b0 100644 --- a/src/shared/computer-use-tree-text.ts +++ b/src/shared/computer-use-tree-text.ts @@ -37,6 +37,16 @@ export function parseComputerTreeText(treeText: string): ComputerTreeText { return { header: lines.slice(0, first), elements, footer: lines.slice(footerStart) } } +/** Rewrites the index of an element line; other lines pass through. */ +export function renumberComputerTreeLine( + line: string, + indexes: ReadonlyMap +): string { + const match = ELEMENT_LINE.exec(line) + const index = match ? indexes.get(Number(match[2])) : undefined + return match && index !== undefined ? `${match[1]}${index}${line.slice(match[0].length)}` : line +} + export function joinComputerTreeText(header: string[], body: string[], footer: string[]): string { return [...trimTrailingBlank(header), '', ...body, ...footer].join('\n') } diff --git a/src/shared/rpc-contract/computer-schemas-params.ts b/src/shared/rpc-contract/computer-schemas-params.ts index 0f0a0f1fc00..6fb6d0bf2ac 100644 --- a/src/shared/rpc-contract/computer-schemas-params.ts +++ b/src/shared/rpc-contract/computer-schemas-params.ts @@ -56,7 +56,10 @@ export function validateComputerTarget( validateWindowTarget(value, ctx) } -export const ComputerObserveTarget = ComputerObserveTargetBase.superRefine(validateComputerTarget) +export const ComputerObserveTarget = ComputerObserveTargetBase.extend({ + // Reads one element's subtree. Older runtimes strip it, and their result has no rootElementIndex. + rootElementIndex: OptionalNonNegativeInt +}).superRefine(validateComputerTarget) export const ListApps = z.object({}).strict() From 8b2caa497cd1aded03b6206d46c70b94a246df42 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:24:53 -0800 Subject: [PATCH 03/16] test(terminal): isolate live zsh hook command lookup (#41678) --- ...-idle-zsh-preexec-hook-order.live-shell.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts b/src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts index 4fe65b9232d..b5b756c4cbf 100644 --- a/src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts +++ b/src/main/runtime/terminal/wait/tui-idle-zsh-preexec-hook-order.live-shell.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import * as pty from 'node-pty' import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { quotePosixShell } from '../../../../shared/shell/launch/shell-argument-quoting' import { getShellLaunchConfig } from '../../../providers/local-pty/local-pty-shell-ready' import { selectShellStartupFeatures } from '../../../pty/shell-startup/shell-startup-features' import { hasZsh, ZSH_PATH } from '../../../pty/shell-startup/zsh-startup-hook-pty-harness' @@ -102,9 +103,14 @@ async function captureLaunch(hookCase: HookCase): Promise { const root = mkdtempSync(join(tmpdir(), 'orca-preexec-order-')) const home = join(root, 'home') const bin = join(root, 'bin') + const fixturePath = `${bin}:/usr/bin:/bin` mkdirSync(home) mkdirSync(bin) - writeFileSync(join(home, '.zshrc'), hookCase.zshrc) + // Login profiles can reorder PATH, so restore the stand-in before the startup command runs. + writeFileSync( + join(home, '.zshrc'), + `export PATH=${quotePosixShell(fixturePath)}\n${hookCase.zshrc}` + ) const standIn = join(bin, 'claude') writeFileSync(standIn, standInClaude(hookCase.agentSetsRestTitle)) chmodSync(standIn, 0o755) @@ -129,12 +135,12 @@ async function captureLaunch(hookCase: HookCase): Promise { rows: 30, cwd: home, env: { - PATH: `${bin}:/usr/bin:/bin`, + ...launchEnv, + PATH: fixturePath, HOME: home, TERM: 'xterm-256color', LANG: 'en_US.UTF-8', - ORCA_TEST_HOOK_ORDER: hookOrderPath, - ...launchEnv + ORCA_TEST_HOOK_ORDER: hookOrderPath } }) const chunks: Chunk[] = [] From 8bb57668ed7dc033ce7a7af0021274fdd52a5bab Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:25:10 +0000 Subject: [PATCH 04/16] fix(computer): refuse quit, log out and lock shortcuts unless explicitly allowed (#41412) * fix(computer): refuse quit, log out and lock shortcuts unless explicitly allowed * fix(computer): close Linux keysym bypasses in the system shortcut guard * fix(computer): refuse Linux keycode, L1/L2 and VT-switch keysym bypasses * fix(computer): catch control characters in Linux keycode tokens and point recovery at the stated effect --- docs/site/content/docs/cli/computer-use.mdx | 2 + skill-guides/orca-computer-use.md | 2 + src/cli/computer/computer-action-flags.ts | 12 +- src/cli/discovery/flag-help-text.ts | 2 + src/cli/skills/bundled-skill-guides.ts | 2 + src/cli/specs/computer.ts | 8 +- .../computer-blocked-key-combos.test.ts | 77 ++++++++ .../computer-blocked-key-combos.ts | 177 ++++++++++++++++++ .../computer-provider-action-validation.ts | 14 +- src/shared/cli-argument-boundary.ts | 1 + src/shared/computer-use-error-recovery.ts | 5 + src/shared/computer-use-runtime-types.ts | 1 + .../rpc-contract/computer-schemas-params.ts | 6 +- 13 files changed, 296 insertions(+), 13 deletions(-) create mode 100644 src/main/computer/action-contracts/computer-blocked-key-combos.test.ts create mode 100644 src/main/computer/action-contracts/computer-blocked-key-combos.ts diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index 776ac5a21a2..5f39edea8e5 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -87,6 +87,8 @@ orca computer perform-secondary-action --app --element-index --action Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over raw `type-text` or `press-key` — they target accessibility elements directly and survive focus changes that keyboard input doesn't. +`hotkey` and `press-key` refuse shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session, such as `Cmd+Q`, `Shift+Cmd+Q`, `Ctrl+Cmd+Q` and `Option+Cmd+Esc` on macOS, `Alt+F4` and `Win+L` on Windows, and `Ctrl+Q`, `Alt+F4`, `Super+L` and `Ctrl+Alt+Delete` on Linux. The error code is `system_shortcut_blocked`. Pass `--allow-system-shortcut` to send one anyway. + On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web ``, use `set-value` on it; otherwise retry once with `--restore-window` after telling the user it will take over their screen. - `window_not_focused`: retry once with `--restore-window`; if the message says restore was already requested, stop retrying restore and bring the app forward manually or check permissions. For editable fields prefer `set-value`, then inspect before assuming keyboard input worked. diff --git a/src/cli/computer/computer-action-flags.ts b/src/cli/computer/computer-action-flags.ts index 3d5134617c9..bb80a69845d 100644 --- a/src/cli/computer/computer-action-flags.ts +++ b/src/cli/computer/computer-action-flags.ts @@ -145,24 +145,32 @@ export async function getComputerTextActionFlags(flags: Map): { key: string + allowSystemShortcut?: boolean } { const key = getRequiredStringFlag(flags, 'key') const message = computerUsePressKeyValidationMessage(key) if (message) { throw new RuntimeClientError('invalid_argument', message) } - return { key } + return { + key, + ...(flags.get('allow-system-shortcut') === true ? { allowSystemShortcut: true } : {}) + } } export function getComputerHotkeyActionFlags(flags: Map): { key: string + allowSystemShortcut?: boolean } { const key = getRequiredStringFlag(flags, 'key') const message = computerUseHotkeyValidationMessage(key) if (message) { throw new RuntimeClientError('invalid_argument', message) } - return { key } + return { + key, + ...(flags.get('allow-system-shortcut') === true ? { allowSystemShortcut: true } : {}) + } } export async function getComputerSetValueActionFlags( diff --git a/src/cli/discovery/flag-help-text.ts b/src/cli/discovery/flag-help-text.ts index 108cf43bf8f..de9456cb499 100644 --- a/src/cli/discovery/flag-help-text.ts +++ b/src/cli/discovery/flag-help-text.ts @@ -10,6 +10,8 @@ export const FLAG_HELP_TEXT: Record = { action: '--action Secondary accessibility action name', activate: '--activate Reveal the new worktree in the Orca app', app: '--app App name, bundle ID, or pid:N', + 'allow-system-shortcut': + '--allow-system-shortcut Send a quit, log out or lock shortcut that is refused by default', direction: '--direction Direction: up|down|left|right for scroll, horizontal|vertical for split', 'display-name': '--display-name Override the Orca display name', diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 6cb5a77d842..0bc7a527040 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1147,6 +1147,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.", "- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.", "- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.", + "- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.", "- Use `click --modifiers ` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.", "- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.", "", @@ -1207,6 +1208,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "", "- `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names.", "- `app_blocked`: stop; the target is intentionally blocked from computer-use.", + "- `system_shortcut_blocked`: the error message says what the shortcut would do. Reach the same goal with a semantic action, such as a button or menu item, or ask the user; retry with `--allow-system-shortcut` only when the user asked for that exact shortcut.", "- `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`.", "- `background_unavailable`: for a popup or `