diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index 4f6b8f31ae7..776ac5a21a2 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -37,7 +37,7 @@ orca computer click --app com.spotify.client --element-index 42 --json Element indexes are scoped to the latest `get-app-state` result and may be **sparse**. In `--json` output, read the tree from `result.snapshot.treeText`. Do not invent indexes from `elementCount`. Refresh state after navigation, focus changes, scrolling, or any app re-render before reusing an index. -Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query ` prints only matching elements and their ancestors, and `--max-elements ` caps the output. +Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query ` prints only matching elements and their ancestors, and `--max-elements ` caps the output. A window larger than the snapshot limit shows `... N more not shown (snapshot limit)` where the tree was cut; `get-app-state --element-index ` reads only that element's subtree, so pass the cut-off children's parent to see them. ## Waiting for the app diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index c81e3812534..5e5fa614185 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -487,13 +487,19 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome has_text = bool(item["name"] or item["automationId"] or item["value"]) return role in {"panel", "filler", "unknown", "section"} and not has_text and not meaningful_actions(item["actions"]) and summary is None and child_count <= 1 + def walk_children(child_items, depth, path): + for position, (child_index, child) in enumerate(child_items): + if len(records) >= MAX_NODES or depth > MAX_DEPTH: + truncation["truncated"] = True + if depth > MAX_DEPTH: + truncation["maxDepthReached"] = True + # Marks where the walk stopped, so the agent can read that parent's subtree by index. + lines.append(("\t" * depth) + f"... {len(child_items) - position} more not shown (snapshot limit)") + return + walk(child, depth, path + [child_index]) + def walk(node, depth, path): nonlocal next_index - if len(records) >= MAX_NODES or depth > MAX_DEPTH: - truncation["truncated"] = True - if depth > MAX_DEPTH: - truncation["maxDepthReached"] = True - return item = record(node, next_index, path) role_key = (item["controlType"] or "").lower() title = item["name"] or item["automationId"] or "" @@ -505,8 +511,7 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome if len(summary_values) >= 2 and is_plain_text_subtree(node): generic_summary = " ".join(summary_values) if should_elide(item, len(child_items), generic_summary): - for child_index, child in child_items: - walk(child, depth, path + [child_index]) + walk_children(child_items, depth, path) return records.append(item) next_index += 1 @@ -529,8 +534,7 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome if generic_summary or suppressed: return child_line_start = len(lines) - for child_index, child in child_items: - walk(child, depth + 1, path + [child_index]) + walk_children(child_items, depth + 1, path) if compact_browser_tabs: compact_rendered_browser_tabs(records, lines, child_line_start, depth + 1) @@ -1203,17 +1207,23 @@ def window_json(app, index, window): } -def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False): +def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False, root=None): app = find_app(query) window_index, window = choose_window(app, window_id, window_index) if restore: restore_window(app, window) window_index, window = choose_window(app, window_id, window_index) bounds = screen_rect(window) + tree_root, root_path = window, [window_index] + if root is not None: + tree_root = find_element(app, root) + if tree_root is None: + raise RuntimeError(f"stale element: element {root.get('index')} is gone; run get-app-state again and use a fresh element index") + root_path = list(root.get("runtimeId") or []) geometry_nodes = {} records, lines, truncation = render_accessibility_tree( - window, - [window_index], + tree_root, + root_path, compact_browser_tabs=is_browser_app(query, app), geometry_nodes=geometry_nodes, ) @@ -1875,6 +1885,7 @@ def run_operation(operation): operation.get("windowId"), operation.get("windowIndex"), bool(operation.get("restoreWindow")), + operation.get("root"), ), } diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py index 6e997611596..207d2de3410 100644 --- a/native/computer-use-linux/runtime_render_test.py +++ b/native/computer-use-linux/runtime_render_test.py @@ -699,6 +699,26 @@ class RuntimeRenderTest(unittest.TestCase): indexes = [item["index"] for item in snapshot["elements"]] self.assertEqual(len(indexes), len(set(indexes))) + def test_cut_tree_marks_hidden_children_and_root_reads_them(self): + items = [FakeAccessible("button", f"A{i}") for i in range(3)] + root = FakeAccessible("frame", "Main", children=[ + FakeAccessible("list", "Items", children=items), FakeAccessible("button", "After"), + ]) + with SnapshotFixture(root), patch.object(runtime, "MAX_NODES", 4): + snapshot = runtime.make_snapshot("Fixture", False) + self.assertEqual(snapshot["treeLines"], [ + "0 frame Main", + "\t1 list Items", + "\t\t2 button A0", + "\t\t3 button A1", + "\t\t... 1 more not shown (snapshot limit)", + "\t... 1 more not shown (snapshot limit)", + ]) + items_record = next(item for item in snapshot["elements"] if item["name"] == "Items") + subtree = runtime.run_operation({"tool": "get_app_state", "app": "Fixture", "noScreenshot": True, "root": items_record})["snapshot"] + self.assertEqual(subtree["treeLines"], ["0 list Items", "\t1 button A0", "\t2 button A1", "\t3 button A2"]) + self.assertEqual(subtree["elements"][-1]["runtimeId"], [0, 0, 2]) + diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 9a43b296403..1765b5803d9 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -143,6 +143,12 @@ final class ElementRecord { } } +/// A snapshot that read one element's subtree; its elements also keep the window's earlier ones. +struct SnapshotScope { + let rootIndex: Int + let walkedIndexes: Set +} + struct Snapshot { let id: String let app: AppDescriptor @@ -161,6 +167,7 @@ struct Snapshot { let nextFreshElementIndex: Int let truncated: Bool let maxDepthReached: Bool + let scope: SnapshotScope? /// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change. let appWindowIds: [CGWindowID] @@ -187,6 +194,7 @@ struct Snapshot { nextFreshElementIndex: nextFreshElementIndex, truncated: truncated, maxDepthReached: maxDepthReached, + scope: scope, appWindowIds: appWindowIds ) } @@ -267,7 +275,7 @@ final class Provider { ? try? cachedSnapshot(params: params) : nil do { - let after = try observe(params: params) + let after = try observeKeepingScope(params: params) guard let before, after.showsSameState(as: before) else { return renderActionResult(action: action, snapshot: after) } @@ -296,10 +304,10 @@ final class Provider { let deadline = Date().addingTimeInterval(1) while Date() < deadline { Thread.sleep(forTimeInterval: 0.2) - let polled = try observe(params: quietParams) + let polled = try observeKeepingScope(params: quietParams) if !polled.showsSameState(as: before) { if params["noScreenshot"]?.bool == true { return polled } - return try observe(params: settleParams) + return try observeKeepingScope(params: settleParams) } } return nil @@ -315,12 +323,14 @@ final class Provider { } let namespace = snapshotNamespace(params) pruneSnapshotCache() + let subtree = try requestedSubtree(params: params, app: app) let snapshot = try buildSnapshot( app: app, includeScreenshot: params["noScreenshot"]?.bool != true, - windowId: windowId, - windowIndex: windowIndex, + windowId: subtree.map(\.base.windowId) ?? windowId, + windowIndex: subtree == nil ? windowIndex : nil, restoreWindow: params["restoreWindow"]?.bool == true, + subtree: subtree, previousSnapshot: { [snapshots] resolvedWindowId in // Why: indexes carry over from this caller's own last view of exactly this window. snapshots[namespacedSnapshotKey(namespace, snapshotCanonicalWindowIdKey(resolvedWindowId))] @@ -339,6 +349,27 @@ final class Provider { return snapshot } + /// Resolves `rootElementIndex` against this caller's last snapshot of the window, whose indexes it extends. + private func requestedSubtree(params: [String: JSONValue], app: AppDescriptor) throws -> (root: ElementRecord, base: Snapshot)? { + guard let rootIndex = try optionalInteger(params, "rootElementIndex") else { return nil } + guard let base = try cachedSnapshot(params: params), base.app.pid == app.pid else { + throw ProviderError.coded("element_not_found", "--element-index needs a get-app-state snapshot of this app/window first") + } + return (try element(base, rootIndex), base) + } + + /// Re-reads the subtree the cached snapshot showed, so actions on its elements report that part of the window. + private func observeKeepingScope(params: [String: JSONValue]) throws -> Snapshot { + guard let rootIndex = (try? cachedSnapshot(params: params))?.scope?.rootIndex else { + return try observe(params: params) + } + do { + return try observe(params: params.merging(["rootElementIndex": .number(Double(rootIndex))]) { _, replacement in replacement }) + } catch let error as ProviderError where error.code == "element_not_found" { + return try observe(params: params) + } + } + private func rememberSnapshot( query: String, app: AppDescriptor, @@ -419,10 +450,16 @@ final class Provider { private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot { pruneSnapshotCache() let cached = try cachedSnapshot(params: params) + let requestedIndexes = try requestedElementIndexes(params) + var observeParams = params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement } + // Why: elements read with --element-index may lie past the whole-window node limit, so only their subtree can re-find them. + if let scope = cached?.scope, !requestedIndexes.isEmpty, requestedIndexes.allSatisfy(scope.walkedIndexes.contains) { + observeParams["rootElementIndex"] = .number(Double(scope.rootIndex)) + } // Why: cached AX frames can be stale after a window move or resize, and // stale geometry can turn an intended action into a misclick. - let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) - try validateRequestedElements(cached: cached, current: snapshot, params: params) + let snapshot = try observe(params: observeParams) + try validateRequestedElements(cached: cached, current: snapshot, requestedIndexes: requestedIndexes) return snapshot } @@ -473,11 +510,14 @@ final class Provider { (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key]) } - private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws { - let requestedIndexes = try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in + private func requestedElementIndexes(_ params: [String: JSONValue]) throws -> [Int] { + try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in guard params[key]?.number != nil else { return nil } return try optionalInteger(params, key) } + } + + private func validateRequestedElements(cached: Snapshot?, current: Snapshot, requestedIndexes: [Int]) throws { guard !requestedIndexes.isEmpty else { return } guard let cached else { throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window") @@ -658,6 +698,7 @@ final class Provider { windowId: CGWindowID?, windowIndex: Int?, restoreWindow: Bool, + subtree: (root: ElementRecord, base: Snapshot)?, previousSnapshot: (CGWindowID) -> Snapshot? ) throws -> Snapshot { guard accessibilityTrustedSettled() else { @@ -703,11 +744,26 @@ final class Provider { focused: focusedElement(appElement: appElement), compactBrowserTabs: app.isKnownBrowser ) - renderer.render(window) + if let subtree { + guard let ancestors = ancestorChain(of: subtree.root.element, in: window) else { + throw ProviderError.coded("element_not_found", "element \(subtree.root.index) is no longer in this window; run get-app-state again and use a fresh element index") + } + renderer.render(subtree.root.element, ancestors: ancestors) + } else { + renderer.render(window) + } if renderer.appStoppedResponding { throw AccessibilityMessaging.appNotResponding(app.name) } - let indexes = renderer.carryOverIndexes(from: previousSnapshot(capture.windowId)) + let previous = subtree?.base ?? previousSnapshot(capture.windowId) + let indexes = renderer.carryOverIndexes(from: previous) + var elements = renderer.records + var elementOrder = indexes.order + if let subtree { + // Why: elements outside the subtree keep resolving, so the agent can still act on what the whole-window read showed. + elements = subtree.base.elements.merging(renderer.records) { _, walked in walked } + elementOrder = subtree.base.elementOrder.filter { renderer.records[$0] == nil } + indexes.order + } let screenshot = includeScreenshot ? capture.screenshotPayload() : nil let screenshotStatus: ScreenshotStatus = if screenshot != nil { .captured @@ -725,17 +781,18 @@ final class Provider { windowBounds: capture.bounds, windowId: capture.windowId, windowLayer: capture.layer, - treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary), + treeText: renderTreeText(app: app, title: title, rootIndex: subtree?.root.index, lines: renderer.lines, focused: renderer.focusedSummary), focusedElementId: renderer.focusedElementId, screenshot: screenshot, screenshotStatus: screenshotStatus, screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds), screenshotEngine: capture.image?.engine, - elements: renderer.records, - elementOrder: indexes.order, + elements: elements, + elementOrder: elementOrder, nextFreshElementIndex: indexes.nextFreshIndex, truncated: renderer.truncated, maxDepthReached: renderer.maxDepthReached, + scope: subtree.map { SnapshotScope(rootIndex: $0.root.index, walkedIndexes: Set(indexes.order)) }, appWindowIds: windowCandidates.map(\.windowId).sorted() ) } @@ -751,39 +808,43 @@ final class Provider { "scale": payload.scale, ] } - return [ - "snapshot": [ - "id": snapshot.id, - "app": [ - "name": snapshot.app.name, - "bundleId": jsonNullable(snapshot.app.bundleId), - "pid": Int(snapshot.app.pid), - ], - "window": [ - "id": Int(snapshot.windowId), - "title": snapshot.windowTitle, - "x": Int(snapshot.windowBounds.origin.x.rounded()), - "y": Int(snapshot.windowBounds.origin.y.rounded()), - "width": Int(snapshot.windowBounds.width.rounded()), - "height": Int(snapshot.windowBounds.height.rounded()), - "isMinimized": false, - "isOffscreen": false, - "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), - "platform": [ - "layer": snapshot.windowLayer, - ], - ], - "coordinateSpace": "window", - "treeText": snapshot.treeText, - "elementCount": snapshot.elements.count, - "focusedElementId": snapshot.focusedElementId as Any, - "truncation": [ - "truncated": snapshot.truncated, - "maxNodes": TreeRenderer.maxNodes, - "maxDepth": TreeRenderer.maxDepth, - "maxDepthReached": snapshot.maxDepthReached, + var snapshotPayload: [String: Any] = [ + "id": snapshot.id, + "app": [ + "name": snapshot.app.name, + "bundleId": jsonNullable(snapshot.app.bundleId), + "pid": Int(snapshot.app.pid), + ], + "window": [ + "id": Int(snapshot.windowId), + "title": snapshot.windowTitle, + "x": Int(snapshot.windowBounds.origin.x.rounded()), + "y": Int(snapshot.windowBounds.origin.y.rounded()), + "width": Int(snapshot.windowBounds.width.rounded()), + "height": Int(snapshot.windowBounds.height.rounded()), + "isMinimized": false, + "isOffscreen": false, + "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), + "platform": [ + "layer": snapshot.windowLayer, ], ], + "coordinateSpace": "window", + "treeText": snapshot.treeText, + "elementCount": snapshot.scope?.walkedIndexes.count ?? snapshot.elements.count, + "focusedElementId": snapshot.focusedElementId as Any, + "truncation": [ + "truncated": snapshot.truncated, + "maxNodes": TreeRenderer.maxNodes, + "maxDepth": TreeRenderer.maxDepth, + "maxDepthReached": snapshot.maxDepthReached, + ], + ] + if let scope = snapshot.scope { + snapshotPayload["rootElementIndex"] = scope.rootIndex + } + return [ + "snapshot": snapshotPayload, "screenshot": screenshot, "screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot), ] @@ -1462,6 +1523,18 @@ private func pointerRecipient( return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)) } +/// The element's parents from `window` down, or nil when it is no longer inside `window`. +private func ancestorChain(of element: AXUIElement, in window: AXUIElement) -> [AXUIElement]? { + var chain: [AXUIElement] = [] + var current = element + while !CFEqual(current, window) { + guard chain.count < 256, let parent = copyElement(current, kAXParentAttribute as String) else { return nil } + chain.append(parent) + current = parent + } + return chain.reversed() +} + private func containingWindow(_ element: AXUIElement) -> AXUIElement? { var current = element for _ in 0..<64 { @@ -2179,12 +2252,15 @@ private func mouseButton(_ raw: String?) throws -> MouseButtonSelection { } } -private func renderTreeText(app: AppDescriptor, title: String, bounds: CGRect, lines: [String], focused: String?) -> String { +private func renderTreeText(app: AppDescriptor, title: String, rootIndex: Int?, lines: [String], focused: String?) -> String { var output = [ "App=\(app.bundleId ?? app.name.replacingOccurrences(of: " ", with: "_")) (pid \(app.pid))", "Window: \"\(sanitize(title))\", App: \(sanitize(app.name)).", - "", ] + if let rootIndex { + output.append("Showing element \(rootIndex) and its descendants.") + } + output.append("") output.append(contentsOf: lines) output.append("") output.append(focused.map { "The focused UI element is \($0)." } ?? "No UI element is currently focused.") @@ -2231,15 +2307,6 @@ private final class TreeRenderer { func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) { guard !reader.appStoppedResponding else { return } - guard nextIndex < Self.maxNodes else { - truncated = true - return - } - guard depth < Self.maxDepth else { - truncated = true - maxDepthReached = true - return - } guard !ancestors.contains(where: { CFEqual($0, element) }) else { return } let role = reader.stringAttribute(element, kAXRoleAttribute as String) ?? "AXUnknown" @@ -2290,9 +2357,7 @@ private final class TreeRenderer { webAreaDepth: webAreaDepth ) if SnapshotRenderHeuristics.shouldElide(node) { - for child in children { - render(child, depth: depth, ancestors: ancestors + [element]) - } + renderChildren(children, depth: depth, ancestors: ancestors + [element]) return } @@ -2315,9 +2380,8 @@ private final class TreeRenderer { return } if compactBrowserTabs, let tabStripCompaction = tabStripCompaction(parent: node, children: children) { - for (childIndex, child) in children.enumerated() where tabStripCompaction.retainedIndexes.contains(childIndex) { - render(child, depth: depth + 1, ancestors: ancestors + [element]) - } + let retained = children.enumerated().filter { tabStripCompaction.retainedIndexes.contains($0.offset) }.map(\.element) + renderChildren(retained, depth: depth + 1, ancestors: ancestors + [element]) lines.append( String(repeating: "\t", count: depth + 1) + "... \(tabStripCompaction.omittedCount) inactive browser tabs omitted" @@ -2325,14 +2389,25 @@ private final class TreeRenderer { return } let childLineStart = lines.count - for child in children { - render(child, depth: depth + 1, ancestors: ancestors + [element]) - } + renderChildren(children, depth: depth + 1, ancestors: ancestors + [element]) if compactBrowserTabs { compactRenderedBrowserTabs(parent: node, startLine: childLineStart, depth: depth + 1) } } + /// Stops at the node or depth limit with a note of how many children were left, so the agent can read that subtree. + private func renderChildren(_ children: [AXUIElement], depth: Int, ancestors: [AXUIElement]) { + for (position, child) in children.enumerated() { + guard nextIndex < Self.maxNodes, depth < Self.maxDepth else { + truncated = true + maxDepthReached = maxDepthReached || depth >= Self.maxDepth + lines.append(String(repeating: "\t", count: depth) + "... \(children.count - position) more not shown (snapshot limit)") + return + } + render(child, depth: depth, ancestors: ancestors) + } + } + private func tabStripCompaction(parent: SnapshotRenderNode, children: [AXUIElement]) -> SnapshotTabStripCompaction? { let childNodes = children.map { child in let role = reader.stringAttribute(child, kAXRoleAttribute as String) ?? "AXUnknown" diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index b8d86341d86..f4e5d1fc62e 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -794,12 +794,20 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = # Tab compaction shrinks records mid-walk, so $records.Count would reuse retained indexes. $nextIndex = [ref]0 - function Visit-OrcaNode($Node, [int]$Depth) { - if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) { - $truncation.truncated = $true - if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true } - return + function Visit-OrcaChildren($Children, [int]$Depth) { + for ($i = 0; $i -lt $Children.Count; $i++) { + if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) { + $truncation.truncated = $true + if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true } + # Marks where the walk stopped, so the agent can read that parent's subtree by index. + $lines.Add(("`t" * $Depth) + "... $($Children.Count - $i) more not shown (snapshot limit)") + return + } + Visit-OrcaNode $Children.Item($i) $Depth } + } + + function Visit-OrcaNode($Node, [int]$Depth) { $identity = try { (@($Node.GetRuntimeId()) -join ".") } catch { [Guid]::NewGuid().ToString() } if (-not $seen.Add($identity)) { return } @@ -820,9 +828,7 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = } } if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) { - for ($i = 0; $i -lt $children.Count; $i++) { - Visit-OrcaNode $children.Item($i) $Depth - } + Visit-OrcaChildren $children $Depth return } @@ -847,9 +853,7 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = if (-not [string]::IsNullOrWhiteSpace($genericSummary) -or (Test-OrcaSuppressChildren $roleKey $title $record.value $genericSummary)) { return } $childLineStart = $lines.Count - for ($i = 0; $i -lt $children.Count; $i++) { - Visit-OrcaNode $children.Item($i) ($Depth + 1) - } + Visit-OrcaChildren $children ($Depth + 1) if ($CompactBrowserTabs) { Compress-OrcaRenderedBrowserTabs $records $lines $childLineStart ($Depth + 1) } @@ -1025,14 +1029,21 @@ function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame, [IntPtr]$Win } } -function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false) { +function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) { $process = Find-OrcaProcess $Query Assert-OrcaWindowTarget $process $WindowId $WindowIndex # Root first: its hang probe must run before ShowWindow, which blocks on a hung window. $root = Get-OrcaRootElement $process if ($RestoreWindow) { Restore-OrcaWindow $process } $windowFrame = Get-OrcaWindowFrame $process $root - $tree = Render-OrcaTree $root $windowFrame (Test-OrcaBrowserProcess $process) + $walkRoot = $root + if ($null -ne $TreeRoot) { + $walkRoot = Find-OrcaElement $root $TreeRoot + if ($null -eq $walkRoot) { + throw "stale element: element $($TreeRoot.index) is gone; run get-app-state again and use a fresh element index" + } + } + $tree = Render-OrcaTree $walkRoot $windowFrame (Test-OrcaBrowserProcess $process) $screenshot = Get-OrcaScreenshot $IncludeScreenshot $windowFrame ([IntPtr]$process.MainWindowHandle) [pscustomobject]@{ @@ -1452,7 +1463,7 @@ function Invoke-OrcaOperation($Operation) { return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) } } if ($Operation.tool -eq "get_app_state") { - return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) } + return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) $Operation.root } } $process = Find-OrcaProcess $Operation.app diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 4e1506d0e88..835ed4ace19 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -46,6 +46,12 @@ In a large window, look for one control with `get-app-state --query `: it ORCA computer get-app-state --app com.apple.finder --query Downloads --json ``` +A snapshot reads at most `truncation.maxNodes` elements and `maxDepth` levels. When it stops early, `truncation.truncated` is true and each cut-off place shows a `... N more not shown (snapshot limit)` line, indented like the children it stands for. To read them, pass the parent element's index (the nearest line above with less indentation) as `get-app-state --element-index `: it reads only that element and its descendants, with the full limit. Elements already shown keep their indexes, hidden ones get new ones, and you can act on both. On macOS, acting on an element from that read returns that subtree again; on Linux and Windows, actions return the whole window with fresh indexes. + +```text +ORCA computer get-app-state --app com.apple.Safari --element-index 312 --json +``` + ## Waiting Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout ` up to 120000). diff --git a/src/cli/computer/computer-format.ts b/src/cli/computer/computer-format.ts index a00f42f2ce7..f13b05919d8 100644 --- a/src/cli/computer/computer-format.ts +++ b/src/cli/computer/computer-format.ts @@ -31,7 +31,7 @@ export function formatGetAppState(result: ComputerSnapshotResult): string { ? '' : ` @ ${result.snapshot.window.x},${result.snapshot.window.y}` const truncation = result.snapshot.truncation?.truncated - ? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'})` + ? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'}); read cut-off children with --element-index ` : ' Truncated: no' return [ `${app.name} (pid ${app.pid}${bundle})`, diff --git a/src/cli/computer/computer.ts b/src/cli/computer/computer.ts index 7a5745c5ec1..cdd3082dfd0 100644 --- a/src/cli/computer/computer.ts +++ b/src/cli/computer/computer.ts @@ -7,6 +7,7 @@ import type { } from '../../shared/computer-use-runtime-types' import type { CommandHandler } from '../dispatch/dispatch' import { + getOptionalNonNegativeIntegerFlag, getOptionalPositiveIntegerFlag, getOptionalStringFlag, getRequiredStringFlag @@ -85,11 +86,23 @@ export const COMPUTER_HANDLERS: Record = { const observeFlags = getComputerObserveFlags(flags) const query = getOptionalStringFlag(flags, 'query') const maxElements = getOptionalPositiveIntegerFlag(flags, 'max-elements') + const rootElementIndex = getOptionalNonNegativeIntegerFlag(flags, 'element-index') const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.getAppState', { ...target, - ...observeFlags + ...observeFlags, + ...(rootElementIndex === undefined ? {} : { rootElementIndex }) }) + // Why: older runtimes drop the param and return the whole window, which would look like the subtree. + if ( + rootElementIndex !== undefined && + result.result.snapshot.rootElementIndex !== rootElementIndex + ) { + throw new RuntimeClientError( + 'unsupported_capability', + 'This Orca runtime cannot read one element subtree; update Orca, or run get-app-state without --element-index' + ) + } if (query !== undefined || maxElements !== undefined) { const { snapshot } = result.result snapshot.treeText = queryComputerTreeText(snapshot.treeText, { query, maxElements }) diff --git a/src/cli/discovery/command-scoped-flag-help.ts b/src/cli/discovery/command-scoped-flag-help.ts index b23764dd9f4..ba18dc9b3c3 100644 --- a/src/cli/discovery/command-scoped-flag-help.ts +++ b/src/cli/discovery/command-scoped-flag-help.ts @@ -19,6 +19,8 @@ const COMMAND_SCOPED_FLAG_HELP: Record> = { key: '--key Opaque key from reference list; repeat for several' }, 'computer get-app-state': { + 'element-index': + '--element-index Read only this element and its descendants, including any the window snapshot cut off', query: '--query Show only elements whose line contains this text, with their ancestors' }, diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 33aeaf25c71..27447c379dc 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1065,6 +1065,12 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "ORCA computer get-app-state --app com.apple.finder --query Downloads --json", "```", "", + "A snapshot reads at most `truncation.maxNodes` elements and `maxDepth` levels. When it stops early, `truncation.truncated` is true and each cut-off place shows a `... N more not shown (snapshot limit)` line, indented like the children it stands for. To read them, pass the parent element's index (the nearest line above with less indentation) as `get-app-state --element-index `: it reads only that element and its descendants, with the full limit. Elements already shown keep their indexes, hidden ones get new ones, and you can act on both. On macOS, acting on an element from that read returns that subtree again; on Linux and Windows, actions return the whole window with fresh indexes.", + "", + "```text", + "ORCA computer get-app-state --app com.apple.Safari --element-index 312 --json", + "```", + "", "## Waiting", "", "Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout ` up to 120000).", diff --git a/src/cli/specs/computer.ts b/src/cli/specs/computer.ts index bd4ea9d5c8d..af216111fc6 100644 --- a/src/cli/specs/computer.ts +++ b/src/cli/specs/computer.ts @@ -44,8 +44,8 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [ path: ['computer', 'get-app-state'], summary: 'Capture a compact accessibility snapshot of an app', usage: - 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--json]', - allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'query', 'max-elements'] + 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--element-index ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--json]', + allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'element-index', 'query', 'max-elements'] }, { path: ['computer', 'wait'], diff --git a/src/main/computer/desktop-script/desktop-script-provider-client.ts b/src/main/computer/desktop-script/desktop-script-provider-client.ts index 1e00ec6f6aa..cb64132de77 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-client.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-client.ts @@ -40,6 +40,7 @@ import { isRuntimeHostUnavailable } from './runtime-host/desktop-script-runtime-host' import { DesktopScriptSnapshotStore } from './snapshots/desktop-script-snapshot-store' +import { readSubtreeSnapshot } from './snapshots/desktop-script-subtree-snapshot' import { normalizeBridgeApp, renderSnapshot } from './snapshots/desktop-script-snapshot-rendering' import { normalizeComputerActionResult } from '../action-contracts/computer-action-verification-normalization' import { RuntimeClientError } from '../runtime-client-error' @@ -119,6 +120,14 @@ export class DesktopScriptProviderClient { async snapshot(params: Record): Promise { const app = stringParam(params, 'app') + const rootElementIndex = optionalNumberParam(params, 'rootElementIndex') + if (rootElementIndex !== undefined) { + return await readSubtreeSnapshot(this.snapshotStore, (request) => this.callBridge(request), { + app, + rootElementIndex, + params + }) + } const response = await this.callBridge({ tool: 'get_app_state', app, diff --git a/src/main/computer/desktop-script/desktop-script-provider-types.ts b/src/main/computer/desktop-script/desktop-script-provider-types.ts index adcdb816766..662d9230323 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-types.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-types.ts @@ -77,6 +77,8 @@ export type BridgeSnapshot = { focusedElementId?: number | null selectedText?: string | null elements?: BridgeElement[] + /** Set by the client when treeLines show only this element's subtree. */ + rootElementIndex?: number } export type BridgeWindow = { @@ -128,6 +130,8 @@ export type BridgeRequest = { requestId?: number app?: string element?: BridgeElement + /** get_app_state renders only this element's subtree. */ + root?: BridgeElement fromElement?: BridgeElement toElement?: BridgeElement x?: number diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts b/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts index f16500f2fbe..7b32866b033 100644 --- a/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts +++ b/src/main/computer/desktop-script/snapshots/desktop-script-snapshot-rendering.ts @@ -49,6 +49,9 @@ export function renderSnapshot( treeText, elementCount: snapshot.elements?.length ?? 0, focusedElementId, + ...(snapshot.rootElementIndex === undefined + ? {} + : { rootElementIndex: snapshot.rootElementIndex }), truncation: { truncated: snapshot.truncation?.truncated === true, maxNodes: snapshot.truncation?.maxNodes, @@ -125,6 +128,9 @@ function renderTreeText(snapshot: BridgeSnapshot): string { const lines = [ `App=${appRef} (pid ${snapshot.app.pid})`, `Window: "${sanitize(snapshot.windowTitle ?? snapshot.app.name)}", App: ${sanitize(snapshot.app.name)}.`, + ...(snapshot.rootElementIndex === undefined + ? [] + : [`Showing element ${snapshot.rootElementIndex} and its descendants.`]), '', ...(snapshot.treeLines ?? []) ] diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts new file mode 100644 index 00000000000..319685e8c91 --- /dev/null +++ b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + createDesktopScriptProviderClient, + mockBridgeResponse, + resetDesktopScriptProviderTestHarness, + sampleBridgeSnapshot, + sampleCapabilities +} from '../desktop-script-provider-test-harness' + +function bridgeSnapshot(treeLines: string[], elements: { index: number; runtimeId: number[] }[]) { + return { ...sampleBridgeSnapshot('Mail', ''), windowIndex: 0, treeLines, elements } +} + +describe('DesktopScriptProviderClient subtree reads', () => { + afterEach(resetDesktopScriptProviderTestHarness) + + it('keeps shown indexes, numbers hidden children after them, and resolves them for actions', async () => { + const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py') + mockBridgeResponse({ + ok: true, + snapshot: bridgeSnapshot( + ['0 frame Mail', '\t1 list Inbox', '\t\t2 button A', '\t\t... 1 more not shown'], + [ + { index: 0, runtimeId: [0] }, + { index: 1, runtimeId: [0, 0] }, + { index: 2, runtimeId: [0, 0, 0] } + ] + ) + }) + await client.snapshot({ app: 'Mail' }) + + let sentRoot: unknown + mockBridgeResponse( + { + ok: true, + snapshot: bridgeSnapshot( + ['0 list Inbox', '\t1 button A', '\t2 button B'], + [ + { index: 0, runtimeId: [0, 0] }, + { index: 1, runtimeId: [0, 0, 0] }, + { index: 2, runtimeId: [0, 0, 1] } + ] + ) + }, + (operation) => { + sentRoot = operation.root + } + ) + const subtree = await client.snapshot({ app: 'Mail', rootElementIndex: 1 }) + + expect(sentRoot).toMatchObject({ index: 1, runtimeId: [0, 0] }) + expect(subtree.snapshot.rootElementIndex).toBe(1) + expect(subtree.snapshot.treeText.split('\n').slice(2, 7)).toEqual([ + 'Showing element 1 and its descendants.', + '', + '1 list Inbox', + '\t2 button A', + '\t3 button B' + ]) + + let clicked: unknown + mockBridgeResponse({ ok: true, capabilities: sampleCapabilities() }) + mockBridgeResponse( + { ok: true, snapshot: bridgeSnapshot(['0 frame Mail'], [{ index: 0, runtimeId: [0] }]) }, + (operation) => { + clicked = operation.element + } + ) + await client.action('click', { app: 'Mail', elementIndex: 3 }) + expect(clicked).toMatchObject({ runtimeId: [0, 0, 1] }) + }) +}) diff --git a/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts new file mode 100644 index 00000000000..4a569d6d26b --- /dev/null +++ b/src/main/computer/desktop-script/snapshots/desktop-script-subtree-snapshot.ts @@ -0,0 +1,103 @@ +import type { ComputerSnapshotResult } from '../../../../shared/computer-use-runtime-types' +import { renumberComputerTreeLine } from '../../../../shared/computer-use-tree-text' +import { RuntimeClientError } from '../../runtime-client-error' +import { elementParam } from '../actions/desktop-script-action' +import { optionalNumberParam } from '../desktop-script-provider-params' +import type { + BridgeElement, + BridgeRequest, + BridgeResponse, + BridgeSnapshot +} from '../desktop-script-provider-types' +import type { DesktopScriptSnapshotStore } from './desktop-script-snapshot-store' +import { renderSnapshot } from './desktop-script-snapshot-rendering' + +/** Reads one element's subtree of the window the store last saw, the target of --element-index. */ +export async function readSubtreeSnapshot( + store: DesktopScriptSnapshotStore, + callBridge: (request: BridgeRequest) => Promise, + { + app, + rootElementIndex, + params + }: { app: string; rootElementIndex: number; params: Record } +): Promise { + const base = store.current(app, optionalNumberParam(params, 'windowId'), params) + if (!base) { + throw new RuntimeClientError( + 'element_not_found', + '--element-index needs a get-app-state snapshot of this app/window first' + ) + } + const response = await callBridge({ + tool: 'get_app_state', + app, + root: elementParam(base, rootElementIndex), + windowId: base.windowId ?? undefined, + windowIndex: base.windowIndex ?? undefined, + noScreenshot: params.noScreenshot === true, + restoreWindow: params.restoreWindow === true + }) + if (!response.snapshot) { + throw new RuntimeClientError('accessibility_error', 'desktop provider returned no snapshot') + } + const { shown, cached } = indexSubtreeSnapshot(base, response.snapshot, rootElementIndex) + store.remember(app, cached, params) + return renderSnapshot(shown, params.noScreenshot === true) +} + +/** + * Gives a subtree read the indexes the window's cached snapshot already uses: elements it showed + * keep theirs, new ones get unused indexes. Returns what to show and what to cache, where the + * cache keeps the earlier elements so their indexes still resolve for actions. + */ +function indexSubtreeSnapshot( + base: BridgeSnapshot, + subtree: BridgeSnapshot, + rootElementIndex: number +): { shown: BridgeSnapshot; cached: BridgeSnapshot } { + const baseElements = base.elements ?? [] + const baseIndexByRuntimeId = new Map() + for (const element of baseElements) { + const key = runtimeIdKey(element) + if (key !== null && !baseIndexByRuntimeId.has(key)) { + baseIndexByRuntimeId.set(key, element.index) + } + } + let nextIndex = baseElements.reduce((max, element) => Math.max(max, element.index), -1) + 1 + const indexes = new Map() + const claimed = new Set() + for (const element of subtree.elements ?? []) { + const key = runtimeIdKey(element) + const known = key === null ? undefined : baseIndexByRuntimeId.get(key) + const index = known !== undefined && !claimed.has(known) ? known : nextIndex++ + claimed.add(index) + indexes.set(element.index, index) + } + const walked = (subtree.elements ?? []).map((element) => ({ + ...element, + index: indexes.get(element.index) ?? element.index + })) + const focusedElementId = + subtree.focusedElementId === null || subtree.focusedElementId === undefined + ? subtree.focusedElementId + : (indexes.get(subtree.focusedElementId) ?? null) + const shown: BridgeSnapshot = { + ...subtree, + treeLines: subtree.treeLines?.map((line) => renumberComputerTreeLine(line, indexes)), + focusedElementId, + elements: walked, + rootElementIndex + } + const cached: BridgeSnapshot = { + ...shown, + elements: [...baseElements.filter((element) => !claimed.has(element.index)), ...walked] + } + return { shown, cached } +} + +function runtimeIdKey(element: BridgeElement): string | null { + return Array.isArray(element.runtimeId) && element.runtimeId.length > 0 + ? JSON.stringify(element.runtimeId) + : null +} diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index 42de55e7b3c..7ae199c70e8 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -52,6 +52,8 @@ export type ComputerSnapshotData = { treeText: string elementCount: number focusedElementId: number | null + // Present only when treeText is the subtree of this element, read with rootElementIndex. + rootElementIndex?: number // Present only when treeText is a diff against the previous snapshot of the same window. treeDiff?: { baseSnapshotId: string diff --git a/src/shared/computer-use-tree-text.ts b/src/shared/computer-use-tree-text.ts index 44edee2de03..2e73c73e6b0 100644 --- a/src/shared/computer-use-tree-text.ts +++ b/src/shared/computer-use-tree-text.ts @@ -37,6 +37,16 @@ export function parseComputerTreeText(treeText: string): ComputerTreeText { return { header: lines.slice(0, first), elements, footer: lines.slice(footerStart) } } +/** Rewrites the index of an element line; other lines pass through. */ +export function renumberComputerTreeLine( + line: string, + indexes: ReadonlyMap +): string { + const match = ELEMENT_LINE.exec(line) + const index = match ? indexes.get(Number(match[2])) : undefined + return match && index !== undefined ? `${match[1]}${index}${line.slice(match[0].length)}` : line +} + export function joinComputerTreeText(header: string[], body: string[], footer: string[]): string { return [...trimTrailingBlank(header), '', ...body, ...footer].join('\n') } diff --git a/src/shared/rpc-contract/computer-schemas-params.ts b/src/shared/rpc-contract/computer-schemas-params.ts index 0f0a0f1fc00..6fb6d0bf2ac 100644 --- a/src/shared/rpc-contract/computer-schemas-params.ts +++ b/src/shared/rpc-contract/computer-schemas-params.ts @@ -56,7 +56,10 @@ export function validateComputerTarget( validateWindowTarget(value, ctx) } -export const ComputerObserveTarget = ComputerObserveTargetBase.superRefine(validateComputerTarget) +export const ComputerObserveTarget = ComputerObserveTargetBase.extend({ + // Reads one element's subtree. Older runtimes strip it, and their result has no rootElementIndex. + rootElementIndex: OptionalNonNegativeInt +}).superRefine(validateComputerTarget) export const ListApps = z.object({}).strict()