mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
feat(computer): return tree diffs after actions and add get-app-state --query (#36023)
* feat(computer): send tree diffs after actions and add get-app-state --query Action results now carry only the accessibility-tree lines that changed since the caller's last view of the same window (~ changed, + added, - removed as index ranges, or an explicit no-change line), plus the focused element. The runtime sidecar keeps the last tree per session/worktree and app, so all three providers get it. The CLI opts in with treeDiff; older runtimes strip the param and keep returning the full tree, and --full-tree opts out. The macOS helper now keeps an element's index across snapshots of a window: AX identity first, then the next unclaimed index with the same signature for views that rebuild their AX objects, and never reuses a removed index. get-app-state --query prints matching elements with their ancestors and --max-elements caps the output, both in the CLI over the rendered tree. Custom action names with raw newlines no longer split an element over lines. * fix(computer): preserve parent changes in snapshot diffs * fix(computer): refuse stale index reuse for lookalike controls * fix(computer): print full trees in action replies * fix(computer): keep controls located in snapshot replies
This commit is contained in:
Kelvin Amoaba
@@ -37,6 +37,8 @@ orca computer click --app com.spotify.client --element-index 42 --json
|
||||
|
||||
Element indexes are scoped to the latest `get-app-state` result and may be **sparse**. In `--json` output, read the tree from `result.snapshot.treeText`. Do not invent indexes from `elementCount`. Refresh state after navigation, focus changes, scrolling, or any app re-render before reusing an index.
|
||||
|
||||
Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query <text>` prints only matching elements and their ancestors, and `--max-elements <n>` caps the output.
|
||||
|
||||
## Multi-window apps
|
||||
|
||||
```bash
|
||||
|
||||
@@ -157,6 +157,8 @@ struct Snapshot {
|
||||
let screenshotScale: CGSize
|
||||
let screenshotEngine: String?
|
||||
let elements: [Int: ElementRecord]
|
||||
let elementOrder: [Int]
|
||||
let nextFreshElementIndex: Int
|
||||
let truncated: Bool
|
||||
let maxDepthReached: Bool
|
||||
/// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change.
|
||||
@@ -181,6 +183,8 @@ struct Snapshot {
|
||||
screenshotScale: CGSize(width: 1, height: 1),
|
||||
screenshotEngine: nil,
|
||||
elements: elements,
|
||||
elementOrder: elementOrder,
|
||||
nextFreshElementIndex: nextFreshElementIndex,
|
||||
truncated: truncated,
|
||||
maxDepthReached: maxDepthReached,
|
||||
appWindowIds: appWindowIds
|
||||
@@ -309,12 +313,19 @@ final class Provider {
|
||||
if params["restoreWindow"]?.bool == true {
|
||||
recoverWindow(app)
|
||||
}
|
||||
let namespace = snapshotNamespace(params)
|
||||
pruneSnapshotCache()
|
||||
let snapshot = try buildSnapshot(
|
||||
app: app,
|
||||
includeScreenshot: params["noScreenshot"]?.bool != true,
|
||||
windowId: windowId,
|
||||
windowIndex: windowIndex,
|
||||
restoreWindow: params["restoreWindow"]?.bool == true
|
||||
restoreWindow: params["restoreWindow"]?.bool == true,
|
||||
previousSnapshot: { [snapshots] resolvedWindowId in
|
||||
// Why: indexes carry over from this caller's own last view of exactly this window.
|
||||
snapshots[namespacedSnapshotKey(namespace, snapshotCanonicalWindowIdKey(resolvedWindowId))]
|
||||
.flatMap { $0.app.pid == app.pid ? $0 : nil }
|
||||
}
|
||||
)
|
||||
// Why: cached snapshots only validate element identity for follow-up
|
||||
// actions; retaining MB-scale screenshot base64 in the long-lived agent grows memory.
|
||||
@@ -645,7 +656,8 @@ final class Provider {
|
||||
includeScreenshot: Bool,
|
||||
windowId: CGWindowID?,
|
||||
windowIndex: Int?,
|
||||
restoreWindow: Bool
|
||||
restoreWindow: Bool,
|
||||
previousSnapshot: (CGWindowID) -> Snapshot?
|
||||
) throws -> Snapshot {
|
||||
guard accessibilityTrustedSettled() else {
|
||||
// Why: agents retry failed observations. Only the explicit setup flow
|
||||
@@ -694,6 +706,7 @@ final class Provider {
|
||||
if renderer.appStoppedResponding {
|
||||
throw AccessibilityMessaging.appNotResponding(app.name)
|
||||
}
|
||||
let indexes = renderer.carryOverIndexes(from: previousSnapshot(capture.windowId))
|
||||
let screenshot = includeScreenshot ? capture.screenshotPayload() : nil
|
||||
let screenshotStatus: ScreenshotStatus = if screenshot != nil {
|
||||
.captured
|
||||
@@ -718,6 +731,8 @@ final class Provider {
|
||||
screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds),
|
||||
screenshotEngine: capture.image?.engine,
|
||||
elements: renderer.records,
|
||||
elementOrder: indexes.order,
|
||||
nextFreshElementIndex: indexes.nextFreshIndex,
|
||||
truncated: renderer.truncated,
|
||||
maxDepthReached: renderer.maxDepthReached,
|
||||
appWindowIds: windowCandidates.map(\.windowId).sorted()
|
||||
@@ -2118,6 +2133,28 @@ private final class TreeRenderer {
|
||||
}
|
||||
}
|
||||
|
||||
extension TreeRenderer {
|
||||
/// Swaps the walk-order indexes for ones carried over from the previous snapshot of this window.
|
||||
func carryOverIndexes(from previous: Snapshot?) -> (order: [Int], nextFreshIndex: Int) {
|
||||
let walked = records.values.sorted { $0.index < $1.index }
|
||||
let assignment = StableElementIndexes.assign(
|
||||
current: walked.map(\.element),
|
||||
previous: previous?.elementOrder.compactMap { index in
|
||||
previous?.elements[index].map { (key: $0.element, index: index) }
|
||||
} ?? [],
|
||||
previousNextFreshIndex: previous?.nextFreshElementIndex ?? 0
|
||||
)
|
||||
let renumbered = Dictionary(uniqueKeysWithValues: zip(walked.map(\.index), assignment.indexes))
|
||||
records = Dictionary(uniqueKeysWithValues: zip(walked, assignment.indexes).map { record, index in
|
||||
(index, ElementRecord(index: index, element: record.element, localFrame: record.localFrame, actions: record.actions, signature: record.signature))
|
||||
})
|
||||
lines = lines.map { StableElementIndexes.renumber(line: $0, indexes: renumbered) }
|
||||
focusedElementId = focusedElementId.flatMap { renumbered[$0] }
|
||||
focusedSummary = focusedSummary.map { StableElementIndexes.renumber(line: $0, indexes: renumbered) }
|
||||
return (assignment.indexes, assignment.nextFreshIndex)
|
||||
}
|
||||
}
|
||||
|
||||
private final class AXSnapshotReader {
|
||||
private enum CachedAttribute {
|
||||
case missing
|
||||
|
||||
@@ -220,7 +220,8 @@ public enum SnapshotRenderHeuristics {
|
||||
return "zoom the window"
|
||||
}
|
||||
let stripped = action.hasPrefix("AX") ? String(action.dropFirst(2)) : action
|
||||
return splitCamelCase(stripped.replacingOccurrences(of: "ByPage", with: "")).lowercased()
|
||||
// Why: custom actions such as toolbar items' carry multi-line names; one element is one line.
|
||||
return sanitize(splitCamelCase(stripped.replacingOccurrences(of: "ByPage", with: "")).lowercased())
|
||||
}
|
||||
|
||||
public static func sanitize(_ value: String) -> String {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import Foundation
|
||||
|
||||
/// Keeps an element's index across snapshots of the same window, so action results can send only
|
||||
/// the lines that changed.
|
||||
public enum StableElementIndexes {
|
||||
/// Only matching AX identities keep their indexes; all other elements receive fresh indexes.
|
||||
public static func assign<Key: Hashable>(
|
||||
current: [Key],
|
||||
previous: [(key: Key, index: Int)],
|
||||
previousNextFreshIndex: Int
|
||||
) -> (indexes: [Int], nextFreshIndex: Int) {
|
||||
let previousIndexByKey = Dictionary(previous.map { ($0.key, $0.index) }, uniquingKeysWith: { first, _ in first })
|
||||
var claimed = Set<Int>()
|
||||
var assigned = current.map { element -> Int? in
|
||||
guard let index = previousIndexByKey[element], claimed.insert(index).inserted else { return nil }
|
||||
return index
|
||||
}
|
||||
var nextFreshIndex = max(previousNextFreshIndex, (previous.map(\.index).max() ?? -1) + 1)
|
||||
for position in assigned.indices where assigned[position] == nil {
|
||||
assigned[position] = nextFreshIndex
|
||||
nextFreshIndex += 1
|
||||
}
|
||||
return (assigned.compactMap { $0 }, nextFreshIndex)
|
||||
}
|
||||
|
||||
/// Rewrites the leading `<tabs><index>` of a rendered element line; other lines pass through.
|
||||
public static func renumber(line: String, indexes: [Int: Int]) -> String {
|
||||
let tabs = line.prefix { $0 == "\t" }
|
||||
let rest = line.dropFirst(tabs.count)
|
||||
let digits = rest.prefix { $0.isASCII && $0.isNumber }
|
||||
guard !digits.isEmpty,
|
||||
rest.dropFirst(digits.count).first.map({ $0 == " " }) ?? true,
|
||||
let old = Int(digits),
|
||||
let new = indexes[old]
|
||||
else {
|
||||
return line
|
||||
}
|
||||
return "\(tabs)\(new)\(rest.dropFirst(digits.count))"
|
||||
}
|
||||
}
|
||||
+10
@@ -32,6 +32,16 @@ final class SnapshotRenderingTests: XCTestCase {
|
||||
XCTAssertFalse(SnapshotRenderHeuristics.shouldElide(node))
|
||||
}
|
||||
|
||||
func testKeepsMultiLineCustomActionNamesOnTheElementLine() {
|
||||
// Activity Monitor's toolbar buttons expose custom actions named like this.
|
||||
let node = SnapshotRenderNode(role: "AXButton", label: "Stop", rawActions: ["Name:move next\nTarget:0x0\nSelector:(null)"])
|
||||
|
||||
XCTAssertEqual(
|
||||
SnapshotRenderHeuristics.line(index: 4, node: node),
|
||||
"4 button Stop, Secondary Actions: name:move next target:0x0 selector:(null)"
|
||||
)
|
||||
}
|
||||
|
||||
func testRendersMarkdownLinksAndSuppressesTheirChildren() {
|
||||
let node = SnapshotRenderNode(role: "AXLink", linkText: "Skip [main]", url: "https://example.com/path")
|
||||
|
||||
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class StableElementIndexesTests: XCTestCase {
|
||||
private typealias Previous = (key: String, index: Int, signature: String)
|
||||
|
||||
private func assign(_ current: [(String, String)], previous: [Previous] = [], nextFresh: Int = 0) -> (indexes: [Int], nextFreshIndex: Int) {
|
||||
StableElementIndexes.assign(
|
||||
current: current.map { $0.0 },
|
||||
previous: previous.map { (key: $0.key, index: $0.index) },
|
||||
previousNextFreshIndex: nextFresh
|
||||
)
|
||||
}
|
||||
|
||||
func testFirstSnapshotNumbersElementsInWalkOrder() {
|
||||
XCTAssertEqual(assign([("window", "AXWindow"), ("search", "AXTextField"), ("row", "AXRow")]).indexes, [0, 1, 2])
|
||||
}
|
||||
|
||||
func testAnElementInsertedEarlyLeavesEveryOtherIndexAlone() {
|
||||
let previous: [Previous] = [("window", 0, "AXWindow"), ("search", 1, "AXTextField"), ("row", 2, "AXRow")]
|
||||
|
||||
let result = assign(
|
||||
[("window", "AXWindow"), ("clear", "AXButton"), ("search", "AXTextField"), ("row", "AXRow")],
|
||||
previous: previous,
|
||||
nextFresh: 3
|
||||
)
|
||||
|
||||
XCTAssertEqual(result.indexes, [0, 3, 1, 2])
|
||||
XCTAssertEqual(result.nextFreshIndex, 4)
|
||||
}
|
||||
|
||||
func testIdentityWinsOverSignatureForLookalikeSiblings() {
|
||||
// A new unnamed container above two existing ones must not take an existing container's index.
|
||||
let previous: [Previous] = [("window", 0, "AXWindow"), ("groupA", 1, "AXGroup"), ("groupB", 2, "AXGroup")]
|
||||
|
||||
let result = assign(
|
||||
[("window", "AXWindow"), ("groupNew", "AXGroup"), ("groupA", "AXGroup"), ("groupB", "AXGroup")],
|
||||
previous: previous,
|
||||
nextFresh: 3
|
||||
)
|
||||
|
||||
XCTAssertEqual(result.indexes, [0, 3, 1, 2])
|
||||
}
|
||||
|
||||
func testElementsRebuiltOnEveryReadReceiveFreshIndexes() {
|
||||
let previous: [Previous] = [("window", 0, "AXWindow"), ("cell-old-1", 1, "AXCell|Name"), ("cell-old-2", 2, "AXCell|Size")]
|
||||
|
||||
let result = assign(
|
||||
[("window", "AXWindow"), ("cell-new-1", "AXCell|Name"), ("cell-new-2", "AXCell|Size")],
|
||||
previous: previous,
|
||||
nextFresh: 3
|
||||
)
|
||||
|
||||
XCTAssertEqual(result.indexes, [0, 3, 4])
|
||||
XCTAssertEqual(result.nextFreshIndex, 5)
|
||||
}
|
||||
|
||||
func testALookalikeReplacementCannotInheritARemovedElementsIndex() {
|
||||
let previous: [Previous] = [("window", 0, "AXWindow"), ("buttonA", 1, "AXButton|Delete"), ("buttonB", 2, "AXButton|Keep")]
|
||||
|
||||
let result = assign(
|
||||
[("window", "AXWindow"), ("buttonC", "AXButton|Delete"), ("buttonB", "AXButton|Keep")],
|
||||
previous: previous,
|
||||
nextFresh: 3
|
||||
)
|
||||
|
||||
XCTAssertEqual(result.indexes, [0, 3, 2])
|
||||
}
|
||||
|
||||
func testARemovedIndexIsNeverHandedToADifferentNewElement() {
|
||||
let previous: [Previous] = [("window", 0, "AXWindow"), ("search", 1, "AXTextField")]
|
||||
|
||||
let result = assign([("window", "AXWindow"), ("new row", "AXRow")], previous: previous, nextFresh: 7)
|
||||
|
||||
XCTAssertEqual(result.indexes, [0, 7])
|
||||
}
|
||||
|
||||
func testRenumbersOnlyTheLeadingElementIndex() {
|
||||
let indexes = [3: 41, 12: 3]
|
||||
|
||||
XCTAssertEqual(StableElementIndexes.renumber(line: "\t\t3 button Upvote 12", indexes: indexes), "\t\t41 button Upvote 12")
|
||||
XCTAssertEqual(StableElementIndexes.renumber(line: "12", indexes: indexes), "3")
|
||||
XCTAssertEqual(StableElementIndexes.renumber(line: "\t... 12 inactive browser tabs omitted", indexes: indexes), "\t... 12 inactive browser tabs omitted")
|
||||
XCTAssertEqual(StableElementIndexes.renumber(line: "\t12px text", indexes: indexes), "\t12px text")
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,16 @@ Use the fresh state returned by each action for the next element index. Element
|
||||
|
||||
In `--json` output, read the accessibility tree and action indexes from `result.snapshot.treeText`; `elementCount` is only a count and must not be used to infer indexes.
|
||||
|
||||
## Reading State Cheaply
|
||||
|
||||
Action results omit unchanged elements when their locations stay unambiguous. `result.snapshot.treeDiff` marks a diff, and `treeText` then lists `~` changed element lines, `-` removed indexes as ranges, or "No change in the accessibility tree", followed by the focused element. Elements not listed keep the index and text from the last tree you read. Additions, moves and parent changes return the whole tree so each control's location stays clear. Without `treeDiff`, always read `treeText` as the whole tree; window changes, large diffs and older runtimes can also return full trees. Older runtimes may include `+` added lines in a diff. On macOS an element keeps its index across reads while it exists. Pass `--full-tree` to an action when you need the whole tree back.
|
||||
|
||||
In a large window, look for one control with `get-app-state --query <text>`: it prints only elements whose line contains the text (case-insensitive), with their ancestors. `--max-elements <n>` caps how many elements are printed. Both only narrow the output; indexes are the same as in the full tree.
|
||||
|
||||
```text
|
||||
ORCA computer get-app-state --app com.apple.finder --query Downloads --json
|
||||
```
|
||||
|
||||
## App Selectors
|
||||
|
||||
Prefer bundle IDs from `list-apps`; names are acceptable when unambiguous. Use `pid:<number>` only when bundle ID or name matching is ambiguous.
|
||||
|
||||
@@ -53,6 +53,16 @@ const COMPUTER_USE_MARKDOWN = [
|
||||
"",
|
||||
"In `--json` output, read the accessibility tree and action indexes from `result.snapshot.treeText`; `elementCount` is only a count and must not be used to infer indexes.",
|
||||
"",
|
||||
"## Reading State Cheaply",
|
||||
"",
|
||||
"Action results omit unchanged elements when their locations stay unambiguous. `result.snapshot.treeDiff` marks a diff, and `treeText` then lists `~` changed element lines, `-` removed indexes as ranges, or \"No change in the accessibility tree\", followed by the focused element. Elements not listed keep the index and text from the last tree you read. Additions, moves and parent changes return the whole tree so each control's location stays clear. Without `treeDiff`, always read `treeText` as the whole tree; window changes, large diffs and older runtimes can also return full trees. Older runtimes may include `+` added lines in a diff. On macOS an element keeps its index across reads while it exists. Pass `--full-tree` to an action when you need the whole tree back.",
|
||||
"",
|
||||
"In a large window, look for one control with `get-app-state --query <text>`: it prints only elements whose line contains the text (case-insensitive), with their ancestors. `--max-elements <n>` caps how many elements are printed. Both only narrow the output; indexes are the same as in the full tree.",
|
||||
"",
|
||||
"```text",
|
||||
"ORCA computer get-app-state --app com.apple.finder --query Downloads --json",
|
||||
"```",
|
||||
"",
|
||||
"## App Selectors",
|
||||
"",
|
||||
"Prefer bundle IDs from `list-apps`; names are acceptable when unambiguous. Use `pid:<number>` only when bundle ID or name matching is ambiguous.",
|
||||
|
||||
@@ -11,6 +11,10 @@ const COMMAND_SCOPED_FLAG_HELP: Record<string, Record<string, string>> = {
|
||||
url: '--url <url> Full review or issue URL; pass all URLs positionally or all as repeated --url',
|
||||
key: '--key <key> Opaque key from reference list; repeat for several'
|
||||
},
|
||||
'computer get-app-state': {
|
||||
query:
|
||||
'--query <text> Show only elements whose line contains this text, with their ancestors'
|
||||
},
|
||||
'reference find': {
|
||||
query: '--query <url-or-key> Full URL or native issue key such as STA-1234',
|
||||
repo: '--repo <selector> Filter by workspace repository membership',
|
||||
|
||||
@@ -54,14 +54,23 @@ describe('prepareComputerCliJsonResult', () => {
|
||||
})
|
||||
|
||||
describe('formatComputerAction', () => {
|
||||
it('does not treat legacy action results without metadata as completed', () => {
|
||||
it.each([
|
||||
{ kind: 'full tree', treeText: 'tree', treeDiff: undefined },
|
||||
{
|
||||
kind: 'diff',
|
||||
treeText: '~ 1 button Save',
|
||||
treeDiff: { baseSnapshotId: 'snap-before', changed: 1, added: 0, removed: 0 }
|
||||
},
|
||||
{ kind: 'empty tree', treeText: '', treeDiff: undefined }
|
||||
])('prints $kind without claiming unverified actions completed', ({ treeText, treeDiff }) => {
|
||||
const result: ComputerActionResult = {
|
||||
snapshot: {
|
||||
id: 'snap-1',
|
||||
app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
|
||||
window: { title: 'Finder', id: 42, width: 800, height: 600 },
|
||||
coordinateSpace: 'window',
|
||||
treeText: 'tree',
|
||||
treeText,
|
||||
treeDiff,
|
||||
elementCount: 5,
|
||||
focusedElementId: null
|
||||
},
|
||||
@@ -74,6 +83,11 @@ describe('formatComputerAction', () => {
|
||||
expect(output).toContain('Click attempted, unverified (verification metadata unavailable)')
|
||||
expect(output).toContain('Inspect with the command above')
|
||||
expect(output).not.toContain('Click completed')
|
||||
if (treeText) {
|
||||
expect(output).toContain(`\n\n${treeText}`)
|
||||
} else {
|
||||
expect(output).not.toContain('\n\n')
|
||||
}
|
||||
})
|
||||
|
||||
it('does not treat accessibility actions without verification as completed', () => {
|
||||
|
||||
@@ -215,7 +215,7 @@ export function formatComputerAction(
|
||||
const inspectTail = unverified
|
||||
? 'Inspect with the command above or use the --json result before assuming it worked.'
|
||||
: 'Use the --json result or rerun state before choosing the next element index.'
|
||||
return `${formatActionVerb(verb)} ${outcome}${path}${verification}; ${result.snapshot.elementCount} visible elements in current window.${screenshotFailure} Use \`${followUpCommand}\` to inspect. ${inspectTail}`
|
||||
return `${formatActionVerb(verb)} ${outcome}${path}${verification}; ${result.snapshot.elementCount} visible elements in current window.${screenshotFailure} Use \`${followUpCommand}\` to inspect. ${inspectTail}${result.snapshot.treeText ? `\n\n${result.snapshot.treeText}` : ''}`
|
||||
}
|
||||
|
||||
function formatComputerFollowUpCommand(
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { queryComputerTreeText } from './computer-tree-query'
|
||||
|
||||
const TREE = [
|
||||
'App=com.apple.TextEdit (pid 96320)',
|
||||
'Window: "Untitled", App: TextEdit.',
|
||||
'',
|
||||
'0 standard window Untitled',
|
||||
'\t1 scroll area, Secondary Actions: scroll up, scroll down',
|
||||
'\t\t2 text entry area (settable) base zed',
|
||||
'\t21 container Formatting',
|
||||
'\t\t22 checkbox bold, Value: 0',
|
||||
'\t\t23 checkbox italic, Value: 0',
|
||||
'',
|
||||
'The focused UI element is 2 text entry area (settable) base zed.'
|
||||
].join('\n')
|
||||
|
||||
describe('queryComputerTreeText', () => {
|
||||
it('caps the printed elements with maxElements and says how many were left out', () => {
|
||||
expect(queryComputerTreeText(TREE, { maxElements: 2 })).toBe(
|
||||
[
|
||||
'App=com.apple.TextEdit (pid 96320)',
|
||||
'Window: "Untitled", App: TextEdit.',
|
||||
'',
|
||||
'Showing the first 2 of 6 elements.',
|
||||
'0 standard window Untitled',
|
||||
'\t1 scroll area, Secondary Actions: scroll up, scroll down',
|
||||
'',
|
||||
'The focused UI element is 2 text entry area (settable) base zed.'
|
||||
].join('\n')
|
||||
)
|
||||
})
|
||||
|
||||
it('says when nothing matches instead of printing an empty tree', () => {
|
||||
expect(queryComputerTreeText(TREE, { query: 'underline' })).toContain(
|
||||
'\n\nNo elements match "underline".\n\nThe focused UI element is'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
joinComputerTreeText,
|
||||
parseComputerTreeText,
|
||||
type ComputerTreeElement
|
||||
} from '../shared/computer-use-tree-text'
|
||||
|
||||
export type ComputerTreeQuery = {
|
||||
query?: string
|
||||
maxElements?: number
|
||||
}
|
||||
|
||||
// Runs in the CLI over the rendered tree, so it works against every runtime version and provider.
|
||||
export function queryComputerTreeText(treeText: string, options: ComputerTreeQuery): string {
|
||||
const { header, elements, footer } = parseComputerTreeText(treeText)
|
||||
const needle = options.query?.toLowerCase()
|
||||
const candidates = needle
|
||||
? elements.filter((element) => elementText(element).toLowerCase().includes(needle))
|
||||
: elements
|
||||
const shown =
|
||||
options.maxElements === undefined ? candidates : candidates.slice(0, options.maxElements)
|
||||
const shownSet = new Set(shown)
|
||||
const keep = withAncestors(elements, shownSet)
|
||||
const body = elements.flatMap((element) =>
|
||||
!keep.has(element) ? [] : shownSet.has(element) ? element.lines : element.lines.slice(0, 1)
|
||||
)
|
||||
const note = needle
|
||||
? candidates.length === 0
|
||||
? `No elements match "${options.query}".`
|
||||
: `${candidates.length} of ${elements.length} elements match "${options.query}"; showing ${shown.length === candidates.length ? 'them' : `the first ${shown.length}`} with their ancestors.`
|
||||
: shown.length < elements.length
|
||||
? `Showing the first ${shown.length} of ${elements.length} elements.`
|
||||
: null
|
||||
return joinComputerTreeText(header, note ? [note, ...body] : body, footer)
|
||||
}
|
||||
|
||||
function elementText(element: ComputerTreeElement): string {
|
||||
return element.lines[0].trimStart().replace(/^\d+ ?/u, '')
|
||||
}
|
||||
|
||||
function withAncestors(
|
||||
elements: ComputerTreeElement[],
|
||||
shown: Set<ComputerTreeElement>
|
||||
): Set<ComputerTreeElement> {
|
||||
const keep = new Set<ComputerTreeElement>()
|
||||
const path: (ComputerTreeElement | undefined)[] = []
|
||||
for (const element of elements) {
|
||||
path.length = Math.min(path.length, element.depth)
|
||||
if (shown.has(element)) {
|
||||
for (const ancestor of path) {
|
||||
if (ancestor) {
|
||||
keep.add(ancestor)
|
||||
}
|
||||
}
|
||||
keep.add(element)
|
||||
}
|
||||
path[element.depth] = element
|
||||
}
|
||||
return keep
|
||||
}
|
||||
@@ -43,6 +43,7 @@ export const FLAG_HELP_TEXT: Record<string, string> = {
|
||||
limit: '--limit <n> Maximum number of rows to return',
|
||||
local: '--local Target the current project instead of the global install',
|
||||
skill: '--skill <name> Bundled skill to act on; repeat for several',
|
||||
'max-elements': '--max-elements <n> Show at most n elements of the accessibility tree',
|
||||
mode: '--mode <mode> Mode such as edit, diff, or both',
|
||||
model: '--model <id> Provider model id for a new agent launch',
|
||||
effort: '--effort <level> Reasoning effort for the selected model',
|
||||
@@ -103,6 +104,8 @@ export const FLAG_HELP_TEXT: Record<string, string> = {
|
||||
relations: '--relations Include blocking, related, and duplicate links',
|
||||
activity: '--activity Include issue field-change history',
|
||||
full: '--full Include all supported V1 issue context within caps',
|
||||
'full-tree':
|
||||
'--full-tree Return the whole accessibility tree instead of changes since the last snapshot',
|
||||
'reuse-session':
|
||||
'--reuse-session Reuse the previous live session for existing-workspace runs',
|
||||
'fresh-session': '--fresh-session Disable session reuse for future runs',
|
||||
|
||||
@@ -49,10 +49,14 @@ export function getComputerObserveFlags(flags: Map<string, string | boolean>): {
|
||||
export function getComputerActionObserveFlags(flags: Map<string, string | boolean>): {
|
||||
noScreenshot?: boolean
|
||||
restoreWindow?: boolean
|
||||
treeDiff?: boolean
|
||||
windowId?: number
|
||||
windowIndex?: number
|
||||
} {
|
||||
return getComputerObserveFlags(flags)
|
||||
return {
|
||||
...getComputerObserveFlags(flags),
|
||||
...(flags.has('full-tree') ? {} : { treeDiff: true })
|
||||
}
|
||||
}
|
||||
|
||||
export function getComputerClickActionFlags(flags: Map<string, string | boolean>): {
|
||||
|
||||
@@ -76,7 +76,8 @@ describe('orca computer action CLI routing', () => {
|
||||
clickCount: undefined,
|
||||
mouseButton: undefined,
|
||||
modifiers: 'CmdOrCtrl+Shift',
|
||||
noScreenshot: undefined
|
||||
noScreenshot: undefined,
|
||||
treeDiff: true
|
||||
})
|
||||
})
|
||||
|
||||
@@ -132,6 +133,68 @@ describe('orca computer action CLI routing', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('asks for a tree diff unless --full-tree is passed', async () => {
|
||||
queueFixtures(callMock, okFixture('req_click', sampleSnapshot()))
|
||||
await main(
|
||||
['computer', 'click', '--session', 'manual', '--app', 'Finder', '--element-index', '3'],
|
||||
'/tmp/repo/src'
|
||||
)
|
||||
expect(callMock).toHaveBeenLastCalledWith(
|
||||
'computer.click',
|
||||
expect.objectContaining({ treeDiff: true })
|
||||
)
|
||||
|
||||
queueFixtures(callMock, okFixture('req_click', sampleSnapshot()))
|
||||
await main(
|
||||
[
|
||||
'computer',
|
||||
'click',
|
||||
'--session',
|
||||
'manual',
|
||||
'--app',
|
||||
'Finder',
|
||||
'--element-index',
|
||||
'3',
|
||||
'--full-tree'
|
||||
],
|
||||
'/tmp/repo/src'
|
||||
)
|
||||
expect(callMock.mock.lastCall?.[1]).not.toHaveProperty('treeDiff')
|
||||
})
|
||||
|
||||
it('prints a returned tree diff after the action summary', async () => {
|
||||
const diffText = [
|
||||
'App=com.apple.finder (pid 100)',
|
||||
'Window: "Finder", App: Finder.',
|
||||
'',
|
||||
'Changes since the previous snapshot (~ changed, + added, - removed). Unchanged elements are omitted and keep their indexes.',
|
||||
'~ \t\t3 text field (settable) Search, Value: notes',
|
||||
'',
|
||||
'The focused UI element is 3 text field (settable) Search, Value: notes.'
|
||||
].join('\n')
|
||||
const result = sampleSnapshot()
|
||||
queueFixtures(
|
||||
callMock,
|
||||
okFixture('req_click', {
|
||||
...result,
|
||||
snapshot: {
|
||||
...result.snapshot,
|
||||
treeText: diffText,
|
||||
treeDiff: { baseSnapshotId: 'snap-before', changed: 1, added: 0, removed: 0 }
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
await main(
|
||||
['computer', 'click', '--session', 'manual', '--app', 'Finder', '--element-index', '3'],
|
||||
'/tmp/repo/src'
|
||||
)
|
||||
|
||||
const output = vi.mocked(console.log).mock.calls[0][0]
|
||||
expect(output).toMatch(/^Click attempted, unverified/u)
|
||||
expect(output).toContain(`\n\n${diffText}`)
|
||||
})
|
||||
|
||||
it('maps action command flags to RPC payloads', async () => {
|
||||
queueFixtures(
|
||||
callMock,
|
||||
@@ -167,7 +230,8 @@ describe('orca computer action CLI routing', () => {
|
||||
fromY: 2,
|
||||
toX: 3,
|
||||
toY: 4,
|
||||
noScreenshot: undefined
|
||||
noScreenshot: undefined,
|
||||
treeDiff: true
|
||||
})
|
||||
})
|
||||
|
||||
@@ -203,7 +267,8 @@ describe('orca computer action CLI routing', () => {
|
||||
y: 20,
|
||||
direction: 'down',
|
||||
pages: 0.5,
|
||||
noScreenshot: undefined
|
||||
noScreenshot: undefined,
|
||||
treeDiff: true
|
||||
})
|
||||
})
|
||||
|
||||
@@ -229,7 +294,8 @@ describe('orca computer action CLI routing', () => {
|
||||
session: 'manual',
|
||||
app: 'Finder',
|
||||
key: 'CmdOrCtrl+L',
|
||||
noScreenshot: true
|
||||
noScreenshot: true,
|
||||
treeDiff: true
|
||||
})
|
||||
|
||||
callMock.mockReset()
|
||||
@@ -254,7 +320,8 @@ describe('orca computer action CLI routing', () => {
|
||||
session: 'manual',
|
||||
app: 'Finder',
|
||||
text: 'hello world',
|
||||
noScreenshot: undefined
|
||||
noScreenshot: undefined,
|
||||
treeDiff: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -136,7 +136,8 @@ describe('orca computer action CLI validation', () => {
|
||||
session: 'manual',
|
||||
app: 'Finder',
|
||||
key: '+',
|
||||
noScreenshot: true
|
||||
noScreenshot: true,
|
||||
treeDiff: true
|
||||
})
|
||||
expect(vi.mocked(console.error)).not.toHaveBeenCalled()
|
||||
expect(process.exitCode).toBeUndefined()
|
||||
@@ -415,7 +416,8 @@ describe('orca computer action CLI validation', () => {
|
||||
app: 'Finder',
|
||||
elementIndex: 1,
|
||||
value: '',
|
||||
noScreenshot: true
|
||||
noScreenshot: true,
|
||||
treeDiff: true
|
||||
})
|
||||
expect(vi.mocked(console.error)).not.toHaveBeenCalled()
|
||||
expect(process.exitCode).toBeUndefined()
|
||||
|
||||
@@ -92,6 +92,66 @@ describe('orca computer get-app-state formatting', () => {
|
||||
)
|
||||
expect(readFileSync(parsed.result.screenshot.path)).toEqual(Buffer.from('png-bytes'))
|
||||
})
|
||||
|
||||
it('filters get-app-state to matching elements and their ancestors with --query', async () => {
|
||||
const result = sampleSnapshot()
|
||||
queueFixtures(
|
||||
callMock,
|
||||
okFixture('req_state', {
|
||||
...result,
|
||||
snapshot: {
|
||||
...result.snapshot,
|
||||
treeText: [
|
||||
'App=com.apple.finder (pid 100)',
|
||||
'Window: "Finder", App: Finder.',
|
||||
'',
|
||||
'0 standard window Finder',
|
||||
'\t1 toolbar',
|
||||
'\t\t2 button Back',
|
||||
'\t\t3 text field (settable) Search',
|
||||
'\t4 split group',
|
||||
'\t\t5 outline Sidebar',
|
||||
'\t\t\t6 row, Text: Downloads',
|
||||
'\t\t\t7 row, Text: Documents',
|
||||
'',
|
||||
'No UI element is currently focused.'
|
||||
].join('\n')
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
await main(
|
||||
[
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'--session',
|
||||
'manual',
|
||||
'--app',
|
||||
'Finder',
|
||||
'--query',
|
||||
'DOWNLOADS',
|
||||
'--json'
|
||||
],
|
||||
'/tmp/repo'
|
||||
)
|
||||
|
||||
const parsed = JSON.parse(vi.mocked(console.log).mock.calls[0][0])
|
||||
expect(parsed.result.snapshot.treeText).toBe(
|
||||
[
|
||||
'App=com.apple.finder (pid 100)',
|
||||
'Window: "Finder", App: Finder.',
|
||||
'',
|
||||
'1 of 8 elements match "DOWNLOADS"; showing them with their ancestors.',
|
||||
'0 standard window Finder',
|
||||
'\t4 split group',
|
||||
'\t\t5 outline Sidebar',
|
||||
'\t\t\t6 row, Text: Downloads',
|
||||
'',
|
||||
'No UI element is currently focused.'
|
||||
].join('\n')
|
||||
)
|
||||
expect(callMock.mock.lastCall?.[1]).not.toHaveProperty('query')
|
||||
})
|
||||
})
|
||||
|
||||
function sampleSnapshot() {
|
||||
|
||||
@@ -6,7 +6,12 @@ import type {
|
||||
ComputerSnapshotResult
|
||||
} from '../../shared/runtime-types'
|
||||
import type { CommandHandler } from '../dispatch'
|
||||
import { getOptionalStringFlag, getRequiredStringFlag } from '../flags'
|
||||
import {
|
||||
getOptionalPositiveIntegerFlag,
|
||||
getOptionalStringFlag,
|
||||
getRequiredStringFlag
|
||||
} from '../flags'
|
||||
import { queryComputerTreeText } from '../computer-tree-query'
|
||||
import { RuntimeClientError } from '../runtime-client'
|
||||
import {
|
||||
formatComputerAction,
|
||||
@@ -78,11 +83,17 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
|
||||
},
|
||||
'computer get-app-state': async ({ flags, client, cwd, json }) => {
|
||||
const observeFlags = getComputerObserveFlags(flags)
|
||||
const query = getOptionalStringFlag(flags, 'query')
|
||||
const maxElements = getOptionalPositiveIntegerFlag(flags, 'max-elements')
|
||||
const target = await getComputerCommandTarget(flags, cwd, client)
|
||||
const result = await client.call<ComputerSnapshotResult>('computer.getAppState', {
|
||||
...target,
|
||||
...observeFlags
|
||||
})
|
||||
if (query !== undefined || maxElements !== undefined) {
|
||||
const { snapshot } = result.result
|
||||
snapshot.treeText = queryComputerTreeText(snapshot.treeText, { query, maxElements })
|
||||
}
|
||||
printResult(result, json, formatGetAppState)
|
||||
},
|
||||
'computer click': async ({ flags, client, cwd, json }) => {
|
||||
|
||||
+13
-12
@@ -3,12 +3,13 @@ import { GLOBAL_FLAGS } from '../args'
|
||||
|
||||
const COMPUTER_FLAGS = [...GLOBAL_FLAGS, 'worktree', 'session', 'app']
|
||||
const COMPUTER_WINDOW_TARGET_FLAGS = ['window-id', 'window-index']
|
||||
const COMPUTER_ACTION_FLAGS = [
|
||||
const COMPUTER_OBSERVE_FLAGS = [
|
||||
...COMPUTER_FLAGS,
|
||||
...COMPUTER_WINDOW_TARGET_FLAGS,
|
||||
'restore-window',
|
||||
'no-screenshot'
|
||||
]
|
||||
const COMPUTER_ACTION_FLAGS = [...COMPUTER_OBSERVE_FLAGS, 'full-tree']
|
||||
|
||||
export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
|
||||
{
|
||||
@@ -39,14 +40,14 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['computer', 'get-app-state'],
|
||||
summary: 'Capture a compact accessibility snapshot of an app',
|
||||
usage:
|
||||
'orca computer get-app-state --app <name|bundle|pid:N> [--window-id <id> | --window-index <n>] [--worktree <selector> | --session <id>] [--restore-window] [--no-screenshot] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, ...COMPUTER_WINDOW_TARGET_FLAGS, 'restore-window']
|
||||
'orca computer get-app-state --app <name|bundle|pid:N> [--window-id <id> | --window-index <n>] [--worktree <selector> | --session <id>] [--query <text>] [--max-elements <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'query', 'max-elements']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'click'],
|
||||
summary: 'Click an app element or window coordinate, optionally with modifiers',
|
||||
usage:
|
||||
'orca computer click --app <app> (--element-index <n> | --x <x> --y <y>) [--window-id <id> | --window-index <n>] [--click-count <n>] [--mouse-button <left|right|middle>] [--modifiers <modifier-chord>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer click --app <app> (--element-index <n> | --x <x> --y <y>) [--window-id <id> | --window-index <n>] [--click-count <n>] [--mouse-button <left|right|middle>] [--modifiers <modifier-chord>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [
|
||||
...COMPUTER_ACTION_FLAGS,
|
||||
'element-index',
|
||||
@@ -61,21 +62,21 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['computer', 'perform-secondary-action'],
|
||||
summary: 'Perform an advertised secondary accessibility action',
|
||||
usage:
|
||||
'orca computer perform-secondary-action --app <app> --element-index <n> --action <name> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer perform-secondary-action --app <app> --element-index <n> --action <name> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'action']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'scroll'],
|
||||
summary: 'Scroll an app element or window coordinate',
|
||||
usage:
|
||||
'orca computer scroll --app <app> (--element-index <n> | --x <x> --y <y>) --direction <up|down|left|right> [--window-id <id> | --window-index <n>] [--pages <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer scroll --app <app> (--element-index <n> | --x <x> --y <y>) --direction <up|down|left|right> [--window-id <id> | --window-index <n>] [--pages <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'x', 'y', 'direction', 'pages']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'drag'],
|
||||
summary: 'Drag between app elements or window coordinates',
|
||||
usage:
|
||||
'orca computer drag --app <app> (--from-element-index <n> --to-element-index <n> | --from-x <x> --from-y <y> --to-x <x> --to-y <y>) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer drag --app <app> (--from-element-index <n> --to-element-index <n> | --from-x <x> --from-y <y> --to-x <x> --to-y <y>) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [
|
||||
...COMPUTER_ACTION_FLAGS,
|
||||
'from-element-index',
|
||||
@@ -90,35 +91,35 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['computer', 'type-text'],
|
||||
summary: 'Type literal text at the current app focus',
|
||||
usage:
|
||||
'orca computer type-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer type-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'press-key'],
|
||||
summary: 'Press a single key such as Return or Escape',
|
||||
usage:
|
||||
'orca computer press-key --app <app> --key <key> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer press-key --app <app> --key <key> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'hotkey'],
|
||||
summary: 'Press a platform-aware key combination',
|
||||
usage:
|
||||
'orca computer hotkey --app <app> --key <key-combo> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer hotkey --app <app> --key <key-combo> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'paste-text'],
|
||||
summary: 'Paste exact text at the current app focus',
|
||||
usage:
|
||||
'orca computer paste-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer paste-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'set-value'],
|
||||
summary: 'Set the value of a settable app element',
|
||||
usage:
|
||||
'orca computer set-value --app <app> --element-index <n> (--value <text> | --value-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
|
||||
'orca computer set-value --app <app> --element-index <n> (--value <text> | --value-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'value', 'value-stdin']
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { computerProviderUnavailableMessage } from './computer-provider-unavailable-message'
|
||||
import { currentComputerProvider, shutdownComputerProviders } from './computer-provider-lifecycle'
|
||||
import { RuntimeClientError } from './runtime-client-error'
|
||||
import { ComputerTreeBaselines } from './computer-snapshot-tree-diff'
|
||||
import type { ComputerProviderCapabilities } from '../../shared/runtime-types'
|
||||
|
||||
type SidecarRequest = {
|
||||
id: number
|
||||
@@ -20,6 +22,8 @@ process.once('SIGINT', () => {
|
||||
})
|
||||
process.once('beforeExit', shutdownProviders)
|
||||
|
||||
const treeBaselines = new ComputerTreeBaselines()
|
||||
|
||||
process.on('message', (message: unknown) => {
|
||||
void handleMessage(message)
|
||||
})
|
||||
@@ -49,7 +53,7 @@ async function dispatch(method: string, params: Record<string, unknown>): Promis
|
||||
|
||||
switch (method) {
|
||||
case 'capabilities': {
|
||||
return await provider.capabilities()
|
||||
return withTreeDiffCapability(await provider.capabilities())
|
||||
}
|
||||
case 'listApps': {
|
||||
return await provider.listApps()
|
||||
@@ -58,34 +62,34 @@ async function dispatch(method: string, params: Record<string, unknown>): Promis
|
||||
return await provider.listWindows(params)
|
||||
}
|
||||
case 'getAppState': {
|
||||
return await provider.snapshot(params)
|
||||
return treeBaselines.apply(params, await provider.snapshot(params))
|
||||
}
|
||||
case 'click': {
|
||||
return await provider.action('click', params)
|
||||
return treeBaselines.apply(params, await provider.action('click', params))
|
||||
}
|
||||
case 'performSecondaryAction': {
|
||||
return await provider.action('performSecondaryAction', params)
|
||||
return treeBaselines.apply(params, await provider.action('performSecondaryAction', params))
|
||||
}
|
||||
case 'scroll': {
|
||||
return await provider.action('scroll', params)
|
||||
return treeBaselines.apply(params, await provider.action('scroll', params))
|
||||
}
|
||||
case 'drag': {
|
||||
return await provider.action('drag', params)
|
||||
return treeBaselines.apply(params, await provider.action('drag', params))
|
||||
}
|
||||
case 'typeText': {
|
||||
return await provider.action('typeText', params)
|
||||
return treeBaselines.apply(params, await provider.action('typeText', params))
|
||||
}
|
||||
case 'pressKey': {
|
||||
return await provider.action('pressKey', params)
|
||||
return treeBaselines.apply(params, await provider.action('pressKey', params))
|
||||
}
|
||||
case 'hotkey': {
|
||||
return await provider.action('hotkey', params)
|
||||
return treeBaselines.apply(params, await provider.action('hotkey', params))
|
||||
}
|
||||
case 'pasteText': {
|
||||
return await provider.action('pasteText', params)
|
||||
return treeBaselines.apply(params, await provider.action('pasteText', params))
|
||||
}
|
||||
case 'setValue': {
|
||||
return await provider.action('setValue', params)
|
||||
return treeBaselines.apply(params, await provider.action('setValue', params))
|
||||
}
|
||||
default:
|
||||
throw new RuntimeClientError(
|
||||
@@ -95,6 +99,16 @@ async function dispatch(method: string, params: Record<string, unknown>): Promis
|
||||
}
|
||||
}
|
||||
|
||||
function withTreeDiffCapability(
|
||||
capabilities: ComputerProviderCapabilities
|
||||
): ComputerProviderCapabilities {
|
||||
const { supports } = capabilities
|
||||
return {
|
||||
...capabilities,
|
||||
supports: { ...supports, observation: { ...supports.observation, treeDiff: true } }
|
||||
}
|
||||
}
|
||||
|
||||
function isRequest(message: unknown): message is SidecarRequest {
|
||||
if (!message || typeof message !== 'object') {
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { ComputerSnapshotResult } from '../../shared/runtime-types'
|
||||
import { ComputerTreeBaselines, diffComputerTreeText } from './computer-snapshot-tree-diff'
|
||||
|
||||
// Trimmed from a macOS helper walk of a WKWebView page.
|
||||
const HEADER = ['App=webfixture (pid 5310)', 'Window: "Fixture News", App: webfixture.', '']
|
||||
const FOOTER = ['', 'The focused UI element is 16 text field (settable) Title, Value: draft.']
|
||||
const BEFORE = [
|
||||
'0 standard window Fixture News',
|
||||
'\t1 scroll area',
|
||||
'\t\t2 HTML content Fixture News',
|
||||
'\t\t\t3 container',
|
||||
'\t\t\t\t4 link Home',
|
||||
'\t\t\t\t6 link New',
|
||||
'\t\t\t\t8 link Ask',
|
||||
'\t\t\t\t10 text field (settable) Placeholder: Search stories',
|
||||
'\t\t\t\t11 button Search',
|
||||
'\t\t\t12 heading Top stories',
|
||||
'\t\t\t13 container',
|
||||
'\t\t\t\t14 container',
|
||||
'\t\t\t\t\t15 text Title',
|
||||
'\t\t\t\t\t16 text field (settable) Title, Value: draft',
|
||||
'\t\t\t\t17 container',
|
||||
'\t\t\t\t\t18 text URL',
|
||||
'\t\t\t\t\t19 text field (settable) URL',
|
||||
'\t\t\t\t20 pop up button, Value: News',
|
||||
'\t\t\t\t21 button Submit',
|
||||
'\t\t\t22 content list',
|
||||
'\t\t\t\t23 container, Value: 0 points',
|
||||
'\t\t\t\t\t25 link Story number 0 about computer use',
|
||||
'\t\t\t\t\t26 button Upvote 0',
|
||||
'\t\t\t\t27 container, Value: 7 points',
|
||||
'\t\t\t\t\t29 link Story number 1 about computer use',
|
||||
'\t\t\t\t\t30 button Upvote 1',
|
||||
'\t985 text Fixture News'
|
||||
]
|
||||
|
||||
function tree(lines: string[], footer = FOOTER): string {
|
||||
return [...HEADER, ...lines, ...footer].join('\n')
|
||||
}
|
||||
|
||||
function replaceLine(lines: string[], from: string, to: string[]): string[] {
|
||||
const at = lines.indexOf(from)
|
||||
return [...lines.slice(0, at), ...to, ...lines.slice(at + 1)]
|
||||
}
|
||||
|
||||
describe('diffComputerTreeText', () => {
|
||||
it('sends only changed lines when elements keep their locations', () => {
|
||||
const after = replaceLine(BEFORE, '\t\t\t\t\t16 text field (settable) Title, Value: draft', [
|
||||
'\t\t\t\t\t16 text field (settable) Title, Value: Computer use diffs'
|
||||
])
|
||||
|
||||
const diff = diffComputerTreeText(tree(BEFORE), tree(after))
|
||||
|
||||
expect(diff).toMatchObject({ changed: 1, added: 0, removed: 0 })
|
||||
expect(diff?.text).toBe(
|
||||
[
|
||||
'App=webfixture (pid 5310)',
|
||||
'Window: "Fixture News", App: webfixture.',
|
||||
'',
|
||||
'Changes since the previous snapshot (~ changed, - removed). Unchanged elements are omitted and keep their indexes.',
|
||||
'~ \t\t\t\t\t16 text field (settable) Title, Value: Computer use diffs',
|
||||
...FOOTER
|
||||
].join('\n')
|
||||
)
|
||||
})
|
||||
|
||||
it('summarizes removed elements as index ranges', () => {
|
||||
const after = BEFORE.filter(
|
||||
(line) => !/^\t+(23|25|26|27|29) /u.test(line) && !line.includes('Upvote 1')
|
||||
)
|
||||
|
||||
const diff = diffComputerTreeText(tree(BEFORE), tree(after))
|
||||
|
||||
expect(diff).toMatchObject({ changed: 0, added: 0, removed: 6 })
|
||||
expect(diff?.text).toContain('\n- 23, 25-27, 29-30\n')
|
||||
})
|
||||
|
||||
it('says nothing changed and still reports the focused element', () => {
|
||||
const diff = diffComputerTreeText(tree(BEFORE), tree(BEFORE))
|
||||
|
||||
expect(diff?.text).toBe(
|
||||
[
|
||||
...HEADER,
|
||||
'No change in the accessibility tree since the previous snapshot.',
|
||||
...FOOTER
|
||||
].join('\n')
|
||||
)
|
||||
})
|
||||
|
||||
it.each(['\t\t\t\t14 container', '\t\t\t\t17 container'])(
|
||||
'returns the full tree for a checkbox inserted under %s',
|
||||
(parent) => {
|
||||
const after = replaceLine(BEFORE, parent, [parent, '\t\t\t\t\t986 checkbox Enabled'])
|
||||
|
||||
expect(diffComputerTreeText(tree(BEFORE), tree(after))).toBeNull()
|
||||
}
|
||||
)
|
||||
|
||||
it('returns the full tree when siblings change order', () => {
|
||||
const story0 = BEFORE.slice(20, 23)
|
||||
const story1 = BEFORE.slice(23, 26)
|
||||
const after = [...BEFORE.slice(0, 20), ...story1, ...story0, ...BEFORE.slice(26)]
|
||||
|
||||
const diff = diffComputerTreeText(tree(BEFORE), tree(after))
|
||||
|
||||
expect(diff).toBeNull()
|
||||
})
|
||||
|
||||
const titleField = '\t\t\t\t\t16 text field (settable) Title, Value: draft'
|
||||
const reparented = replaceLine(BEFORE, '\t\t\t\t\t15 text Title', [
|
||||
'\t\t\t\t986 container New fields',
|
||||
'\t\t\t\t\t15 text Title'
|
||||
])
|
||||
|
||||
it.each([
|
||||
{ change: 'inserted parent', before: BEFORE, after: reparented },
|
||||
{ change: 'removed parent', before: reparented, after: BEFORE },
|
||||
{
|
||||
change: 'existing parent',
|
||||
before: BEFORE,
|
||||
after: replaceLine(replaceLine(BEFORE, titleField, []), '\t\t\t\t17 container', [
|
||||
'\t\t\t\t17 container',
|
||||
titleField
|
||||
])
|
||||
}
|
||||
])('returns the full tree after reparenting through $change', ({ before, after }) => {
|
||||
expect(diffComputerTreeText(tree(before), tree(after))).toBeNull()
|
||||
})
|
||||
|
||||
it('falls back to the full tree when an insert renumbered everything after it', () => {
|
||||
const renumbered = BEFORE.map((line, position) =>
|
||||
position < 9
|
||||
? line
|
||||
: line.replace(/^(\t*)(\d+)/u, (_, tabs, index) => `${tabs}${Number(index) + 1}`)
|
||||
)
|
||||
|
||||
expect(diffComputerTreeText(tree(BEFORE), tree(renumbered))).toBeNull()
|
||||
})
|
||||
|
||||
it('keeps names that carried raw newlines on their own element', () => {
|
||||
// Older helpers printed multi-line custom action names verbatim.
|
||||
const before = [
|
||||
...BEFORE.slice(0, -1),
|
||||
'\t990 toolbar',
|
||||
'\t\t991 button (disabled) Stop, Secondary Actions: name: move next',
|
||||
' target:0x0',
|
||||
' selector:(null)',
|
||||
'\t\t992 button Inspector',
|
||||
...BEFORE.slice(-1)
|
||||
]
|
||||
const after = replaceLine(before, '\t\t992 button Inspector', [
|
||||
'\t\t992 button (disabled) Inspector'
|
||||
])
|
||||
|
||||
const diff = diffComputerTreeText(tree(before), tree(after))
|
||||
|
||||
expect(diff?.text).toContain('~ \t\t992 button (disabled) Inspector')
|
||||
expect(diff?.text).not.toContain('target:0x0')
|
||||
})
|
||||
})
|
||||
|
||||
describe('ComputerTreeBaselines', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
function result(treeText: string, windowId = 42, pid = 5310): ComputerSnapshotResult {
|
||||
return {
|
||||
snapshot: {
|
||||
id: `snap-${treeText.length}-${windowId}`,
|
||||
app: { name: 'webfixture', bundleId: null, pid },
|
||||
window: { id: windowId, title: 'Fixture News', width: 1200, height: 900 },
|
||||
coordinateSpace: 'window',
|
||||
treeText,
|
||||
elementCount: 27,
|
||||
focusedElementId: 16
|
||||
},
|
||||
screenshot: null,
|
||||
screenshotStatus: { state: 'skipped', reason: 'no_screenshot_flag' }
|
||||
}
|
||||
}
|
||||
|
||||
const edited = tree(
|
||||
replaceLine(BEFORE, '\t\t\t\t\t16 text field (settable) Title, Value: draft', [
|
||||
'\t\t\t\t\t16 text field (settable) Title, Value: hello'
|
||||
])
|
||||
)
|
||||
|
||||
it('diffs an action result against the tree this caller saw last for the window', () => {
|
||||
const baselines = new ComputerTreeBaselines()
|
||||
const first = baselines.apply({ app: 'webfixture', session: 'a' }, result(tree(BEFORE)))
|
||||
|
||||
const second = baselines.apply(
|
||||
{ app: 'webfixture', session: 'a', treeDiff: true },
|
||||
result(edited)
|
||||
)
|
||||
|
||||
expect(second.snapshot.treeDiff).toEqual({
|
||||
baseSnapshotId: first.snapshot.id,
|
||||
changed: 1,
|
||||
added: 0,
|
||||
removed: 0
|
||||
})
|
||||
expect(second.snapshot.treeText).toContain(
|
||||
'~ \t\t\t\t\t16 text field (settable) Title, Value: hello'
|
||||
)
|
||||
expect(second.snapshot.elementCount).toBe(27)
|
||||
})
|
||||
|
||||
it('returns the full tree to callers that did not ask for a diff', () => {
|
||||
const baselines = new ComputerTreeBaselines()
|
||||
baselines.apply({ app: 'webfixture' }, result(tree(BEFORE)))
|
||||
|
||||
const second = baselines.apply({ app: 'webfixture' }, result(edited))
|
||||
|
||||
expect(second.snapshot.treeText).toBe(edited)
|
||||
expect(second.snapshot.treeDiff).toBeUndefined()
|
||||
})
|
||||
|
||||
it('returns the full tree when the action landed in another window', () => {
|
||||
const baselines = new ComputerTreeBaselines()
|
||||
baselines.apply({ app: 'webfixture' }, result(tree(BEFORE), 42))
|
||||
|
||||
const second = baselines.apply({ app: 'webfixture', treeDiff: true }, result(edited, 43))
|
||||
|
||||
expect(second.snapshot.treeText).toBe(edited)
|
||||
})
|
||||
|
||||
it('keeps each session to its own last view', () => {
|
||||
const baselines = new ComputerTreeBaselines()
|
||||
baselines.apply({ app: 'webfixture', session: 'a' }, result(tree(BEFORE)))
|
||||
baselines.apply({ app: 'webfixture', session: 'b' }, result(edited))
|
||||
|
||||
const fromA = baselines.apply(
|
||||
{ app: 'webfixture', session: 'a', treeDiff: true },
|
||||
result(edited)
|
||||
)
|
||||
|
||||
expect(fromA.snapshot.treeDiff?.changed).toBe(1)
|
||||
})
|
||||
|
||||
it('stops diffing against a view older than the provider snapshot cache', () => {
|
||||
vi.useFakeTimers()
|
||||
const baselines = new ComputerTreeBaselines()
|
||||
baselines.apply({ app: 'webfixture' }, result(tree(BEFORE)))
|
||||
vi.advanceTimersByTime(2 * 60 * 1000 + 1)
|
||||
|
||||
const second = baselines.apply({ app: 'webfixture', treeDiff: true }, result(edited))
|
||||
|
||||
expect(second.snapshot.treeText).toBe(edited)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,168 @@
|
||||
import type { ComputerSnapshotResult } from '../../shared/runtime-types'
|
||||
import {
|
||||
joinComputerTreeText,
|
||||
parseComputerTreeText,
|
||||
type ComputerTreeElement
|
||||
} from '../../shared/computer-use-tree-text'
|
||||
|
||||
const MAX_BASELINES = 32
|
||||
// Matches the provider snapshot caches: older element indexes are refused by actions anyway.
|
||||
const MAX_BASELINE_AGE_MS = 2 * 60 * 1000
|
||||
// A diff that rewrites most of the tree is harder to read than the tree itself.
|
||||
const MAX_DIFF_TO_FULL_RATIO = 0.5
|
||||
|
||||
type ComputerTreeDiff = { text: string; changed: number; added: number; removed: number }
|
||||
|
||||
// Added or moved controls need the full tree so omitted lines cannot hide their location.
|
||||
export function diffComputerTreeText(previous: string, next: string): ComputerTreeDiff | null {
|
||||
const before = parseComputerTreeText(previous)
|
||||
const after = parseComputerTreeText(next)
|
||||
if (before.elements.length === 0 || after.elements.length === 0) {
|
||||
return null
|
||||
}
|
||||
const beforeParents = elementParents(before.elements)
|
||||
const afterParents = elementParents(after.elements)
|
||||
const afterIndexes = new Set(after.elements.map((element) => element.index))
|
||||
const retainedBefore = before.elements.filter((element) => afterIndexes.has(element.index))
|
||||
const body: string[] = []
|
||||
let changed = 0
|
||||
for (const [position, element] of after.elements.entries()) {
|
||||
const old = retainedBefore[position]
|
||||
if (
|
||||
!old ||
|
||||
old.index !== element.index ||
|
||||
old.depth !== element.depth ||
|
||||
beforeParents.get(element.index) !== afterParents.get(element.index)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
if (old.lines.join('\n') !== element.lines.join('\n')) {
|
||||
changed += 1
|
||||
body.push(...element.lines.map((line) => `~ ${line}`))
|
||||
}
|
||||
}
|
||||
const removedIndexes = before.elements
|
||||
.map((element) => element.index)
|
||||
.filter((index) => !afterIndexes.has(index))
|
||||
if (removedIndexes.length > 0) {
|
||||
body.push(`- ${formatIndexRanges(removedIndexes)}`)
|
||||
}
|
||||
const summary = { changed, added: 0, removed: removedIndexes.length }
|
||||
if (body.length === 0) {
|
||||
return {
|
||||
...summary,
|
||||
text: joinComputerTreeText(
|
||||
after.header,
|
||||
['No change in the accessibility tree since the previous snapshot.'],
|
||||
after.footer
|
||||
)
|
||||
}
|
||||
}
|
||||
const text = joinComputerTreeText(
|
||||
after.header,
|
||||
[
|
||||
'Changes since the previous snapshot (~ changed, - removed). Unchanged elements are omitted and keep their indexes.',
|
||||
...body
|
||||
],
|
||||
after.footer
|
||||
)
|
||||
return text.length > next.length * MAX_DIFF_TO_FULL_RATIO ? null : { ...summary, text }
|
||||
}
|
||||
|
||||
function elementParents(elements: ComputerTreeElement[]): Map<number, number | undefined> {
|
||||
const parents = new Map<number, number | undefined>()
|
||||
const path: number[] = []
|
||||
for (const element of elements) {
|
||||
path.length = Math.min(path.length, element.depth)
|
||||
parents.set(element.index, path.at(-1))
|
||||
path[element.depth] = element.index
|
||||
}
|
||||
return parents
|
||||
}
|
||||
|
||||
function formatIndexRanges(indexes: number[]): string {
|
||||
const sorted = [...indexes].sort((a, b) => a - b)
|
||||
const ranges: string[] = []
|
||||
let start = sorted[0]
|
||||
let end = start
|
||||
for (const index of sorted.slice(1)) {
|
||||
if (index === end + 1) {
|
||||
end = index
|
||||
continue
|
||||
}
|
||||
ranges.push(start === end ? String(start) : `${start}-${end}`)
|
||||
start = index
|
||||
end = index
|
||||
}
|
||||
ranges.push(start === end ? String(start) : `${start}-${end}`)
|
||||
return ranges.join(', ')
|
||||
}
|
||||
|
||||
type Baseline = {
|
||||
windowKey: string
|
||||
snapshotId: string
|
||||
treeText: string
|
||||
createdAtMs: number
|
||||
}
|
||||
|
||||
// Remembers the last full tree returned per snapshot namespace and app process.
|
||||
export class ComputerTreeBaselines {
|
||||
private readonly entries = new Map<string, Baseline>()
|
||||
|
||||
apply<T extends ComputerSnapshotResult>(params: Record<string, unknown>, result: T): T {
|
||||
const { snapshot } = result
|
||||
const key = baselineKey(params, snapshot.app.pid)
|
||||
const windowKey = snapshotWindowKey(snapshot.window)
|
||||
this.prune()
|
||||
const previous = this.entries.get(key)
|
||||
this.entries.delete(key)
|
||||
this.entries.set(key, {
|
||||
windowKey,
|
||||
snapshotId: snapshot.id,
|
||||
treeText: snapshot.treeText,
|
||||
createdAtMs: Date.now()
|
||||
})
|
||||
if (params.treeDiff !== true || !previous || previous.windowKey !== windowKey) {
|
||||
return result
|
||||
}
|
||||
const diff = diffComputerTreeText(previous.treeText, snapshot.treeText)
|
||||
if (!diff) {
|
||||
return result
|
||||
}
|
||||
const { text, ...summary } = diff
|
||||
return {
|
||||
...result,
|
||||
snapshot: {
|
||||
...snapshot,
|
||||
treeText: text,
|
||||
treeDiff: { baseSnapshotId: previous.snapshotId, ...summary }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private prune(): void {
|
||||
const cutoff = Date.now() - MAX_BASELINE_AGE_MS
|
||||
for (const [key, entry] of this.entries) {
|
||||
if (this.entries.size > MAX_BASELINES || entry.createdAtMs < cutoff) {
|
||||
this.entries.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function baselineKey(params: Record<string, unknown>, pid: number): string {
|
||||
const namespace =
|
||||
typeof params.session === 'string' && params.session
|
||||
? `session:${params.session}`
|
||||
: typeof params.worktree === 'string' && params.worktree
|
||||
? `worktree:${params.worktree}`
|
||||
: 'default'
|
||||
return `${namespace}\u0000${pid}`
|
||||
}
|
||||
|
||||
function snapshotWindowKey(window: ComputerSnapshotResult['snapshot']['window']): string {
|
||||
if (window.id !== null && window.id !== undefined) {
|
||||
return `id:${window.id}`
|
||||
}
|
||||
return window.index !== null && window.index !== undefined ? `index:${window.index}` : 'window'
|
||||
}
|
||||
@@ -46,7 +46,7 @@ describe.skipIf(process.platform === 'win32')('real sidecar exit reaping', () =>
|
||||
await once(child.stdout, 'data');
|
||||
child.stdout.destroy();
|
||||
child.kill('SIGSTOP');
|
||||
return { ready: true };
|
||||
return { ready: true, supports: { observation: {} } };
|
||||
}};
|
||||
}
|
||||
export function shutdownComputerProviders() {
|
||||
|
||||
@@ -21,6 +21,7 @@ export const CLI_BOOLEAN_FLAGS = new Set([
|
||||
'force',
|
||||
'fresh',
|
||||
'full',
|
||||
'full-tree',
|
||||
'help',
|
||||
'inject',
|
||||
'include-archived',
|
||||
|
||||
@@ -51,6 +51,13 @@ export type ComputerSnapshotData = {
|
||||
treeText: string
|
||||
elementCount: number
|
||||
focusedElementId: number | null
|
||||
// Present only when treeText is a diff against the previous snapshot of the same window.
|
||||
treeDiff?: {
|
||||
baseSnapshotId: string
|
||||
changed: number
|
||||
added: number
|
||||
removed: number
|
||||
}
|
||||
truncation?: {
|
||||
truncated: boolean
|
||||
maxNodes?: number
|
||||
@@ -152,6 +159,8 @@ export type ComputerProviderCapabilities = {
|
||||
annotatedScreenshot: boolean
|
||||
elementFrames: boolean
|
||||
ocr: boolean
|
||||
// Absent from runtimes that predate action-result tree diffs.
|
||||
treeDiff?: boolean
|
||||
}
|
||||
actions: {
|
||||
click: boolean
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
// Every provider renders `<tabs><index> <role> ...` element lines between a header and a footer.
|
||||
export type ComputerTreeElement = {
|
||||
index: number
|
||||
depth: number
|
||||
// The element line plus any unindexed lines that follow it, such as omitted-tab notes.
|
||||
lines: string[]
|
||||
}
|
||||
|
||||
export type ComputerTreeText = {
|
||||
header: string[]
|
||||
elements: ComputerTreeElement[]
|
||||
footer: string[]
|
||||
}
|
||||
|
||||
const ELEMENT_LINE = /^(\t*)(\d+)(?= |$)/u
|
||||
|
||||
export function parseComputerTreeText(treeText: string): ComputerTreeText {
|
||||
const lines = treeText.split('\n')
|
||||
const first = lines.findIndex((line) => ELEMENT_LINE.test(line))
|
||||
if (first === -1) {
|
||||
return { header: lines, elements: [], footer: [] }
|
||||
}
|
||||
const lastElement = lines.findLastIndex((line) => ELEMENT_LINE.test(line))
|
||||
// Why: the footer starts at the first blank line after the last element; anything else between
|
||||
// elements belongs to the element above, including names that carried raw newlines.
|
||||
const blank = lines.indexOf('', lastElement)
|
||||
const footerStart = blank === -1 ? lines.length : blank
|
||||
const elements: ComputerTreeElement[] = []
|
||||
for (const line of lines.slice(first, footerStart)) {
|
||||
const match = ELEMENT_LINE.exec(line)
|
||||
if (match) {
|
||||
elements.push({ index: Number(match[2]), depth: match[1].length, lines: [line] })
|
||||
} else {
|
||||
elements.at(-1)?.lines.push(line)
|
||||
}
|
||||
}
|
||||
return { header: lines.slice(0, first), elements, footer: lines.slice(footerStart) }
|
||||
}
|
||||
|
||||
export function joinComputerTreeText(header: string[], body: string[], footer: string[]): string {
|
||||
return [...trimTrailingBlank(header), '', ...body, ...footer].join('\n')
|
||||
}
|
||||
|
||||
function trimTrailingBlank(lines: string[]): string[] {
|
||||
let end = lines.length
|
||||
while (end > 0 && lines[end - 1] === '') {
|
||||
end -= 1
|
||||
}
|
||||
return lines.slice(0, end)
|
||||
}
|
||||
@@ -25,6 +25,8 @@ export const ComputerTarget = z.object({
|
||||
export const ComputerObserveTargetBase = ComputerTarget.extend({
|
||||
noScreenshot: OptionalBoolean,
|
||||
restoreWindow: OptionalBoolean,
|
||||
// Older runtimes strip this and return the full tree, so callers need no capability check.
|
||||
treeDiff: OptionalBoolean,
|
||||
windowId: OptionalNonNegativeInt,
|
||||
windowIndex: OptionalNonNegativeInt
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user