diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index be075f766f0..4f6b8f31ae7 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -39,6 +39,18 @@ Element indexes are scoped to the latest `get-app-state` result and may be **spa Action results return only the accessibility-tree lines that changed since the last state for that window (`~` changed, `+` added, `-` removed), marked by `snapshot.treeDiff`; pass `--full-tree` to get the whole tree. To find one control in a large window, `get-app-state --query ` prints only matching elements and their ancestors, and `--max-elements ` caps the output. +## Waiting for the app + +Instead of sleeping between actions, wait for what the app should show next. `wait` returns as soon as the condition holds, prints the matching elements or windows, and fails with `wait_timeout` after `--timeout` milliseconds (default 10000, at most 120000): + +```bash +orca computer wait --app com.apple.TextEdit --text "Saved" --json +orca computer wait --app com.apple.TextEdit --enabled "Continue" --json +orca computer wait --app com.apple.TextEdit --window-title "Save As" --json +``` + +`--text` matches element lines the same way as `get-app-state --query`. `--enabled` holds once elements containing the text exist and none of them is disabled. `--window-title` checks the window titles `list-windows` reports (on Windows, only the main window's). `wait` only reads the app and keeps checking while the app or window doesn't exist yet. `--enabled` needs a runtime whose provider marks disabled elements (`supports.observation.disabledState` in `capabilities`). + ## Multi-window apps ```bash diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index b276ecd05dc..c81e3812534 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -135,8 +135,11 @@ def pid_of(node): def has_state(node, state): - state_set = attempt(node.get_state_set) - return bool(state_set is not None and attempt(lambda: state_set.contains(state), False)) + return state_set_contains(attempt(node.get_state_set), state) + + +def state_set_contains(state_set, state, default=False): + return bool(state_set is not None and attempt(lambda: state_set.contains(state), default)) class GeometryReader: @@ -401,8 +404,17 @@ def accessible_id(node): return str(attempt(node.get_accessible_id, "") or "") +def is_disabled(state_set): + # Why: toolkits set ENABLED and SENSITIVE on usable nodes; a node whose states can't be read is not called disabled. + usable = [getattr(Atspi.StateType, name, None) for name in ("ENABLED", "SENSITIVE")] + if state_set is None or None in usable: + return False + return not any(state_set_contains(state_set, state, True) for state in usable) + + def record(node, index, path): role = role_of(node) + state_set = attempt(node.get_state_set) return { "index": index, "runtimeId": path, @@ -412,7 +424,8 @@ def record(node, index, path): "localizedControlType": role, "className": str(attempt(node.get_toolkit_name, "") or ""), "value": string_value(node), - "isSelected": has_state(node, Atspi.StateType.SELECTED), + "isSelected": state_set_contains(state_set, Atspi.StateType.SELECTED), + "isDisabled": is_disabled(state_set), "nativeWindowHandle": 0, "frame": None, "actions": action_labels(node), @@ -500,7 +513,8 @@ def render_accessibility_tree(root, root_path, compact_browser_tabs=False, geome if geometry_nodes is not None: geometry_nodes[id(item)] = node role_label = item["localizedControlType"] or item["controlType"] - line = f'{item["index"]} {role_label} {sanitize_text(title)}'.rstrip() + traits = " (disabled)" if item["isDisabled"] else "" + line = f'{item["index"]} {role_label}{traits} {sanitize_text(title)}'.rstrip() line += formatted_value(role_key, title, item["value"]) if generic_summary and generic_summary != title: line += ", Text: " + sanitize_text(generic_summary) @@ -1264,7 +1278,7 @@ def handshake_response(): "supports": { "apps": {"list": True, "bundleIds": False, "pids": True}, "windows": {"list": True, "targetById": False, "targetByIndex": True, "focus": False, "moveResize": False}, - "observation": {"screenshot": has_screenshot, "annotatedScreenshot": False, "elementFrames": True, "ocr": False}, + "observation": {"screenshot": has_screenshot, "annotatedScreenshot": False, "elementFrames": True, "ocr": False, "disabledState": True}, "actions": { "click": True, "typeText": True, diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py index 5c31d6126f4..6e997611596 100644 --- a/native/computer-use-linux/runtime_render_test.py +++ b/native/computer-use-linux/runtime_render_test.py @@ -18,6 +18,8 @@ class FakeStateType: SHOWING = 4 FOCUSED = 5 EDITABLE = 6 + ENABLED = 7 + SENSITIVE = 8 class FakeStateSet: @@ -668,6 +670,19 @@ class RuntimeRenderTest(unittest.TestCase): keys.assert_not_called() paste.assert_not_called() + def test_tree_marks_controls_without_enabled_or_sensitive_state_disabled(self): + disabled = FakeAccessible("push button", "Continue") + disabled.get_state_set = lambda: FakeStateSet(FakeStateType.SHOWING) + enabled = FakeAccessible("push button", "Cancel") + enabled.get_state_set = lambda: FakeStateSet(FakeStateType.ENABLED, FakeStateType.SENSITIVE) + unreadable = FakeAccessible("push button", "Help") + root = FakeAccessible("frame", "Form", children=[disabled, enabled, unreadable]) + with SnapshotFixture(root): + lines = runtime.make_snapshot("Fixture", False)["treeLines"] + self.assertIn("\t1 push button (disabled) Continue", lines) + self.assertIn("\t2 push button Cancel", lines) + self.assertIn("\t3 push button Help", lines) + def test_compacted_browser_records_keep_their_own_frames_and_indexes(self): tabs = [FakeAccessible("page tab", f"Tab {i}", value="1" if i == 9 else "") for i in range(10)] buttons = [FakeAccessible("button", f"Other {i}") for i in range(10)] diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 43ab1f782f3..9a43b296403 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -553,6 +553,7 @@ final class Provider { "annotatedScreenshot": false, "elementFrames": true, "ocr": false, + "disabledState": true, ], "actions": [ "click": true, diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index 6ac335bbe97..b8d86341d86 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -761,12 +761,17 @@ function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) { className = Get-OrcaProperty $Element "ClassName" value = Get-OrcaValueText $Element isSelected = Test-OrcaElementSelected $Element + isDisabled = Test-OrcaElementDisabled $Element nativeWindowHandle = $nativeWindowHandle frame = Get-OrcaElementFrame $Element $WindowFrame actions = @(Get-OrcaActions $Element) } } +function Test-OrcaElementDisabled($Element) { + try { return -not [bool]$Element.Current.IsEnabled } catch { return $false } +} + function Test-OrcaElementSelected($Element) { try { $pattern = $Element.GetCurrentPattern([Windows.Automation.SelectionItemPattern]::Pattern) @@ -824,7 +829,8 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = $records.Add($record) $nextIndex.Value++ - $line = "$($record.index) $role $(Format-OrcaSnapshotText $title)".TrimEnd() + $traits = if ($record.isDisabled) { " (disabled)" } else { "" } + $line = "$($record.index) $role$traits $(Format-OrcaSnapshotText $title)".TrimEnd() $line += Format-OrcaValueSegment $roleKey $title $record.value if (-not [string]::IsNullOrWhiteSpace($genericSummary) -and $genericSummary -ne $title) { $line += ", Text: " + (Format-OrcaSnapshotText $genericSummary) @@ -1099,7 +1105,7 @@ function Get-OrcaHandshake { supports = [pscustomobject]@{ apps = [pscustomobject]@{ list = $true; bundleIds = $false; pids = $true } windows = [pscustomobject]@{ list = $true; targetById = $true; targetByIndex = $true; focus = $false; moveResize = $false } - observation = [pscustomobject]@{ screenshot = $true; annotatedScreenshot = $false; elementFrames = $true; ocr = $false } + observation = [pscustomobject]@{ screenshot = $true; annotatedScreenshot = $false; elementFrames = $true; ocr = $false; disabledState = $true } actions = [pscustomobject]@{ click = $true typeText = $true diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 8cc9609a614..4e1506d0e88 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -46,6 +46,22 @@ In a large window, look for one control with `get-app-state --query `: it ORCA computer get-app-state --app com.apple.finder --query Downloads --json ``` +## Waiting + +Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout ` up to 120000). + +```text +ORCA computer wait --app --text "Saved" --json +ORCA computer wait --app --enabled "Continue" --json +ORCA computer wait --app --window-title "Save As" --json +``` + +- `--text `: an element line contains the text (case-insensitive, like `--query`). +- `--enabled `: an element contains the text and none of the elements containing it is disabled. Use text only the intended control shows; a label that repeats the button text also has to be enabled. +- `--window-title `: one of the app's windows has the text in its title, as `list-windows` reports them. Then target that window with its `--window-id` or `--window-index`. On Windows, `list-windows` reports only the app's main window, so wait for a dialog with `--text` instead. + +`wait` only reads the app; it never acts, raises or restores a window. The app or window may be missing at first; `wait` keeps checking until it appears. Element indexes in the output come from a fresh tree, so you can act on them next. + ## App Selectors Prefer bundle IDs from `list-apps`; names are acceptable when unambiguous. Use `pid:` only when bundle ID or name matching is ambiguous. @@ -67,6 +83,7 @@ ORCA computer list-apps --json ORCA computer list-windows --app --json ORCA computer get-app-state --app --json ORCA computer get-app-state --app --restore-window --json +ORCA computer wait --app (--text | --enabled | --window-title ) [--timeout ] --json ORCA computer click --app --element-index --json ORCA computer click --app --x 100 --y 100 --json ORCA computer click --app --x 100 --y 100 --modifiers CmdOrCtrl+Shift --json @@ -157,7 +174,7 @@ ORCA computer set-value --app com.microsoft.edgemac --element-index { + it.each([ + ['12 (disabled) File', true], + ['\t3 push button (selected, disabled, settable) Save', true], + ['4 action (primary) (disabled) Continue', true], + ['5 push button (selected) Save (beta)', false], + ['6 push button Continue, Description: Next step', false] + ])('reads %s as disabled=%s', (line, disabled) => { + expect(isComputerTreeElementDisabled({ index: 0, depth: 0, lines: [line] })).toBe(disabled) + }) +}) diff --git a/src/cli/computer/computer-tree-query.ts b/src/cli/computer/computer-tree-query.ts index a7971a81f3e..08e141f2df6 100644 --- a/src/cli/computer/computer-tree-query.ts +++ b/src/cli/computer/computer-tree-query.ts @@ -13,9 +13,7 @@ export type ComputerTreeQuery = { export function queryComputerTreeText(treeText: string, options: ComputerTreeQuery): string { const { header, elements, footer } = parseComputerTreeText(treeText) const needle = options.query?.toLowerCase() - const candidates = needle - ? elements.filter((element) => elementText(element).toLowerCase().includes(needle)) - : elements + const candidates = needle ? computerTreeElementsMatching(elements, needle) : elements const shown = options.maxElements === undefined ? candidates : candidates.slice(0, options.maxElements) const shownSet = new Set(shown) @@ -33,6 +31,23 @@ export function queryComputerTreeText(treeText: string, options: ComputerTreeQue return joinComputerTreeText(header, note ? [note, ...body] : body, footer) } +// Case-insensitive, like --query, over each element's line without its index. +export function computerTreeElementsMatching( + elements: ComputerTreeElement[], + query: string +): ComputerTreeElement[] { + const needle = query.toLowerCase() + return elements.filter((element) => elementText(element).toLowerCase().includes(needle)) +} + +// Providers render traits after the role, whose own text may hold parentheses: `12 button (selected, disabled) Save`. +const DISABLED_TRAIT = + /\((?:(?:selected|expanded|settable), )*disabled(?:, (?:selected|expanded|settable))*\)/u + +export function isComputerTreeElementDisabled(element: ComputerTreeElement): boolean { + return DISABLED_TRAIT.test(element.lines[0]) +} + function elementText(element: ComputerTreeElement): string { return element.lines[0].trimStart().replace(/^\d+ ?/u, '') } diff --git a/src/cli/computer/computer-wait-command.ts b/src/cli/computer/computer-wait-command.ts new file mode 100644 index 00000000000..16852c07b0b --- /dev/null +++ b/src/cli/computer/computer-wait-command.ts @@ -0,0 +1,73 @@ +import { + COMPUTER_WAIT_DEFAULT_TIMEOUT_MS, + COMPUTER_WAIT_MAX_TIMEOUT_MS +} from '../../shared/computer-wait-timeout' +import { + formatComputerWaitSnapshot, + formatComputerWaitWindows, + waitForComputerSnapshot, + waitForComputerWindowTitle, + type ComputerWaitCondition +} from './computer-wait' +import type { CommandHandler } from '../dispatch/dispatch' +import { + getOptionalPositiveIntegerFlag, + getOptionalStringFlag, + getRequiredStringFlag +} from '../arguments/flags' +import { printResult } from '../output/command-result-output' +import { RuntimeClientError } from '../runtime/types' +import { getComputerCommandTarget } from './computer-command-target' +import { getComputerWindowTargetFlags } from './computer-action-flags' + +export const runComputerWait: CommandHandler = async ({ flags, client, cwd, json }) => { + const condition = getComputerWaitCondition(flags) + const timeoutMs = getComputerWaitTimeoutMs(flags) + const windowTarget = getComputerWindowTargetFlags(flags) + if (condition.kind === 'window-title') { + if (Object.keys(windowTarget).length > 0) { + throw new RuntimeClientError( + 'invalid_argument', + "--window-title waits across all of the app's windows; drop --window-id and --window-index" + ) + } + const app = getRequiredStringFlag(flags, 'app') + const result = await waitForComputerWindowTitle(client, condition, app, timeoutMs) + printResult(result, json, formatComputerWaitWindows) + return + } + const target = await getComputerCommandTarget(flags, cwd, client) + const result = await waitForComputerSnapshot( + client, + condition, + { ...target, ...windowTarget }, + timeoutMs + ) + printResult(result, json, formatComputerWaitSnapshot) +} + +function getComputerWaitCondition(flags: Map): ComputerWaitCondition { + const conditions = (['text', 'enabled', 'window-title'] as const).flatMap((kind) => { + const value = getOptionalStringFlag(flags, kind) + return value === undefined ? [] : [{ kind, value }] + }) + if (conditions.length !== 1) { + throw new RuntimeClientError( + 'invalid_argument', + 'Pass exactly one of --text , --enabled or --window-title ' + ) + } + return conditions[0] +} + +function getComputerWaitTimeoutMs(flags: Map): number { + const timeoutMs = + getOptionalPositiveIntegerFlag(flags, 'timeout') ?? COMPUTER_WAIT_DEFAULT_TIMEOUT_MS + if (timeoutMs > COMPUTER_WAIT_MAX_TIMEOUT_MS) { + throw new RuntimeClientError( + 'invalid_argument', + `--timeout must be at most ${COMPUTER_WAIT_MAX_TIMEOUT_MS} ms` + ) + } + return timeoutMs +} diff --git a/src/cli/computer/computer-wait.test.ts b/src/cli/computer/computer-wait.test.ts new file mode 100644 index 00000000000..96ede0e5102 --- /dev/null +++ b/src/cli/computer/computer-wait.test.ts @@ -0,0 +1,166 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const callMock = vi.fn() + +type RuntimeClientModule = typeof RuntimeClientExports + +vi.mock('../runtime/client', async (importOriginal) => ({ + ...(await importOriginal()), + RuntimeClient: class { + call = callMock + getCliStatus = vi.fn() + openOrca = vi.fn() + } +})) + +import { main } from '../index' +import type * as RuntimeClientExports from '../runtime/client' +import { RuntimeRpcFailureError } from '../runtime/types' +import { okFixture, queueFixtures } from '../runtime/fixtures/runtime-rpc-fixtures' + +describe('orca computer wait', () => { + beforeEach(() => { + callMock.mockReset() + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + process.exitCode = undefined + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('polls until the text appears, waiting out a window that is not open yet', async () => { + callMock.mockRejectedValueOnce( + new RuntimeRpcFailureError({ + id: 'req_0', + ok: false, + error: { code: 'window_not_found', message: 'no window' }, + _meta: { runtimeId: 'runtime-1' } + }) + ) + queueFixtures( + callMock, + okFixture('req_1', snapshot('0 window Editor\n\t1 text Saving…')), + okFixture('req_2', snapshot('0 window Editor\n\t1 text All changes saved\n\t2 button Close')) + ) + + await run('--text', 'saved') + + expect(callMock).toHaveBeenCalledTimes(3) + expect(callMock).toHaveBeenLastCalledWith( + 'computer.getAppState', + { session: 'manual', app: 'TextEdit', noScreenshot: true }, + expect.anything() + ) + const output = JSON.parse(vi.mocked(console.log).mock.calls[0][0]) + expect(output.result.wait).toMatchObject({ kind: 'text', value: 'saved' }) + expect(output.result.snapshot.treeText).toContain('1 text All changes saved') + expect(output.result.snapshot.treeText).not.toContain('2 button Close') + }) + + it('waits until every element containing the text is enabled', async () => { + queueFixtures( + callMock, + okFixture('req_caps', capabilities(true)), + okFixture( + 'req_1', + snapshot('0 window Form\n\t1 text Press Continue\n\t2 button (disabled) Continue') + ), + okFixture('req_2', snapshot('0 window Form\n\t1 text Press Continue\n\t2 button Continue')) + ) + + await run('--enabled', 'Continue') + + expect(callMock).toHaveBeenCalledTimes(3) + expect(process.exitCode).toBeUndefined() + }) + + it('refuses --enabled on a provider that does not mark disabled elements', async () => { + queueFixtures(callMock, okFixture('req_caps', capabilities(undefined))) + + await run('--enabled', 'Continue') + + expect(callMock).toHaveBeenCalledTimes(1) + expect(process.exitCode).toBe(1) + expect(errorOutput()).toContain('unsupported_capability') + }) + + it('fails with wait_timeout and what it last saw', async () => { + callMock.mockResolvedValue(okFixture('req_1', snapshot('0 window Editor\n\t1 text Saving…'))) + + await run('--text', 'saved', '--timeout', '1') + + expect(process.exitCode).toBe(1) + const error = errorOutput() + expect(error).toContain('wait_timeout') + expect(error).toContain('no element contains \\"saved\\"') + }) + + it('waits for a window title across the app windows', async () => { + queueFixtures( + callMock, + okFixture('req_1', windows(['Untitled'])), + okFixture('req_2', windows(['Untitled', 'Save As'])) + ) + + await run('--window-title', 'save as') + + expect(callMock).toHaveBeenLastCalledWith( + 'computer.listWindows', + { app: 'TextEdit' }, + expect.anything() + ) + const output = JSON.parse(vi.mocked(console.log).mock.calls[0][0]) + expect(output.result.windows.map((window: { title: string }) => window.title)).toEqual([ + 'Save As' + ]) + }) +}) + +async function run(...args: string[]): Promise { + await main( + ['computer', 'wait', '--session', 'manual', '--app', 'TextEdit', ...args, '--json'], + '/tmp/repo' + ) +} + +function errorOutput(): string { + return vi + .mocked(console.log) + .mock.calls.map((call) => String(call[0])) + .join('\n') +} + +function snapshot(treeText: string) { + return { + snapshot: { + id: 'snap', + app: { name: 'TextEdit', bundleId: 'com.apple.TextEdit', pid: 7 }, + window: { title: 'Editor', width: 800, height: 600 }, + coordinateSpace: 'window', + treeText: `App=com.apple.TextEdit (pid 7)\nWindow: "Editor"\n\n${treeText}`, + elementCount: 3, + focusedElementId: null + }, + screenshot: null, + screenshotStatus: { state: 'skipped', reason: 'no_screenshot_flag' } + } +} + +function capabilities(disabledState: boolean | undefined) { + return { + platform: 'linux', + provider: 'orca-computer-use-linux', + providerVersion: '1', + protocolVersion: 1, + supports: { observation: { screenshot: true, disabledState } } + } +} + +function windows(titles: string[]) { + return { + app: { name: 'TextEdit', bundleId: 'com.apple.TextEdit', pid: 7 }, + windows: titles.map((title, index) => ({ title, index, id: index, width: 800, height: 600 })) + } +} diff --git a/src/cli/computer/computer-wait.ts b/src/cli/computer/computer-wait.ts new file mode 100644 index 00000000000..520e5375d23 --- /dev/null +++ b/src/cli/computer/computer-wait.ts @@ -0,0 +1,229 @@ +import type { + ComputerListWindowsResult, + ComputerProviderCapabilities, + ComputerSnapshotResult, + ComputerWindowListWindow +} from '../../shared/computer-use-runtime-types' +import { parseComputerTreeText } from '../../shared/computer-use-tree-text' +import { + computerTreeElementsMatching, + isComputerTreeElementDisabled, + queryComputerTreeText +} from './computer-tree-query' +import { cliErrorCode } from '../output/cli-error' +import { formatGetAppState, formatListWindows } from './computer-format' +import type { RuntimeClient } from '../runtime/client' +import { RuntimeClientError } from '../runtime/types' +import type { RuntimeRpcSuccess } from '../../shared/runtime-rpc-envelope' + +export type ComputerWaitCondition = + | { kind: 'text'; value: string } + | { kind: 'enabled'; value: string } + | { kind: 'window-title'; value: string } + +type ComputerTreeWaitCondition = Exclude + +export type ComputerWaitSummary = ComputerWaitCondition & { elapsedMs: number } + +export type ComputerWaitSnapshotResult = ComputerSnapshotResult & { wait: ComputerWaitSummary } + +export type ComputerWaitWindowsResult = ComputerListWindowsResult & { wait: ComputerWaitSummary } + +const POLL_INTERVAL_MS = 250 +// Why: a read started near the deadline still gets long enough to answer for a large window. +const MIN_READ_TIMEOUT_MS = 2_000 +// Why: the app or window may not exist yet; waiting for it to appear is part of the wait. +const NOT_YET_CODES = new Set(['app_not_found', 'window_not_found', 'window_stale']) + +type Poll = { + call: (timeoutMs: number) => Promise> + check: (result: T) => string | null +} + +// Polls the same reads every provider and runtime version serves, so the wait needs no host support. +export async function waitForComputerSnapshot( + client: RuntimeClient, + condition: ComputerTreeWaitCondition, + params: Record, + timeoutMs: number +): Promise> { + const startedAt = Date.now() + if (condition.kind === 'enabled') { + await assertDisabledStateSupported(client, Math.max(timeoutMs, MIN_READ_TIMEOUT_MS)) + } + const { response, elapsedMs } = await pollUntil(condition, startedAt, timeoutMs, { + call: (readTimeoutMs) => + client.call( + 'computer.getAppState', + { ...params, noScreenshot: true }, + { timeoutMs: readTimeoutMs } + ), + check: (result) => snapshotMiss(condition, result.snapshot.treeText) + }) + const { snapshot } = response.result + return { + ...response, + result: { + ...response.result, + snapshot: { + ...snapshot, + treeText: queryComputerTreeText(snapshot.treeText, { query: condition.value }) + }, + wait: { ...condition, elapsedMs } + } + } +} + +export async function waitForComputerWindowTitle( + client: RuntimeClient, + condition: Extract, + app: string, + timeoutMs: number +): Promise> { + const { response, elapsedMs } = await pollUntil(condition, Date.now(), timeoutMs, { + call: (readTimeoutMs) => + client.call( + 'computer.listWindows', + { app }, + { timeoutMs: readTimeoutMs } + ), + check: (result) => windowTitleMiss(result.windows, condition.value) + }) + return { + ...response, + result: { + ...response.result, + windows: matchingWindows(response.result.windows, condition.value), + wait: { ...condition, elapsedMs } + } + } +} + +function snapshotMiss(condition: ComputerTreeWaitCondition, treeText: string): string | null { + const matches = computerTreeElementsMatching( + parseComputerTreeText(treeText).elements, + condition.value + ) + if (matches.length === 0) { + return `no element contains "${condition.value}"` + } + if (condition.kind === 'text') { + return null + } + const disabled = matches.filter(isComputerTreeElementDisabled) + if (disabled.length === 0) { + return null + } + return `${disabled.length} of ${matches.length} matching elements are disabled: ${disabled + .slice(0, 3) + .map((element) => element.lines[0].trim()) + .join(' | ')}` +} + +function windowTitleMiss(windows: ComputerWindowListWindow[], title: string): string | null { + if (matchingWindows(windows, title).length > 0) { + return null + } + if (windows.length > 0 && windows.every((window) => window.title === '')) { + return 'every window title is empty (on macOS, window titles need the Screen Recording permission)' + } + return `its windows are ${windows.map((window) => `"${window.title}"`).join(', ') || 'none'}` +} + +function matchingWindows( + windows: ComputerWindowListWindow[], + title: string +): ComputerWindowListWindow[] { + const needle = title.toLowerCase() + return windows.filter((window) => window.title.toLowerCase().includes(needle)) +} + +async function assertDisabledStateSupported( + client: RuntimeClient, + readTimeoutMs: number +): Promise { + const { result } = await client.call( + 'computer.capabilities', + {}, + { timeoutMs: readTimeoutMs } + ) + if (result.supports.observation.disabledState !== true) { + // Why: without the trait every element reads as enabled, so the wait would pass at once. + throw new RuntimeClientError( + 'unsupported_capability', + `The ${result.provider} computer-use provider on this host does not report disabled elements, so --enabled cannot be checked. Update Orca on that host, or wait with --text instead.` + ) + } +} + +async function pollUntil( + condition: ComputerWaitCondition, + startedAt: number, + timeoutMs: number, + poll: Poll +): Promise<{ response: RuntimeRpcSuccess; elapsedMs: number }> { + for (;;) { + let miss: string + try { + const remainingMs = timeoutMs - (Date.now() - startedAt) + const response = await poll.call(Math.max(remainingMs, MIN_READ_TIMEOUT_MS)) + const result = poll.check(response.result) + if (result === null) { + return { response, elapsedMs: Date.now() - startedAt } + } + miss = result + } catch (error) { + const code = cliErrorCode(error) + if (code === 'runtime_timeout' && Date.now() - startedAt >= timeoutMs) { + miss = 'the last read did not answer before the deadline' + } else if (code && NOT_YET_CODES.has(code)) { + miss = `${code}: ${error instanceof Error ? error.message : String(error)}` + } else { + throw error + } + } + const remainingMs = timeoutMs - (Date.now() - startedAt) + if (remainingMs <= 0) { + throw waitTimeoutError(condition, timeoutMs, miss) + } + await new Promise((resolve) => setTimeout(resolve, Math.min(POLL_INTERVAL_MS, remainingMs))) + } +} + +function waitTimeoutError( + condition: ComputerWaitCondition, + timeoutMs: number, + lastSeen: string +): RuntimeClientError { + const goal = + condition.kind === 'text' + ? `an element containing "${condition.value}"` + : condition.kind === 'enabled' + ? `elements containing "${condition.value}" to be enabled` + : `a window titled "${condition.value}"` + return new RuntimeClientError( + 'wait_timeout', + `Timed out after ${timeoutMs} ms waiting for ${goal}; last check: ${lastSeen}.`, + { + nextSteps: [ + 'Run `orca computer get-app-state --app --json` to see what the app shows now.', + 'If the app is still working, wait again with a longer --timeout; otherwise match text the tree actually shows.', + ...(condition.kind === 'enabled' + ? ['Every matching element must be enabled, so use text only the intended control shows.'] + : []) + ] + } + ) +} + +export function formatComputerWaitSnapshot(result: ComputerWaitSnapshotResult): string { + return `${waitHeadline(result.wait)}\n${formatGetAppState(result)}` +} + +export function formatComputerWaitWindows(result: ComputerWaitWindowsResult): string { + return `${waitHeadline(result.wait)}\n${formatListWindows(result)}` +} + +function waitHeadline(wait: ComputerWaitSummary): string { + return `Condition --${wait.kind} "${wait.value}" held after ${wait.elapsedMs} ms.` +} diff --git a/src/cli/computer/computer.ts b/src/cli/computer/computer.ts index af844b40ef5..7a5745c5ec1 100644 --- a/src/cli/computer/computer.ts +++ b/src/cli/computer/computer.ts @@ -12,6 +12,7 @@ import { getRequiredStringFlag } from '../arguments/flags' import { queryComputerTreeText } from './computer-tree-query' +import { runComputerWait } from './computer-wait-command' import { RuntimeClientError } from '../runtime/types' import { formatComputerAction, @@ -95,6 +96,7 @@ export const COMPUTER_HANDLERS: Record = { } printResult(result, json, formatGetAppState) }, + 'computer wait': runComputerWait, 'computer click': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) diff --git a/src/cli/discovery/command-scoped-flag-help.ts b/src/cli/discovery/command-scoped-flag-help.ts index a3ecbf8389d..b23764dd9f4 100644 --- a/src/cli/discovery/command-scoped-flag-help.ts +++ b/src/cli/discovery/command-scoped-flag-help.ts @@ -1,3 +1,5 @@ +import { COMPUTER_WAIT_DEFAULT_TIMEOUT_MS } from '../../shared/computer-wait-timeout' + // File focus differs from terminal create's shared --focus help. const FILE_OPEN_FOCUS_HELP = "--focus Bring the user to the file (switches Orca's window to its worktree)" @@ -20,6 +22,14 @@ const COMMAND_SCOPED_FLAG_HELP: Record> = { query: '--query Show only elements whose line contains this text, with their ancestors' }, + 'computer wait': { + text: '--text Wait until an element line contains this text', + enabled: + '--enabled Wait until elements containing this text exist and none is disabled', + 'window-title': + "--window-title Wait until one of the app's windows has this in its title", + timeout: `--timeout How long to wait before failing with wait_timeout (default ${COMPUTER_WAIT_DEFAULT_TIMEOUT_MS})` + }, 'reference find': { query: '--query Full URL or native issue key such as STA-1234', repo: '--repo Filter by workspace repository membership', diff --git a/src/cli/discovery/root-help-text-primary.ts b/src/cli/discovery/root-help-text-primary.ts index a58515e3b2a..d073961bd86 100644 --- a/src/cli/discovery/root-help-text-primary.ts +++ b/src/cli/discovery/root-help-text-primary.ts @@ -186,6 +186,7 @@ export const ROOT_HELP_TEXT_PRIMARY = [ ' computer list-apps List running apps available to computer-use', ' computer list-windows List visible windows for a target app', ' computer get-app-state Capture a compact accessibility snapshot of an app', + ' computer wait Wait until an app shows text, enables a control, or opens a window', ' computer click Click an app element or window coordinate', ' computer perform-secondary-action Run an advertised accessibility action', ' computer scroll Scroll an app element', diff --git a/src/cli/dispatch/handler-group-manifest.ts b/src/cli/dispatch/handler-group-manifest.ts index b1671a76bb2..44bbf6624ff 100644 --- a/src/cli/dispatch/handler-group-manifest.ts +++ b/src/cli/dispatch/handler-group-manifest.ts @@ -187,6 +187,7 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [ 'computer permissions', 'computer list-windows', 'computer get-app-state', + 'computer wait', 'computer click', 'computer perform-secondary-action', 'computer scroll', diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 8f7e90423c3..33aeaf25c71 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1065,6 +1065,22 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "ORCA computer get-app-state --app com.apple.finder --query Downloads --json", "```", "", + "## Waiting", + "", + "Never `sleep` between actions. When the UI changes on its own after an action (a save finishes, a dialog opens, a button becomes clickable), wait for the change with `wait`. It returns as soon as the condition holds and prints the matching elements or windows; it fails with `wait_timeout` if the condition does not hold in time (default 10 s, `--timeout ` up to 120000).", + "", + "```text", + "ORCA computer wait --app --text \"Saved\" --json", + "ORCA computer wait --app --enabled \"Continue\" --json", + "ORCA computer wait --app --window-title \"Save As\" --json", + "```", + "", + "- `--text `: an element line contains the text (case-insensitive, like `--query`).", + "- `--enabled `: an element contains the text and none of the elements containing it is disabled. Use text only the intended control shows; a label that repeats the button text also has to be enabled.", + "- `--window-title `: one of the app's windows has the text in its title, as `list-windows` reports them. Then target that window with its `--window-id` or `--window-index`. On Windows, `list-windows` reports only the app's main window, so wait for a dialog with `--text` instead.", + "", + "`wait` only reads the app; it never acts, raises or restores a window. The app or window may be missing at first; `wait` keeps checking until it appears. Element indexes in the output come from a fresh tree, so you can act on them next.", + "", "## App Selectors", "", "Prefer bundle IDs from `list-apps`; names are acceptable when unambiguous. Use `pid:` only when bundle ID or name matching is ambiguous.", @@ -1086,6 +1102,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "ORCA computer list-windows --app --json", "ORCA computer get-app-state --app --json", "ORCA computer get-app-state --app --restore-window --json", + "ORCA computer wait --app (--text | --enabled | --window-title ) [--timeout ] --json", "ORCA computer click --app --element-index --json", "ORCA computer click --app --x 100 --y 100 --json", "ORCA computer click --app --x 100 --y 100 --modifiers CmdOrCtrl+Shift --json", @@ -1176,7 +1193,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "ORCA computer press-key --app com.microsoft.edgemac --key Return --json", "```", "", - "Spotify: refresh after playback clicks; the UI often changes asynchronously.", + "Spotify: the UI often changes asynchronously after playback clicks; `wait --text` for the change instead of re-reading in a loop.", "", "Slack: the accessibility tree may be shallow while the screenshot contains useful information. Reading visible Slack UI is fine when requested; sending messages or triggering workflows still needs explicit permission.", "", @@ -1195,6 +1212,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "- `element_not_enabled`: the element is disabled (its line lists `disabled`), so nothing was sent. Do what enables it first, then act once a fresh `get-app-state` no longer lists it as disabled.", "- `invalid_argument`: fix the command flags; do not retry the same command unchanged.", "- `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow.", + "- `wait_timeout`: the condition did not hold in time. Run `get-app-state` to see what the app shows; wait again with a longer `--timeout` only if it is still working, otherwise match text the tree actually shows.", "- `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`.", "- `screenshot_failed`: use `--no-screenshot` if tree state is enough; if the message names Screen Recording or screenshots permission, run `ORCA computer permissions --id screenshots --json`.", "- `accessibility_error`: run `ORCA computer capabilities --json`; if the message names Accessibility permission, run `ORCA computer permissions --id accessibility --json`.", diff --git a/src/cli/specs/computer.ts b/src/cli/specs/computer.ts index 6ddc43e110a..bd4ea9d5c8d 100644 --- a/src/cli/specs/computer.ts +++ b/src/cli/specs/computer.ts @@ -1,5 +1,9 @@ import type { CommandSpec } from '../discovery/command-spec' import { CLI_GLOBAL_FLAGS as GLOBAL_FLAGS } from '../../shared/cli-argument-boundary' +import { + COMPUTER_WAIT_DEFAULT_TIMEOUT_MS, + COMPUTER_WAIT_MAX_TIMEOUT_MS +} from '../../shared/computer-wait-timeout' const COMPUTER_FLAGS = [...GLOBAL_FLAGS, 'worktree', 'session', 'app'] const COMPUTER_WINDOW_TARGET_FLAGS = ['window-id', 'window-index'] @@ -43,6 +47,25 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [ 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--query ] [--max-elements ] [--restore-window] [--no-screenshot] [--json]', allowedFlags: [...COMPUTER_OBSERVE_FLAGS, 'query', 'max-elements'] }, + { + path: ['computer', 'wait'], + summary: 'Wait until an app shows text, enables a control, or opens a window', + usage: + 'orca computer wait --app (--text | --enabled | --window-title ) [--timeout ] [--window-id | --window-index ] [--worktree | --session ] [--json]', + allowedFlags: [ + ...COMPUTER_FLAGS, + ...COMPUTER_WINDOW_TARGET_FLAGS, + 'text', + 'enabled', + 'window-title', + 'timeout' + ], + notes: [ + 'Polls the accessibility tree (or, for --window-title, the window list) until the condition holds, then prints the matching elements or windows. Matching is a case-insensitive substring, like get-app-state --query.', + '--enabled holds once an element contains the text and no element containing it is disabled.', + `--timeout defaults to ${COMPUTER_WAIT_DEFAULT_TIMEOUT_MS} ms (at most ${COMPUTER_WAIT_MAX_TIMEOUT_MS}); on timeout the command fails with wait_timeout.` + ] + }, { path: ['computer', 'click'], summary: 'Click an app element or window coordinate, optionally with modifiers', diff --git a/src/main/computer/desktop-script/desktop-script-provider-bridge.ts b/src/main/computer/desktop-script/desktop-script-provider-bridge.ts index 1e75632529b..75cc71d6938 100644 --- a/src/main/computer/desktop-script/desktop-script-provider-bridge.ts +++ b/src/main/computer/desktop-script/desktop-script-provider-bridge.ts @@ -220,7 +220,7 @@ export function mapBridgeError(message: string): RuntimeClientError { return new RuntimeClientError('screenshot_failed', text) } if ( - /window_not_found|No top-level(?: AT-SPI| UI Automation)? window|has no (?:on-screen |accessibility )?window|could not match accessibility window|unknown window(?:_index| id)?/i.test( + /window_not_found|windowNotFound|No top-level(?: AT-SPI| UI Automation)? window|has no (?:on-screen |accessibility )?window|could not match accessibility window|unknown window(?:_index| id)?/i.test( text ) ) { diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index 1b853a115dd..42de55e7b3c 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -160,6 +160,8 @@ export type ComputerProviderCapabilities = { ocr: boolean // Absent from runtimes that predate action-result tree diffs. treeDiff?: boolean + // Tree lines mark disabled elements with a `(disabled)` trait. Absent from older Linux and Windows providers. + disabledState?: boolean } actions: { click: boolean diff --git a/src/shared/computer-wait-timeout.ts b/src/shared/computer-wait-timeout.ts new file mode 100644 index 00000000000..e89358603cc --- /dev/null +++ b/src/shared/computer-wait-timeout.ts @@ -0,0 +1,2 @@ +export const COMPUTER_WAIT_DEFAULT_TIMEOUT_MS = 10_000 +export const COMPUTER_WAIT_MAX_TIMEOUT_MS = 120_000