feat(computer-use): add orca computer wait for text, enabled controls and window titles (#41433)

* feat(computer-use): add orca computer wait for text, enabled controls and window titles

* fix(computer-use): bound wait reads by the deadline and wait out missing Linux/Windows windows

* fix(computer-use): read disabled traits after role text with parentheses and bound the capability probe
This commit is contained in:
Kelvin Amoaba
2026-11-12 21:35:26 +00:00
committed by GitHub
parent 0664f1c9bb
commit cd933ae08e
20 changed files with 635 additions and 13 deletions
@@ -0,0 +1,14 @@
import { describe, expect, it } from 'vitest'
import { isComputerTreeElementDisabled } from './computer-tree-query'
describe('isComputerTreeElementDisabled', () => {
it.each([
['12 (disabled) File', true],
['\t3 push button (selected, disabled, settable) Save', true],
['4 action (primary) (disabled) Continue', true],
['5 push button (selected) Save (beta)', false],
['6 push button Continue, Description: Next step', false]
])('reads %s as disabled=%s', (line, disabled) => {
expect(isComputerTreeElementDisabled({ index: 0, depth: 0, lines: [line] })).toBe(disabled)
})
})
+18 -3
View File
@@ -13,9 +13,7 @@ export type ComputerTreeQuery = {
export function queryComputerTreeText(treeText: string, options: ComputerTreeQuery): string {
const { header, elements, footer } = parseComputerTreeText(treeText)
const needle = options.query?.toLowerCase()
const candidates = needle
? elements.filter((element) => elementText(element).toLowerCase().includes(needle))
: elements
const candidates = needle ? computerTreeElementsMatching(elements, needle) : elements
const shown =
options.maxElements === undefined ? candidates : candidates.slice(0, options.maxElements)
const shownSet = new Set(shown)
@@ -33,6 +31,23 @@ export function queryComputerTreeText(treeText: string, options: ComputerTreeQue
return joinComputerTreeText(header, note ? [note, ...body] : body, footer)
}
// Case-insensitive, like --query, over each element's line without its index.
export function computerTreeElementsMatching(
elements: ComputerTreeElement[],
query: string
): ComputerTreeElement[] {
const needle = query.toLowerCase()
return elements.filter((element) => elementText(element).toLowerCase().includes(needle))
}
// Providers render traits after the role, whose own text may hold parentheses: `12 button (selected, disabled) Save`.
const DISABLED_TRAIT =
/\((?:(?:selected|expanded|settable), )*disabled(?:, (?:selected|expanded|settable))*\)/u
export function isComputerTreeElementDisabled(element: ComputerTreeElement): boolean {
return DISABLED_TRAIT.test(element.lines[0])
}
function elementText(element: ComputerTreeElement): string {
return element.lines[0].trimStart().replace(/^\d+ ?/u, '')
}
+73
View File
@@ -0,0 +1,73 @@
import {
COMPUTER_WAIT_DEFAULT_TIMEOUT_MS,
COMPUTER_WAIT_MAX_TIMEOUT_MS
} from '../../shared/computer-wait-timeout'
import {
formatComputerWaitSnapshot,
formatComputerWaitWindows,
waitForComputerSnapshot,
waitForComputerWindowTitle,
type ComputerWaitCondition
} from './computer-wait'
import type { CommandHandler } from '../dispatch/dispatch'
import {
getOptionalPositiveIntegerFlag,
getOptionalStringFlag,
getRequiredStringFlag
} from '../arguments/flags'
import { printResult } from '../output/command-result-output'
import { RuntimeClientError } from '../runtime/types'
import { getComputerCommandTarget } from './computer-command-target'
import { getComputerWindowTargetFlags } from './computer-action-flags'
export const runComputerWait: CommandHandler = async ({ flags, client, cwd, json }) => {
const condition = getComputerWaitCondition(flags)
const timeoutMs = getComputerWaitTimeoutMs(flags)
const windowTarget = getComputerWindowTargetFlags(flags)
if (condition.kind === 'window-title') {
if (Object.keys(windowTarget).length > 0) {
throw new RuntimeClientError(
'invalid_argument',
"--window-title waits across all of the app's windows; drop --window-id and --window-index"
)
}
const app = getRequiredStringFlag(flags, 'app')
const result = await waitForComputerWindowTitle(client, condition, app, timeoutMs)
printResult(result, json, formatComputerWaitWindows)
return
}
const target = await getComputerCommandTarget(flags, cwd, client)
const result = await waitForComputerSnapshot(
client,
condition,
{ ...target, ...windowTarget },
timeoutMs
)
printResult(result, json, formatComputerWaitSnapshot)
}
function getComputerWaitCondition(flags: Map<string, string | boolean>): ComputerWaitCondition {
const conditions = (['text', 'enabled', 'window-title'] as const).flatMap((kind) => {
const value = getOptionalStringFlag(flags, kind)
return value === undefined ? [] : [{ kind, value }]
})
if (conditions.length !== 1) {
throw new RuntimeClientError(
'invalid_argument',
'Pass exactly one of --text <text>, --enabled <text> or --window-title <text>'
)
}
return conditions[0]
}
function getComputerWaitTimeoutMs(flags: Map<string, string | boolean>): number {
const timeoutMs =
getOptionalPositiveIntegerFlag(flags, 'timeout') ?? COMPUTER_WAIT_DEFAULT_TIMEOUT_MS
if (timeoutMs > COMPUTER_WAIT_MAX_TIMEOUT_MS) {
throw new RuntimeClientError(
'invalid_argument',
`--timeout must be at most ${COMPUTER_WAIT_MAX_TIMEOUT_MS} ms`
)
}
return timeoutMs
}
+166
View File
@@ -0,0 +1,166 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const callMock = vi.fn()
type RuntimeClientModule = typeof RuntimeClientExports
vi.mock('../runtime/client', async (importOriginal) => ({
...(await importOriginal<RuntimeClientModule>()),
RuntimeClient: class {
call = callMock
getCliStatus = vi.fn()
openOrca = vi.fn()
}
}))
import { main } from '../index'
import type * as RuntimeClientExports from '../runtime/client'
import { RuntimeRpcFailureError } from '../runtime/types'
import { okFixture, queueFixtures } from '../runtime/fixtures/runtime-rpc-fixtures'
describe('orca computer wait', () => {
beforeEach(() => {
callMock.mockReset()
vi.spyOn(console, 'log').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
process.exitCode = undefined
})
afterEach(() => {
vi.restoreAllMocks()
})
it('polls until the text appears, waiting out a window that is not open yet', async () => {
callMock.mockRejectedValueOnce(
new RuntimeRpcFailureError({
id: 'req_0',
ok: false,
error: { code: 'window_not_found', message: 'no window' },
_meta: { runtimeId: 'runtime-1' }
})
)
queueFixtures(
callMock,
okFixture('req_1', snapshot('0 window Editor\n\t1 text Saving…')),
okFixture('req_2', snapshot('0 window Editor\n\t1 text All changes saved\n\t2 button Close'))
)
await run('--text', 'saved')
expect(callMock).toHaveBeenCalledTimes(3)
expect(callMock).toHaveBeenLastCalledWith(
'computer.getAppState',
{ session: 'manual', app: 'TextEdit', noScreenshot: true },
expect.anything()
)
const output = JSON.parse(vi.mocked(console.log).mock.calls[0][0])
expect(output.result.wait).toMatchObject({ kind: 'text', value: 'saved' })
expect(output.result.snapshot.treeText).toContain('1 text All changes saved')
expect(output.result.snapshot.treeText).not.toContain('2 button Close')
})
it('waits until every element containing the text is enabled', async () => {
queueFixtures(
callMock,
okFixture('req_caps', capabilities(true)),
okFixture(
'req_1',
snapshot('0 window Form\n\t1 text Press Continue\n\t2 button (disabled) Continue')
),
okFixture('req_2', snapshot('0 window Form\n\t1 text Press Continue\n\t2 button Continue'))
)
await run('--enabled', 'Continue')
expect(callMock).toHaveBeenCalledTimes(3)
expect(process.exitCode).toBeUndefined()
})
it('refuses --enabled on a provider that does not mark disabled elements', async () => {
queueFixtures(callMock, okFixture('req_caps', capabilities(undefined)))
await run('--enabled', 'Continue')
expect(callMock).toHaveBeenCalledTimes(1)
expect(process.exitCode).toBe(1)
expect(errorOutput()).toContain('unsupported_capability')
})
it('fails with wait_timeout and what it last saw', async () => {
callMock.mockResolvedValue(okFixture('req_1', snapshot('0 window Editor\n\t1 text Saving…')))
await run('--text', 'saved', '--timeout', '1')
expect(process.exitCode).toBe(1)
const error = errorOutput()
expect(error).toContain('wait_timeout')
expect(error).toContain('no element contains \\"saved\\"')
})
it('waits for a window title across the app windows', async () => {
queueFixtures(
callMock,
okFixture('req_1', windows(['Untitled'])),
okFixture('req_2', windows(['Untitled', 'Save As']))
)
await run('--window-title', 'save as')
expect(callMock).toHaveBeenLastCalledWith(
'computer.listWindows',
{ app: 'TextEdit' },
expect.anything()
)
const output = JSON.parse(vi.mocked(console.log).mock.calls[0][0])
expect(output.result.windows.map((window: { title: string }) => window.title)).toEqual([
'Save As'
])
})
})
async function run(...args: string[]): Promise<void> {
await main(
['computer', 'wait', '--session', 'manual', '--app', 'TextEdit', ...args, '--json'],
'/tmp/repo'
)
}
function errorOutput(): string {
return vi
.mocked(console.log)
.mock.calls.map((call) => String(call[0]))
.join('\n')
}
function snapshot(treeText: string) {
return {
snapshot: {
id: 'snap',
app: { name: 'TextEdit', bundleId: 'com.apple.TextEdit', pid: 7 },
window: { title: 'Editor', width: 800, height: 600 },
coordinateSpace: 'window',
treeText: `App=com.apple.TextEdit (pid 7)\nWindow: "Editor"\n\n${treeText}`,
elementCount: 3,
focusedElementId: null
},
screenshot: null,
screenshotStatus: { state: 'skipped', reason: 'no_screenshot_flag' }
}
}
function capabilities(disabledState: boolean | undefined) {
return {
platform: 'linux',
provider: 'orca-computer-use-linux',
providerVersion: '1',
protocolVersion: 1,
supports: { observation: { screenshot: true, disabledState } }
}
}
function windows(titles: string[]) {
return {
app: { name: 'TextEdit', bundleId: 'com.apple.TextEdit', pid: 7 },
windows: titles.map((title, index) => ({ title, index, id: index, width: 800, height: 600 }))
}
}
+229
View File
@@ -0,0 +1,229 @@
import type {
ComputerListWindowsResult,
ComputerProviderCapabilities,
ComputerSnapshotResult,
ComputerWindowListWindow
} from '../../shared/computer-use-runtime-types'
import { parseComputerTreeText } from '../../shared/computer-use-tree-text'
import {
computerTreeElementsMatching,
isComputerTreeElementDisabled,
queryComputerTreeText
} from './computer-tree-query'
import { cliErrorCode } from '../output/cli-error'
import { formatGetAppState, formatListWindows } from './computer-format'
import type { RuntimeClient } from '../runtime/client'
import { RuntimeClientError } from '../runtime/types'
import type { RuntimeRpcSuccess } from '../../shared/runtime-rpc-envelope'
export type ComputerWaitCondition =
| { kind: 'text'; value: string }
| { kind: 'enabled'; value: string }
| { kind: 'window-title'; value: string }
type ComputerTreeWaitCondition = Exclude<ComputerWaitCondition, { kind: 'window-title' }>
export type ComputerWaitSummary = ComputerWaitCondition & { elapsedMs: number }
export type ComputerWaitSnapshotResult = ComputerSnapshotResult & { wait: ComputerWaitSummary }
export type ComputerWaitWindowsResult = ComputerListWindowsResult & { wait: ComputerWaitSummary }
const POLL_INTERVAL_MS = 250
// Why: a read started near the deadline still gets long enough to answer for a large window.
const MIN_READ_TIMEOUT_MS = 2_000
// Why: the app or window may not exist yet; waiting for it to appear is part of the wait.
const NOT_YET_CODES = new Set(['app_not_found', 'window_not_found', 'window_stale'])
type Poll<T> = {
call: (timeoutMs: number) => Promise<RuntimeRpcSuccess<T>>
check: (result: T) => string | null
}
// Polls the same reads every provider and runtime version serves, so the wait needs no host support.
export async function waitForComputerSnapshot(
client: RuntimeClient,
condition: ComputerTreeWaitCondition,
params: Record<string, unknown>,
timeoutMs: number
): Promise<RuntimeRpcSuccess<ComputerWaitSnapshotResult>> {
const startedAt = Date.now()
if (condition.kind === 'enabled') {
await assertDisabledStateSupported(client, Math.max(timeoutMs, MIN_READ_TIMEOUT_MS))
}
const { response, elapsedMs } = await pollUntil(condition, startedAt, timeoutMs, {
call: (readTimeoutMs) =>
client.call<ComputerSnapshotResult>(
'computer.getAppState',
{ ...params, noScreenshot: true },
{ timeoutMs: readTimeoutMs }
),
check: (result) => snapshotMiss(condition, result.snapshot.treeText)
})
const { snapshot } = response.result
return {
...response,
result: {
...response.result,
snapshot: {
...snapshot,
treeText: queryComputerTreeText(snapshot.treeText, { query: condition.value })
},
wait: { ...condition, elapsedMs }
}
}
}
export async function waitForComputerWindowTitle(
client: RuntimeClient,
condition: Extract<ComputerWaitCondition, { kind: 'window-title' }>,
app: string,
timeoutMs: number
): Promise<RuntimeRpcSuccess<ComputerWaitWindowsResult>> {
const { response, elapsedMs } = await pollUntil(condition, Date.now(), timeoutMs, {
call: (readTimeoutMs) =>
client.call<ComputerListWindowsResult>(
'computer.listWindows',
{ app },
{ timeoutMs: readTimeoutMs }
),
check: (result) => windowTitleMiss(result.windows, condition.value)
})
return {
...response,
result: {
...response.result,
windows: matchingWindows(response.result.windows, condition.value),
wait: { ...condition, elapsedMs }
}
}
}
function snapshotMiss(condition: ComputerTreeWaitCondition, treeText: string): string | null {
const matches = computerTreeElementsMatching(
parseComputerTreeText(treeText).elements,
condition.value
)
if (matches.length === 0) {
return `no element contains "${condition.value}"`
}
if (condition.kind === 'text') {
return null
}
const disabled = matches.filter(isComputerTreeElementDisabled)
if (disabled.length === 0) {
return null
}
return `${disabled.length} of ${matches.length} matching elements are disabled: ${disabled
.slice(0, 3)
.map((element) => element.lines[0].trim())
.join(' | ')}`
}
function windowTitleMiss(windows: ComputerWindowListWindow[], title: string): string | null {
if (matchingWindows(windows, title).length > 0) {
return null
}
if (windows.length > 0 && windows.every((window) => window.title === '')) {
return 'every window title is empty (on macOS, window titles need the Screen Recording permission)'
}
return `its windows are ${windows.map((window) => `"${window.title}"`).join(', ') || 'none'}`
}
function matchingWindows(
windows: ComputerWindowListWindow[],
title: string
): ComputerWindowListWindow[] {
const needle = title.toLowerCase()
return windows.filter((window) => window.title.toLowerCase().includes(needle))
}
async function assertDisabledStateSupported(
client: RuntimeClient,
readTimeoutMs: number
): Promise<void> {
const { result } = await client.call<ComputerProviderCapabilities>(
'computer.capabilities',
{},
{ timeoutMs: readTimeoutMs }
)
if (result.supports.observation.disabledState !== true) {
// Why: without the trait every element reads as enabled, so the wait would pass at once.
throw new RuntimeClientError(
'unsupported_capability',
`The ${result.provider} computer-use provider on this host does not report disabled elements, so --enabled cannot be checked. Update Orca on that host, or wait with --text instead.`
)
}
}
async function pollUntil<T>(
condition: ComputerWaitCondition,
startedAt: number,
timeoutMs: number,
poll: Poll<T>
): Promise<{ response: RuntimeRpcSuccess<T>; elapsedMs: number }> {
for (;;) {
let miss: string
try {
const remainingMs = timeoutMs - (Date.now() - startedAt)
const response = await poll.call(Math.max(remainingMs, MIN_READ_TIMEOUT_MS))
const result = poll.check(response.result)
if (result === null) {
return { response, elapsedMs: Date.now() - startedAt }
}
miss = result
} catch (error) {
const code = cliErrorCode(error)
if (code === 'runtime_timeout' && Date.now() - startedAt >= timeoutMs) {
miss = 'the last read did not answer before the deadline'
} else if (code && NOT_YET_CODES.has(code)) {
miss = `${code}: ${error instanceof Error ? error.message : String(error)}`
} else {
throw error
}
}
const remainingMs = timeoutMs - (Date.now() - startedAt)
if (remainingMs <= 0) {
throw waitTimeoutError(condition, timeoutMs, miss)
}
await new Promise((resolve) => setTimeout(resolve, Math.min(POLL_INTERVAL_MS, remainingMs)))
}
}
function waitTimeoutError(
condition: ComputerWaitCondition,
timeoutMs: number,
lastSeen: string
): RuntimeClientError {
const goal =
condition.kind === 'text'
? `an element containing "${condition.value}"`
: condition.kind === 'enabled'
? `elements containing "${condition.value}" to be enabled`
: `a window titled "${condition.value}"`
return new RuntimeClientError(
'wait_timeout',
`Timed out after ${timeoutMs} ms waiting for ${goal}; last check: ${lastSeen}.`,
{
nextSteps: [
'Run `orca computer get-app-state --app <app> --json` to see what the app shows now.',
'If the app is still working, wait again with a longer --timeout; otherwise match text the tree actually shows.',
...(condition.kind === 'enabled'
? ['Every matching element must be enabled, so use text only the intended control shows.']
: [])
]
}
)
}
export function formatComputerWaitSnapshot(result: ComputerWaitSnapshotResult): string {
return `${waitHeadline(result.wait)}\n${formatGetAppState(result)}`
}
export function formatComputerWaitWindows(result: ComputerWaitWindowsResult): string {
return `${waitHeadline(result.wait)}\n${formatListWindows(result)}`
}
function waitHeadline(wait: ComputerWaitSummary): string {
return `Condition --${wait.kind} "${wait.value}" held after ${wait.elapsedMs} ms.`
}
+2
View File
@@ -12,6 +12,7 @@ import {
getRequiredStringFlag
} from '../arguments/flags'
import { queryComputerTreeText } from './computer-tree-query'
import { runComputerWait } from './computer-wait-command'
import { RuntimeClientError } from '../runtime/types'
import {
formatComputerAction,
@@ -95,6 +96,7 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
}
printResult(result, json, formatGetAppState)
},
'computer wait': runComputerWait,
'computer click': async ({ flags, client, cwd, json }) => {
assertComputerAppFlag(flags)
const observeFlags = getComputerActionObserveFlags(flags)