import type { ComputerActionResult, ComputerLaunchAppResult, ComputerListAppsResult, ComputerListWindowsResult, ComputerProviderCapabilities, ComputerSnapshotResult } from '../../shared/computer-use-runtime-types' import type { CommandHandler } from '../dispatch/dispatch' import { getOptionalNonNegativeIntegerFlag, getOptionalPositiveIntegerFlag, getOptionalStringFlag, getRequiredStringFlag } from '../arguments/flags' import { queryComputerTreeText } from './computer-tree-query' import { runComputerWait } from './computer-wait-command' import { RuntimeClientError } from '../runtime/connection/types' import { formatComputerAction, formatGetAppState, formatLaunchApp, formatListApps, formatListWindows } from './computer-format' import { printResult } from '../output/command-result-output' import { getComputerCommandTarget } from './computer-command-target' import { getComputerActionObserveFlags, getComputerObserveFlags, getComputerClickActionFlags, getComputerDragActionFlags, getComputerHotkeyActionFlags, getComputerKeyActionFlags, getComputerScrollActionFlags, getComputerSecondaryActionFlags, getComputerSetValueActionFlags, getComputerTextActionFlags, getComputerTypeTextActionFlags } from './computer-action-flags' import { computerTypeTextTargetUnsupportedMessage } from '../../shared/computer-use-type-text-target' export const COMPUTER_HANDLERS: Record = { 'computer capabilities': async ({ client, json }) => { const result = await client.call('computer.capabilities', {}) printResult(result, json, formatComputerCapabilities) }, 'computer list-apps': async ({ client, json }) => { const result = await client.call('computer.listApps', {}) printResult(result, json, formatListApps) }, 'computer permissions': async ({ flags, client, json }) => { const id = getComputerPermissionSetupId(flags) // Agents run this on errors; System Settings over the user's work is only worth it when something is missing. const result = await client.call<{ platform: NodeJS.Platform helperAppPath: string | null openedSettings: boolean launchedHelper: boolean launchQueued?: boolean permissions?: { id: string; status: string }[] nextStep?: string | null }>('computer.permissions', id ? { id, skipGranted: true } : {}) printResult(result, json, (value) => { if (value.platform !== 'darwin') { return 'Computer-use permission setup is only required on macOS.' } const firstLine = permissionSetupHeadline(value) return [ firstLine, `Helper app: ${value.helperAppPath}`, `Permissions: ${value.permissions?.map((permission) => `${permission.id}=${permission.status}`).join(', ') ?? 'unknown'}`, value.nextStep ? `Next: ${value.nextStep}` : 'Computer Use permissions are already granted.', value.launchedHelper ? 'Use the Allow buttons or drag "Orca Computer Use" into the macOS permission list.' : null ] .filter((line) => line !== null) .join('\n') }) }, 'computer launch-app': async ({ flags, client, json }) => { const result = await client.call('computer.launchApp', { app: getRequiredStringFlag(flags, 'app'), ...(flags.get('activate') === true ? { activate: true } : {}) }) printResult(result, json, formatLaunchApp) }, 'computer list-windows': async ({ flags, client, json }) => { const result = await client.call('computer.listWindows', { app: getRequiredStringFlag(flags, 'app') }) printResult(result, json, formatListWindows) }, 'computer get-app-state': async ({ flags, client, cwd, json }) => { const observeFlags = getComputerObserveFlags(flags) const query = getOptionalStringFlag(flags, 'query') const maxElements = getOptionalPositiveIntegerFlag(flags, 'max-elements') const rootElementIndex = getOptionalNonNegativeIntegerFlag(flags, 'element-index') const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.getAppState', { ...target, ...observeFlags, ...(rootElementIndex === undefined ? {} : { rootElementIndex }) }) // Why: older runtimes drop the param and return the whole window, which would look like the subtree. if ( rootElementIndex !== undefined && result.result.snapshot.rootElementIndex !== rootElementIndex ) { throw new RuntimeClientError( 'unsupported_capability', 'This Orca runtime cannot read one element subtree; update Orca, or run get-app-state without --element-index' ) } if (query !== undefined || maxElements !== undefined) { const { snapshot } = result.result snapshot.treeText = queryComputerTreeText(snapshot.treeText, { query, maxElements }) } printResult(result, json, formatGetAppState) }, 'computer wait': runComputerWait, 'computer click': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = getComputerClickActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.click', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('click', value, { ...target, ...observeFlags }) ) }, 'computer perform-secondary-action': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = getComputerSecondaryActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.performSecondaryAction', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('perform-secondary-action', value, { ...target, ...observeFlags }) ) }, 'computer scroll': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = getComputerScrollActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.scroll', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('scroll', value, { ...target, ...observeFlags }) ) }, 'computer drag': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = getComputerDragActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.drag', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('drag', value, { ...target, ...observeFlags }) ) }, 'computer type-text': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = await getComputerTypeTextActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) if (actionParams.elementIndex !== undefined) { // Why: older runtimes strip elementIndex, so ask first rather than type into the focused field. const capabilities = await client.call( 'computer.capabilities', {} ) const unsupported = computerTypeTextTargetUnsupportedMessage(capabilities.result) if (unsupported) { throw new RuntimeClientError('unsupported_capability', unsupported) } } const result = await client.call('computer.typeText', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('type-text', value, { ...target, ...observeFlags }) ) }, 'computer press-key': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = getComputerKeyActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.pressKey', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('press-key', value, { ...target, ...observeFlags }) ) }, 'computer hotkey': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = getComputerHotkeyActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.hotkey', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('hotkey', value, { ...target, ...observeFlags }) ) }, 'computer paste-text': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = await getComputerTextActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.pasteText', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('paste-text', value, { ...target, ...observeFlags }) ) }, 'computer set-value': async ({ flags, client, cwd, json }) => { assertComputerAppFlag(flags) const observeFlags = getComputerActionObserveFlags(flags) const actionParams = await getComputerSetValueActionFlags(flags) const target = await getComputerCommandTarget(flags, cwd, client) const result = await client.call('computer.setValue', { ...target, ...actionParams, ...observeFlags }) printResult(result, json, (value) => formatComputerAction('set-value', value, { ...target, ...observeFlags }) ) } } function assertComputerAppFlag(flags: Map): void { getRequiredStringFlag(flags, 'app') } function getComputerPermissionSetupId( flags: Map ): 'accessibility' | 'screenshots' | undefined { const id = getOptionalStringFlag(flags, 'id') if (id === undefined || id === 'accessibility' || id === 'screenshots') { return id } throw new RuntimeClientError('invalid_argument', '--id must be "accessibility" or "screenshots"') } function formatComputerCapabilities(value: ComputerProviderCapabilities): string { return [ `${value.provider} (${value.platform}, protocol ${value.protocolVersion})`, ` Apps: list=${value.supports.apps.list} bundleIds=${value.supports.apps.bundleIds} pids=${value.supports.apps.pids} launch=${value.supports.apps.launch === true}`, ` Windows: list=${value.supports.windows.list} targetById=${value.supports.windows.targetById} targetByIndex=${value.supports.windows.targetByIndex}`, ` Observation: screenshot=${value.supports.observation.screenshot} elementFrames=${value.supports.observation.elementFrames} annotatedScreenshot=${value.supports.observation.annotatedScreenshot}`, ` Actions: ${Object.entries(value.supports.actions) .filter(([, enabled]) => enabled) .map(([name]) => name) .join(', ')}`, ...(value.supports.input ? [` Input: background=${value.supports.input.background}`] : []) ].join('\n') } function permissionSetupHeadline(result: { launchedHelper: boolean launchQueued?: boolean }): string { if (result.launchQueued) { return 'Queued Orca Computer Use permission setup; a newer request or a reset may replace it.' } return result.launchedHelper ? 'Opened Orca Computer Use permission setup.' : 'Computer Use permissions checked.' }