diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml
new file mode 100644
index 00000000000..6d68765f903
--- /dev/null
+++ b/.github/workflows/computer-e2e.yml
@@ -0,0 +1,104 @@
+name: Computer-use e2e
+
+on:
+ pull_request:
+ paths:
+ - '.github/workflows/computer-e2e.yml'
+ - 'config/electron-builder.config.cjs'
+ - 'config/scripts/build-computer-macos.mjs'
+ - 'config/scripts/verify-computer-native.mjs'
+ - 'native/computer-use-macos/**'
+ - 'native/computer-use-linux/**'
+ - 'native/computer-use-windows/**'
+ - 'src/cli/**'
+ - 'src/main/computer/**'
+ - 'src/main/runtime/rpc/methods/computer.ts'
+ - 'src/shared/runtime-types.ts'
+ - 'tests/e2e/computer-linux.e2e.ts'
+ - 'tests/e2e/computer-mac.e2e.ts'
+ - 'tests/e2e/computer-windows.e2e.ts'
+ - 'tests/e2e/helpers/computer-driver.ts'
+ workflow_dispatch:
+ schedule:
+ - cron: '0 6 * * *'
+
+jobs:
+ native-smoke:
+ if: github.event_name == 'pull_request'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-22.04, windows-latest]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version-file: package.json
+ - uses: pnpm/action-setup@v4
+ with:
+ run_install: false
+ - if: runner.os == 'Linux'
+ run: sudo apt-get update && sudo apt-get install -y python3 python3-gi gir1.2-atspi-2.0 at-spi2-core xclip xdotool
+ - run: pnpm install --frozen-lockfile
+ - run: pnpm verify:computer-native
+ - run: pnpm build:cli
+
+ mac:
+ # macOS Accessibility and Screen Recording require user-granted TCC entries.
+ # Keep this on manual/scheduled permission-bearing runners instead of PR CI.
+ if: github.event_name != 'pull_request'
+ runs-on: macos-14
+ env:
+ ORCA_COMPUTER_E2E: '1'
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version-file: package.json
+ - uses: pnpm/action-setup@v4
+ with:
+ run_install: false
+ - run: pnpm install --frozen-lockfile
+ - run: pnpm build:computer-macos
+ - run: pnpm verify:computer-native
+ - run: pnpm build:cli
+ - run: pnpm test:e2e:computer -- --reporter=verbose tests/e2e/computer-mac.e2e.ts
+
+ linux:
+ if: github.event_name != 'pull_request'
+ runs-on: ubuntu-22.04
+ env:
+ ORCA_COMPUTER_E2E: '1'
+ ACCESSIBILITY_ENABLED: '1'
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version-file: package.json
+ - uses: pnpm/action-setup@v4
+ with:
+ run_install: false
+ - run: sudo apt-get update && sudo apt-get install -y build-essential python3 python3-gi gir1.2-atspi-2.0 gedit at-spi2-core xvfb xclip xdotool
+ - run: pnpm install --frozen-lockfile
+ - run: pnpm verify:computer-native
+ - run: pnpm build:cli
+ - run: xvfb-run --auto-servernum dbus-run-session -- pnpm test:e2e:computer -- --reporter=verbose tests/e2e/computer-linux.e2e.ts
+
+ windows:
+ if: github.event_name != 'pull_request'
+ runs-on: windows-latest
+ env:
+ ORCA_COMPUTER_E2E: '1'
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version-file: package.json
+ - uses: pnpm/action-setup@v4
+ with:
+ run_install: false
+ - run: pnpm install --frozen-lockfile
+ - run: pnpm verify:computer-native
+ - run: pnpm build:cli
+ - run: pnpm test:e2e:computer -- --reporter=verbose tests/e2e/computer-windows.e2e.ts
diff --git a/.gitignore b/.gitignore
index 58486129a44..6bc6a97afeb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ dist-electron/
out/
/build/
release/
+native/**/.build/
# pnpm
.pnpm-store/
diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs
index aae41fe4025..62a044e808e 100644
--- a/config/electron-builder.config.cjs
+++ b/config/electron-builder.config.cjs
@@ -1,5 +1,6 @@
const { chmodSync, existsSync, readdirSync } = require('node:fs')
-const { join } = require('node:path')
+const { execFileSync } = require('node:child_process')
+const { join, resolve } = require('node:path')
const isMacRelease = process.env.ORCA_MAC_RELEASE === '1'
@@ -35,6 +36,7 @@ module.exports = {
'out/cli/**',
'out/shared/**',
'out/main/daemon-entry.js',
+ 'out/main/computer-sidecar.js',
'out/main/chunks/**',
'resources/**',
'node_modules/zod/**'
@@ -42,7 +44,12 @@ module.exports = {
afterPack: async (context) => {
const resourcesDir =
context.electronPlatformName === 'darwin'
- ? join(context.appOutDir, `${context.packager.appInfo.productFilename}.app`, 'Contents', 'Resources')
+ ? join(
+ context.appOutDir,
+ `${context.packager.appInfo.productFilename}.app`,
+ 'Contents',
+ 'Resources'
+ )
: join(context.appOutDir, 'resources')
if (!existsSync(resourcesDir)) {
return
@@ -56,6 +63,9 @@ module.exports = {
// the copied binary to be executable in packaged apps.
chmodSync(join(resourcesDir, filename), 0o755)
}
+ if (context.electronPlatformName === 'darwin') {
+ await signMacComputerUseHelper(join(resourcesDir, 'Orca Computer Use.app'), context.packager)
+ }
},
win: {
executableName: 'Orca',
@@ -67,6 +77,10 @@ module.exports = {
{
from: 'node_modules/agent-browser/bin/agent-browser-win32-x64.exe',
to: 'agent-browser-win32-x64.exe'
+ },
+ {
+ from: 'native/computer-use-windows/runtime.ps1',
+ to: 'computer-use-windows/runtime.ps1'
}
]
},
@@ -115,6 +129,10 @@ module.exports = {
{
from: 'node_modules/agent-browser/bin/agent-browser-darwin-${arch}',
to: 'agent-browser-darwin-${arch}'
+ },
+ {
+ from: 'native/computer-use-macos/.build/release/Orca Computer Use.app',
+ to: 'Orca Computer Use.app'
}
],
target: [
@@ -134,6 +152,9 @@ module.exports = {
dmg: {
artifactName: 'orca-macos-${arch}.${ext}'
},
+ deb: {
+ depends: ['python3', 'python3-gi', 'gir1.2-atspi-2.0', 'at-spi2-core', 'xdotool', 'xclip']
+ },
linux: {
extraResources: [
{
@@ -143,6 +164,10 @@ module.exports = {
{
from: 'node_modules/agent-browser/bin/agent-browser-linux-${arch}',
to: 'agent-browser-linux-${arch}'
+ },
+ {
+ from: 'native/computer-use-linux/runtime.py',
+ to: 'computer-use-linux/runtime.py'
}
],
target: ['AppImage', 'deb'],
@@ -165,3 +190,67 @@ module.exports = {
releaseType: 'release'
}
}
+
+async function signMacComputerUseHelper(helperAppPath, packager) {
+ if (!existsSync(helperAppPath)) {
+ if (isMacRelease) {
+ throw new Error(`Missing Orca Computer Use helper app at ${helperAppPath}`)
+ }
+ return
+ }
+ const codeSigningInfo =
+ isMacRelease && process.env.CSC_LINK && packager?.codeSigningInfo?.value
+ ? await packager.codeSigningInfo.value
+ : null
+ const identity =
+ process.env.ORCA_COMPUTER_MACOS_SIGN_IDENTITY ??
+ process.env.CSC_NAME ??
+ findInstalledMacSigningIdentity(codeSigningInfo?.keychainFile) ??
+ (isMacRelease ? null : '-')
+ if (!identity) {
+ throw new Error('Missing signing identity for Orca Computer Use helper app')
+ }
+ // Why: TCC grants attach to this nested app's code identity. Sign it before
+ // the outer Orca.app is sealed so production builds preserve that identity.
+ execFileSync('codesign', codesignArgs(identity, helperAppPath), { stdio: 'inherit' })
+ execFileSync('codesign', ['--verify', '--deep', '--strict', helperAppPath], {
+ stdio: 'inherit'
+ })
+}
+
+function codesignArgs(identity, targetPath) {
+ const args = ['--force', '--deep', '--sign', identity]
+ if (isMacRelease) {
+ args.push(
+ '--options',
+ 'runtime',
+ '--timestamp',
+ '--entitlements',
+ resolve(__dirname, '../resources/build/entitlements.computer-use.mac.plist')
+ )
+ }
+ args.push(targetPath)
+ return args
+}
+
+function findInstalledMacSigningIdentity(keychainFile) {
+ try {
+ const output = execFileSync(
+ 'security',
+ ['find-identity', '-v', '-p', 'codesigning', ...(keychainFile ? [keychainFile] : [])],
+ {
+ encoding: 'utf8'
+ }
+ )
+ const releaseMatch =
+ output.match(/"([^"]*Developer ID Application:[^"]+)"/) ??
+ output.match(/"([^"]*Apple Distribution:[^"]+)"/)
+ if (releaseMatch?.[1]) {
+ return releaseMatch[1]
+ }
+ if (!isMacRelease) {
+ return output.match(/"([^"]*Apple Development:[^"]+)"/)?.[1] ?? null
+ }
+ } catch {}
+ return null
+}
diff --git a/config/scripts/build-computer-macos.mjs b/config/scripts/build-computer-macos.mjs
new file mode 100644
index 00000000000..6ef3a959ee8
--- /dev/null
+++ b/config/scripts/build-computer-macos.mjs
@@ -0,0 +1,142 @@
+import { spawnSync } from 'node:child_process'
+import { chmodSync, copyFileSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
+const packagePath = path.join(repoRoot, 'native', 'computer-use-macos')
+const binaryPath = path.join(packagePath, '.build', 'release', 'orca-computer-use-macos')
+const appPath = path.join(packagePath, '.build', 'release', 'Orca Computer Use.app')
+const appExecutablePath = path.join(appPath, 'Contents', 'MacOS', 'orca-computer-use-macos')
+const appIconPath = path.join(appPath, 'Contents', 'Resources', 'AppIcon.icns')
+const entitlementsPath = path.join(
+ repoRoot,
+ 'resources',
+ 'build',
+ 'entitlements.computer-use.mac.plist'
+)
+const bundleId = process.env.ORCA_COMPUTER_MACOS_BUNDLE_ID ?? 'com.stablyai.orca.computer-use'
+const displayName = 'Orca Computer Use'
+const signingIdentity = resolveSigningIdentity()
+const universalTriples = ['arm64-apple-macosx', 'x86_64-apple-macosx']
+
+if (process.platform !== 'darwin') {
+ process.exit(0)
+}
+
+buildUniversalBinary()
+chmodSync(binaryPath, 0o755)
+createHelperApp()
+
+function buildUniversalBinary() {
+ const builtBinaries = universalTriples.map((triple) => {
+ run('swift', ['build', '-c', 'release', '--package-path', packagePath, '--triple', triple])
+ return path.join(packagePath, '.build', triple, 'release', 'orca-computer-use-macos')
+ })
+ mkdirSync(path.dirname(binaryPath), { recursive: true })
+ run('lipo', ['-create', ...builtBinaries, '-output', binaryPath])
+}
+
+function createHelperApp() {
+ rmSync(appPath, { recursive: true, force: true })
+ mkdirSync(path.dirname(appExecutablePath), { recursive: true })
+ mkdirSync(path.join(appPath, 'Contents', 'Resources'), { recursive: true })
+ copyFileSync(binaryPath, appExecutablePath)
+ copyFileSync(path.join(repoRoot, 'resources', 'build', 'icon.icns'), appIconPath)
+ chmodSync(appExecutablePath, 0o755)
+ writeFileSync(path.join(appPath, 'Contents', 'Info.plist'), infoPlist(), 'utf8')
+ const signer = spawnSync('codesign', codesignArgs(signingIdentity, appPath), { stdio: 'inherit' })
+ if (signer.signal) {
+ process.kill(process.pid, signer.signal)
+ }
+ if (signer.status !== 0) {
+ process.exit(signer.status ?? 1)
+ }
+}
+
+function codesignArgs(identity, targetPath) {
+ const args = ['--force', '--deep', '--sign', identity]
+ if (process.env.ORCA_MAC_RELEASE === '1' && identity !== '-') {
+ args.push('--options', 'runtime', '--timestamp', '--entitlements', entitlementsPath)
+ }
+ args.push(targetPath)
+ return args
+}
+
+function resolveSigningIdentity() {
+ const explicitIdentity = process.env.ORCA_COMPUTER_MACOS_SIGN_IDENTITY ?? process.env.CSC_NAME
+ if (explicitIdentity) {
+ return explicitIdentity
+ }
+ const identities = spawnSync('security', ['find-identity', '-v', '-p', 'codesigning'], {
+ encoding: 'utf8'
+ })
+ if (identities.status !== 0 || !identities.stdout) {
+ return '-'
+ }
+ const developmentMatch = identities.stdout.match(/"([^"]*Apple Development:[^"]+)"/)
+ if (process.env.ORCA_MAC_RELEASE !== '1' && developmentMatch) {
+ return developmentMatch[1]
+ }
+ const releaseMatch =
+ identities.stdout.match(/"([^"]*Developer ID Application:[^"]+)"/) ??
+ identities.stdout.match(/"([^"]*Apple Distribution:[^"]+)"/)
+ return releaseMatch?.[1] ?? developmentMatch?.[1] ?? '-'
+}
+
+function run(command, args) {
+ const result = spawnSync(command, args, { stdio: 'inherit' })
+ if (result.signal) {
+ process.kill(process.pid, result.signal)
+ }
+ if (result.status !== 0) {
+ process.exit(result.status ?? 1)
+ }
+}
+
+function infoPlist() {
+ return `
+
+
+
+ CFBundleDevelopmentRegion
+ en
+ CFBundleExecutable
+ orca-computer-use-macos
+ CFBundleIdentifier
+ ${escapePlist(bundleId)}
+ CFBundleInfoDictionaryVersion
+ 6.0
+ CFBundleIconFile
+ AppIcon
+ CFBundleName
+ ${escapePlist(displayName)}
+ CFBundleDisplayName
+ ${escapePlist(displayName)}
+ CFBundlePackageType
+ APPL
+ CFBundleShortVersionString
+ 1.0
+ CFBundleVersion
+ 1
+ LSMinimumSystemVersion
+ 14.0
+ LSUIElement
+
+ NSAccessibilityUsageDescription
+ Orca Computer Use needs Accessibility permission to read and interact with app interfaces when you ask Orca to use apps.
+ NSScreenCaptureUsageDescription
+ Orca Computer Use needs Screen Recording permission to capture app windows when you ask Orca to inspect your screen.
+
+
+`
+}
+
+function escapePlist(value) {
+ return value
+ .replaceAll('&', '&')
+ .replaceAll('<', '<')
+ .replaceAll('>', '>')
+ .replaceAll('"', '"')
+ .replaceAll("'", ''')
+}
diff --git a/config/scripts/computer-use-smoke.mjs b/config/scripts/computer-use-smoke.mjs
new file mode 100644
index 00000000000..1d5290a7ec0
--- /dev/null
+++ b/config/scripts/computer-use-smoke.mjs
@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { resolve } from 'node:path'
+import { spawnSync } from 'node:child_process'
+
+const repoRoot = resolve(import.meta.dirname, '..', '..')
+const cliPath = resolve(repoRoot, 'out', 'cli', 'index.js')
+const args = new Set(process.argv.slice(2))
+const requestedApps = valueFlag('--apps')
+const includeScreenshot = args.has('--screenshot')
+const session = valueFlag('--session') ?? `computer-smoke-${process.pid}`
+const preferredApps = (requestedApps ?? process.env.ORCA_COMPUTER_SMOKE_APPS ?? 'Finder,TextEdit,Spotify,Slack,Microsoft Edge')
+ .split(',')
+ .map((app) => app.trim())
+ .filter(Boolean)
+
+if (!existsSync(cliPath)) {
+ fail(`Missing built CLI at ${cliPath}. Run pnpm build:cli first.`)
+}
+
+const list = unwrapResult(runCli(['computer', 'list-apps', '--json']))
+const apps = Array.isArray(list.apps) ? list.apps : []
+const availableNames = new Set(apps.map((app) => String(app.name ?? '').toLowerCase()))
+const availableBundles = new Set(apps.map((app) => String(app.bundleId ?? '').toLowerCase()).filter(Boolean))
+const targets = preferredApps.filter((app) => availableNames.has(app.toLowerCase()) || availableBundles.has(app.toLowerCase()))
+
+console.log(`computer-use smoke: ${apps.length} apps listed`)
+if (targets.length === 0) {
+ console.log(`computer-use smoke: no preferred apps are running (${preferredApps.join(', ')})`)
+ process.exit(0)
+}
+
+let failures = 0
+for (const app of targets) {
+ const result = runCli([
+ 'computer',
+ 'get-app-state',
+ '--session',
+ session,
+ '--app',
+ app,
+ ...(includeScreenshot ? [] : ['--no-screenshot']),
+ '--json'
+ ], { allowFailure: true })
+
+ if (!result.ok) {
+ failures += 1
+ console.log(`computer-use smoke: ${app}: failed: ${result.error}`)
+ continue
+ }
+
+ const state = unwrapResult(result.value)
+ const snapshot = state.snapshot
+ const treeText = String(snapshot.treeText ?? '')
+ const lineCount = treeText.split('\n').filter(Boolean).length
+ const secondaryActions = (treeText.match(/Secondary Actions:/g) ?? []).length
+ const settable = (treeText.match(/\bsettable\b/g) ?? []).length
+ const screenshotState = state.screenshot ? `${state.screenshot.width}x${state.screenshot.height}` : 'missing'
+ console.log([
+ `computer-use smoke: ${snapshot.app.name}`,
+ `${snapshot.elementCount} elements`,
+ `${lineCount} lines`,
+ `${secondaryActions} secondary-action lines`,
+ `${settable} settable elements`,
+ `screenshot=${screenshotState}`
+ ].join(' | '))
+}
+
+if (failures > 0) {
+ fail(`${failures} app snapshot smoke check(s) failed`)
+}
+
+function valueFlag(name) {
+ const index = process.argv.indexOf(name)
+ if (index === -1) {
+ return null
+ }
+ return process.argv[index + 1] ?? null
+}
+
+function runCli(cliArgs, options = {}) {
+ const child = spawnSync(process.execPath, [cliPath, ...cliArgs], {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ ORCA_USER_DATA_PATH: process.env.ORCA_COMPUTER_SMOKE_USER_DATA_PATH ?? defaultDevUserDataPath()
+ }
+ })
+ if (child.status !== 0) {
+ const error = (child.stderr || child.stdout || `exit ${child.status}`).trim()
+ if (options.allowFailure) {
+ return { ok: false, error }
+ }
+ fail(error)
+ }
+ try {
+ return options.allowFailure
+ ? { ok: true, value: JSON.parse(child.stdout) }
+ : JSON.parse(child.stdout)
+ } catch (error) {
+ const detail = error instanceof Error ? error.message : String(error)
+ fail(`Could not parse CLI JSON for ${cliArgs.join(' ')}: ${detail}\n${child.stdout}`)
+ }
+}
+
+function defaultDevUserDataPath() {
+ if (process.platform === 'darwin') {
+ return resolve(homedir(), 'Library', 'Application Support', 'orca-dev')
+ }
+ if (process.platform === 'win32') {
+ return resolve(process.env.APPDATA ?? resolve(homedir(), 'AppData', 'Roaming'), 'orca-dev')
+ }
+ return resolve(process.env.XDG_CONFIG_HOME ?? resolve(homedir(), '.config'), 'orca-dev')
+}
+
+function unwrapResult(value) {
+ if (value && typeof value === 'object' && 'result' in value) {
+ return value.result
+ }
+ return value
+}
+
+function fail(message) {
+ console.error(`computer-use smoke: ${message}`)
+ process.exit(1)
+}
diff --git a/config/scripts/verify-computer-native.mjs b/config/scripts/verify-computer-native.mjs
new file mode 100644
index 00000000000..4249a893a2f
--- /dev/null
+++ b/config/scripts/verify-computer-native.mjs
@@ -0,0 +1,180 @@
+#!/usr/bin/env node
+
+import { spawnSync } from 'node:child_process'
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+const repoRoot = resolve(import.meta.dirname, '..', '..')
+const checks = [
+ {
+ name: 'macOS Swift renderer/provider tests',
+ command: 'swift',
+ args: ['test', '--package-path', 'native/computer-use-macos'],
+ enabled: process.platform === 'darwin'
+ },
+ {
+ name: 'Linux provider Python syntax',
+ command: 'python3',
+ args: ['-m', 'py_compile', 'native/computer-use-linux/runtime.py'],
+ enabled: true
+ },
+ {
+ name: 'Linux provider imports',
+ command: 'python3',
+ args: [
+ '-c',
+ [
+ 'import importlib.util',
+ 'spec=importlib.util.spec_from_file_location("orca_linux","native/computer-use-linux/runtime.py")',
+ 'module=importlib.util.module_from_spec(spec)',
+ 'spec.loader.exec_module(module)',
+ 'print("import-ok")'
+ ].join(';')
+ ],
+ enabled: process.platform === 'linux'
+ },
+ {
+ name: 'Windows provider PowerShell parse',
+ command: process.platform === 'win32' ? 'powershell.exe' : 'pwsh',
+ args: [
+ '-NoLogo',
+ '-NoProfile',
+ '-Command',
+ [
+ '$errors=$null',
+ '$tokens=$null',
+ '[System.Management.Automation.Language.Parser]::ParseFile("native/computer-use-windows/runtime.ps1",[ref]$tokens,[ref]$errors) > $null',
+ 'if ($errors.Count) { $errors | Format-List *; exit 1 }',
+ '"parse-ok"'
+ ].join('; ')
+ ],
+ enabled: true
+ },
+ {
+ name: 'Windows provider handshake',
+ run: verifyWindowsProviderHandshake,
+ enabled: process.platform === 'win32'
+ },
+ {
+ name: 'macOS helper app bundle and signature',
+ run: verifyMacOSHelperApp,
+ enabled: process.platform === 'darwin'
+ }
+]
+
+let failed = false
+for (const check of checks) {
+ if (!check.enabled) {
+ console.log(`[computer-native] skip ${check.name}`)
+ continue
+ }
+ if (check.run) {
+ console.log(`[computer-native] ${check.name}`)
+ if (!check.run()) {
+ failed = true
+ }
+ continue
+ }
+ if (!hasCommand(check.command)) {
+ console.log(`[computer-native] skip ${check.name}: ${check.command} not found`)
+ continue
+ }
+ console.log(`[computer-native] ${check.name}`)
+ const result = spawnSync(check.command, check.args, {
+ cwd: repoRoot,
+ stdio: 'inherit'
+ })
+ if (result.status !== 0 || result.error) {
+ failed = true
+ }
+}
+
+if (failed) {
+ process.exit(1)
+}
+
+function hasCommand(command) {
+ if (process.platform === 'win32') {
+ const result = spawnSync('where.exe', [command], { stdio: 'ignore' })
+ return result.status === 0
+ }
+ if (existsSync(command)) {
+ return true
+ }
+ const result = spawnSync('/bin/sh', ['-lc', `command -v ${quoteShell(command)}`], {
+ stdio: 'ignore'
+ })
+ return result.status === 0
+}
+
+function verifyMacOSHelperApp() {
+ const appPath = join(
+ repoRoot,
+ 'native',
+ 'computer-use-macos',
+ '.build',
+ 'release',
+ 'Orca Computer Use.app'
+ )
+ if (!existsSync(appPath)) {
+ console.error(
+ `[computer-native] missing helper app at ${appPath}; run pnpm build:computer-macos`
+ )
+ return false
+ }
+ return run('codesign', ['--verify', '--deep', '--strict', appPath])
+}
+
+function verifyWindowsProviderHandshake() {
+ const dir = mkdtempSync(join(tmpdir(), 'orca-computer-use-verify-'))
+ const operationPath = join(dir, 'operation.json')
+ try {
+ writeFileSync(operationPath, JSON.stringify({ tool: 'handshake' }), { mode: 0o600 })
+ const result = spawnSync(
+ 'powershell.exe',
+ [
+ '-NoLogo',
+ '-NoProfile',
+ '-ExecutionPolicy',
+ 'Bypass',
+ '-File',
+ 'native/computer-use-windows/runtime.ps1',
+ operationPath
+ ],
+ { cwd: repoRoot, encoding: 'utf8' }
+ )
+ if (result.status !== 0 || result.error) {
+ process.stderr.write(result.stderr ?? '')
+ return false
+ }
+ const response = JSON.parse(result.stdout.trim())
+ if (response.ok === true && response.capabilities?.protocolVersion === 1) {
+ console.log('[computer-native] windows-handshake-ok')
+ return true
+ }
+ console.error(`[computer-native] invalid Windows handshake response: ${result.stdout}`)
+ return false
+ } catch (error) {
+ console.error(error instanceof Error ? error.message : String(error))
+ return false
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+}
+
+function run(command, args) {
+ if (!hasCommand(command)) {
+ console.log(`[computer-native] skip ${command}: command not found`)
+ return true
+ }
+ const result = spawnSync(command, args, {
+ cwd: repoRoot,
+ stdio: 'inherit'
+ })
+ return result.status === 0 && !result.error
+}
+
+function quoteShell(value) {
+ return `'${value.replaceAll("'", "'\\''")}'`
+}
diff --git a/electron.vite.config.ts b/electron.vite.config.ts
index 611999e8457..63bda2501b9 100644
--- a/electron.vite.config.ts
+++ b/electron.vite.config.ts
@@ -41,7 +41,8 @@ export default defineConfig({
rollupOptions: {
input: {
index: resolve('src/main/index.ts'),
- 'daemon-entry': resolve('src/main/daemon/daemon-entry.ts')
+ 'daemon-entry': resolve('src/main/daemon/daemon-entry.ts'),
+ 'computer-sidecar': resolve('src/main/computer/sidecar-entry.ts')
}
}
},
diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py
new file mode 100644
index 00000000000..ac6495ac50e
--- /dev/null
+++ b/native/computer-use-linux/runtime.py
@@ -0,0 +1,819 @@
+#!/usr/bin/env python3
+"""Orca Linux computer-use bridge.
+
+The Node sidecar owns Orca's public API. This process is intentionally a small
+AT-SPI adapter: read one JSON operation file, execute it in the user's desktop
+session, and print one JSON response.
+"""
+
+import base64
+import json
+import math
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+
+import gi
+
+gi.require_version("Atspi", "2.0")
+try:
+ gi.require_version("Gdk", "3.0")
+ from gi.repository import Gdk
+except (ImportError, ValueError):
+ Gdk = None
+from gi.repository import Atspi
+
+MAX_NODES = 1200
+MAX_DEPTH = 64
+TEXT_LIMIT = 500
+BLOCKED_APP_FRAGMENTS = (
+ "1password",
+ "bitwarden",
+ "dashlane",
+ "lastpass",
+ "nordpass",
+ "proton pass",
+)
+
+
+@dataclass
+class Rect:
+ x: float
+ y: float
+ width: float
+ height: float
+
+ def to_json(self):
+ return {"x": self.x, "y": self.y, "width": self.width, "height": self.height}
+
+
+def attempt(fn, fallback=None):
+ try:
+ value = fn()
+ return fallback if value is None else value
+ except Exception:
+ return fallback
+
+
+def ensure_desktop_bus():
+ missing = [name for name in ("XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS") if not os.environ.get(name)]
+ if missing:
+ raise RuntimeError("Linux computer use requires an active desktop session; missing " + ", ".join(missing))
+
+
+def desktop_root():
+ return Atspi.get_desktop(0)
+
+
+def children(node):
+ count = int(attempt(node.get_child_count, 0) or 0)
+ for index in range(count):
+ child = attempt(lambda i=index: node.get_child_at_index(i))
+ if child is not None:
+ yield index, child
+
+
+def text_attr(node, getter):
+ return str(attempt(getter, "") or "")
+
+
+def name_of(node):
+ return text_attr(node, node.get_name)
+
+
+def role_of(node):
+ return text_attr(node, node.get_role_name)
+
+
+def pid_of(node):
+ return int(attempt(node.get_process_id, 0) or 0)
+
+
+def has_state(node, state):
+ state_set = attempt(node.get_state_set)
+ return bool(state_set is not None and attempt(lambda: state_set.contains(state), False))
+
+
+def screen_rect(node):
+ component = attempt(node.get_component_iface)
+ if component is None:
+ return None
+ rect = attempt(lambda: Atspi.Component.get_extents(component, Atspi.CoordType.SCREEN))
+ if rect is None or rect.width <= 0 or rect.height <= 0:
+ return None
+ return Rect(float(rect.x), float(rect.y), float(rect.width), float(rect.height))
+
+
+def relative_rect(node, window_rect):
+ rect = screen_rect(node)
+ if rect is None or window_rect is None:
+ return rect
+ return Rect(rect.x - window_rect.x, rect.y - window_rect.y, rect.width, rect.height)
+
+
+def desktop_apps():
+ for _, app in children(desktop_root()):
+ if name_of(app):
+ yield app
+
+
+def windows_for(app):
+ result = []
+ for index, child in children(app):
+ role = role_of(child).lower()
+ rect = screen_rect(child)
+ if rect is not None or role in {"frame", "window", "dialog", "alert"}:
+ result.append((index, child))
+ return result
+
+
+def choose_window(app, window_id=None, window_index=None):
+ windows = windows_for(app)
+ if not windows:
+ raise RuntimeError("No top-level AT-SPI window is available for " + name_of(app))
+ target = window_id if window_id is not None else window_index
+ if target is not None:
+ for item in windows:
+ if item[0] == int(target):
+ return item
+ raise RuntimeError(f'windowNotFound("{target}")')
+ for item in windows:
+ if has_state(item[1], Atspi.StateType.ACTIVE):
+ return item
+ for item in windows:
+ if has_state(item[1], Atspi.StateType.SHOWING):
+ return item
+ return windows[0]
+
+
+def restore_window(app):
+ pid = pid_of(app)
+ if not pid or not shutil.which("xdotool"):
+ return
+ subprocess.run(
+ ["xdotool", "search", "--pid", str(pid), "windowactivate", "--sync"],
+ check=False,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+
+
+def require_keyboard_focus(window, operation):
+ if operation.get("restoreWindow") or has_state(window, Atspi.StateType.ACTIVE):
+ return
+ raise RuntimeError("window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window")
+
+
+def app_matches(app, query):
+ needle = str(query or "").strip().lower()
+ if not needle:
+ return False
+ if needle.startswith("pid:"):
+ return str(pid_of(app)) == needle[4:]
+ if needle.isdigit() and pid_of(app) == int(needle):
+ return True
+ haystacks = [name_of(app).lower()] + [name_of(window).lower() for _, window in windows_for(app)]
+ return any(value == needle or needle in value for value in haystacks)
+
+
+def find_app(query):
+ for app in desktop_apps():
+ if app_matches(app, query):
+ reject_blocked_app(app)
+ return app
+ raise RuntimeError(f'appNotFound("{query}")')
+
+
+def reject_blocked_app(app):
+ haystacks = [name_of(app).lower()] + [name_of(window).lower() for _, window in windows_for(app)]
+ if any(fragment in value for fragment in BLOCKED_APP_FRAGMENTS for value in haystacks):
+ raise RuntimeError(f'appBlocked("{name_of(app)}")')
+
+
+def action_labels(node):
+ labels = []
+ count = int(attempt(node.get_n_actions, 0) or 0)
+ for index in range(count):
+ label = str(attempt(lambda i=index: node.get_action_name(i), "") or "")
+ description = str(attempt(lambda i=index: node.get_action_description(i), "") or "")
+ value = label or description
+ if value and value not in labels:
+ labels.append(value)
+ return labels
+
+
+def meaningful_actions(actions):
+ noisy = {
+ "click",
+ "press",
+ "show default ui",
+ "show alternate ui",
+ "show menu",
+ "scroll to visible",
+ "raise",
+ }
+ return [action for action in actions if action.strip().lower() not in noisy]
+
+
+def display_action(action):
+ value = str(action or "").strip()
+ if not value:
+ return value
+ return " ".join(value.replace("_", " ").split())
+
+
+def sanitize_text(value):
+ return " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split())
+
+
+def formatted_value(role_key, title, value):
+ clean = sanitize_text(value)
+ if not clean or clean == title:
+ return ""
+ if role_key in {"label", "static", "static text", "text", "entry", "text entry"}:
+ return " " + clean
+ return ", Value: " + clean
+
+
+def suppress_children(role_key, title, value, summary):
+ has_compact_label = bool(title or sanitize_text(value) or sanitize_text(summary))
+ return has_compact_label and role_key in {
+ "button",
+ "check box",
+ "checkbox",
+ "combo box",
+ "heading",
+ "link",
+ "menu item",
+ "page tab",
+ "push button",
+ "radio button",
+ }
+
+
+def string_value(node):
+ if is_secure_node(node):
+ return "[redacted]"
+ if bool(attempt(node.is_text, False)):
+ iface = attempt(node.get_text_iface)
+ count = int(attempt(lambda: Atspi.Text.get_character_count(iface), 0) or 0)
+ if iface is not None and count > 0:
+ value = str(attempt(lambda: Atspi.Text.get_text(iface, 0, min(count, TEXT_LIMIT)), "") or "")
+ return value + ("..." if count > TEXT_LIMIT else "")
+ value_iface = attempt(node.get_value_iface)
+ if value_iface is not None:
+ current = attempt(lambda: Atspi.Value.get_current_value(value_iface))
+ if current is not None:
+ return str(current)
+ return ""
+
+
+def is_secure_node(node):
+ role = role_of(node).lower()
+ label = " ".join([role, name_of(node).lower(), accessible_id(node).lower()])
+ if any(term in label for term in ("password", "passcode", "pin", "secret", "one-time code")):
+ return True
+ state_set = attempt(node.get_state_set)
+ protected_state = getattr(Atspi.StateType, "PROTECTED", None)
+ if state_set is not None and protected_state is not None and attempt(lambda: state_set.contains(protected_state), False):
+ return True
+ return False
+
+
+def accessible_id(node):
+ return str(attempt(node.get_accessible_id, "") or "")
+
+
+def record(node, index, path, window_rect):
+ role = role_of(node)
+ rect = relative_rect(node, window_rect)
+ return {
+ "index": index,
+ "runtimeId": path,
+ "automationId": accessible_id(node),
+ "name": name_of(node),
+ "controlType": role,
+ "localizedControlType": role,
+ "className": str(attempt(node.get_toolkit_name, "") or ""),
+ "value": string_value(node),
+ "nativeWindowHandle": 0,
+ "frame": rect.to_json() if rect else None,
+ "actions": action_labels(node),
+ }
+
+
+def render_accessibility_tree(root, window_rect, root_path):
+ records = []
+ lines = []
+ truncation = {"truncated": False, "maxNodes": MAX_NODES, "maxDepth": MAX_DEPTH, "maxDepthReached": False}
+
+ def text_snippets(node, limit=6, max_depth=3):
+ values = []
+ seen = set()
+
+ def collect(candidate, depth):
+ if len(values) >= limit or depth > max_depth:
+ return
+ role = role_of(candidate).lower()
+ if role in {"label", "static", "static text", "text", "link"}:
+ for raw in (name_of(candidate), string_value(candidate)):
+ value = " ".join(str(raw or "").split())
+ if value and value not in seen:
+ seen.add(value)
+ values.append(value[:80])
+ if len(values) >= limit:
+ return
+ for _, child in children(candidate):
+ collect(child, depth + 1)
+ if len(values) >= limit:
+ return
+
+ collect(node, 0)
+ return values
+
+ def is_plain_text_subtree(node, max_depth=4):
+ saw_text = False
+ allowed = {"panel", "filler", "unknown", "section", "label", "static", "static text", "text", "link", "image"}
+
+ def visit(candidate, depth):
+ nonlocal saw_text
+ if depth > max_depth:
+ return False
+ role = role_of(candidate).lower()
+ if role not in allowed:
+ return False
+ if role in {"label", "static", "static text", "text", "link"}:
+ saw_text = True
+ if meaningful_actions(action_labels(candidate)):
+ return False
+ return all(visit(child, depth + 1) for _, child in children(candidate))
+
+ return visit(node, 0) and saw_text
+
+ def should_elide(item, child_count, summary):
+ role = (item["controlType"] or "").lower()
+ has_text = bool(item["name"] or item["automationId"] or item["value"])
+ return role in {"panel", "filler", "unknown", "section"} and not has_text and not meaningful_actions(item["actions"]) and summary is None and child_count <= 1
+
+ def walk(node, depth, path):
+ if len(records) >= MAX_NODES or depth > MAX_DEPTH:
+ truncation["truncated"] = True
+ if depth > MAX_DEPTH:
+ truncation["maxDepthReached"] = True
+ return
+ item = record(node, len(records), path, window_rect)
+ child_items = list(children(node))
+ role_key = (item["controlType"] or "").lower()
+ summary_values = text_snippets(node, limit=8, max_depth=4)
+ generic_summary = " ".join(summary_values) if role_key in {"panel", "filler", "unknown", "section"} and not item["name"] and not item["value"] and len(summary_values) >= 2 and is_plain_text_subtree(node) else None
+ if should_elide(item, len(child_items), generic_summary):
+ for child_index, child in child_items:
+ walk(child, depth, path + [child_index])
+ return
+ records.append(item)
+ title = item["name"] or item["automationId"] or ""
+ role_label = item["localizedControlType"] or item["controlType"]
+ line = f'{item["index"]} {role_label} {sanitize_text(title)}'.rstrip()
+ line += formatted_value(role_key, title, item["value"])
+ if generic_summary and generic_summary != title:
+ line += ", Text: " + sanitize_text(generic_summary)
+ elif role_key in {"row", "table row", "list item"}:
+ row_summary = " ".join(text_snippets(node))
+ if row_summary and row_summary != title:
+ line += ", Text: " + sanitize_text(row_summary)
+ filtered_actions = meaningful_actions(item["actions"])
+ if filtered_actions:
+ line += ", Secondary Actions: " + ", ".join(display_action(action) for action in filtered_actions)
+ lines.append(("\t" * depth) + line)
+ if generic_summary or suppress_children(role_key, title, item["value"], generic_summary):
+ return
+ for child_index, child in child_items:
+ walk(child, depth + 1, path + [child_index])
+
+ walk(root, 0, root_path)
+ return records, lines, truncation
+
+
+def capture_png(rect):
+ if Gdk is None or rect is None or os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland":
+ return None
+ screen = Gdk.Screen.get_default()
+ root = screen.get_root_window() if screen else None
+ if root is None:
+ return None
+ pixbuf = Gdk.pixbuf_get_from_window(root, round(rect.x), round(rect.y), max(1, round(rect.width)), max(1, round(rect.height)))
+ if pixbuf is None:
+ return None
+ ok, data = pixbuf.save_to_bufferv("png", [], [])
+ return base64.b64encode(bytes(data)).decode("ascii") if ok else None
+
+
+def first_descendant(root, predicate):
+ if predicate(root):
+ return root
+ for _, child in children(root):
+ found = first_descendant(child, predicate)
+ if found is not None:
+ return found
+ return None
+
+
+def focused_summary(window):
+ node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED))
+ if node is None:
+ return None
+ return (role_of(node) + " " + name_of(node)).strip()
+
+
+def selected_text(window):
+ node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED) and bool(attempt(candidate.is_text, False)))
+ iface = attempt(node.get_text_iface) if node is not None else None
+ selections = attempt(lambda: Atspi.Text.get_text_selections(iface), [])
+ if not selections:
+ return None
+ selection = selections[0]
+ return Atspi.Text.get_text(iface, selection.start_offset, selection.end_offset)
+
+
+def app_json(app):
+ return {"name": name_of(app), "bundleIdentifier": name_of(app), "pid": pid_of(app)}
+
+
+def window_json(app, index, window):
+ bounds = screen_rect(window)
+ showing = has_state(window, Atspi.StateType.SHOWING)
+ return {
+ "index": index,
+ "app": app_json(app),
+ "id": index,
+ "title": name_of(window),
+ "x": round(bounds.x) if bounds else None,
+ "y": round(bounds.y) if bounds else None,
+ "width": max(0, round(bounds.width if bounds else 0)),
+ "height": max(0, round(bounds.height if bounds else 0)),
+ "isMinimized": not showing,
+ "isOffscreen": not showing,
+ "screenIndex": None,
+ "platform": {"backend": "at-spi", "runtimeId": [index], "role": role_of(window)},
+ }
+
+
+def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False):
+ app = find_app(query)
+ if restore:
+ restore_window(app)
+ window_index, window = choose_window(app, window_id, window_index)
+ bounds = screen_rect(window)
+ records, lines, truncation = render_accessibility_tree(window, bounds, [window_index])
+ return {
+ "snapshotId": str(uuid.uuid4()),
+ "app": app_json(app),
+ "windowTitle": name_of(window),
+ "windowId": window_index,
+ "windowBounds": bounds.to_json() if bounds else None,
+ "screenshotPngBase64": capture_png(bounds) if include_screenshot else None,
+ "coordinateSpace": "window",
+ "truncation": truncation,
+ "treeLines": lines,
+ "focusedSummary": focused_summary(window),
+ "selectedText": selected_text(window),
+ "elements": records,
+ }
+
+
+def list_apps_response():
+ apps = []
+ for app in sorted(desktop_apps(), key=lambda value: (name_of(value).lower(), pid_of(value))):
+ if windows_for(app):
+ apps.append({"name": name_of(app), "bundleIdentifier": name_of(app), "pid": pid_of(app)})
+ return apps
+
+
+def list_windows_response(query):
+ app = find_app(query)
+ return {"app": app_json(app), "windows": [window_json(app, index, window) for index, window in windows_for(app)]}
+
+
+def handshake_response():
+ is_wayland = os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland"
+ has_hotkey = shutil.which("xdotool") is not None and not is_wayland
+ has_clipboard = any(shutil.which(command) for command in ("wl-copy", "xclip", "xsel"))
+ has_screenshot = Gdk is not None and not is_wayland
+ return {
+ "platform": "linux",
+ "provider": "orca-computer-use-linux",
+ "providerVersion": "1.0.0",
+ "protocolVersion": 1,
+ "supports": {
+ "apps": {"list": True, "bundleIds": False, "pids": True},
+ "windows": {"list": True, "targetById": True, "targetByIndex": True, "focus": False, "moveResize": False},
+ "observation": {"screenshot": has_screenshot, "annotatedScreenshot": False, "elementFrames": True, "ocr": False},
+ "actions": {
+ "click": True,
+ "typeText": True,
+ "pressKey": True,
+ "hotkey": has_hotkey,
+ "pasteText": has_clipboard,
+ "scroll": True,
+ "drag": True,
+ "setValue": True,
+ "performAction": True,
+ },
+ "surfaces": {"menus": False, "dialogs": False, "dock": False, "menubar": False},
+ },
+ }
+
+
+def find_by_path(app, path):
+ node = app
+ for index in path or []:
+ node = dict(children(node)).get(int(index))
+ if node is None:
+ return None
+ return node
+
+
+def all_nodes(root):
+ result = []
+
+ def walk(node):
+ if len(result) >= MAX_NODES:
+ return
+ result.append(node)
+ for _, child in children(node):
+ walk(child)
+
+ walk(root)
+ return result
+
+
+def find_element(app, saved):
+ if not saved:
+ return None
+ by_path = find_by_path(app, saved.get("runtimeId"))
+ if by_path is not None and same_element_signature(by_path, saved):
+ return by_path
+ return None
+
+
+def same_element_signature(node, saved):
+ if role_of(node) != str(saved.get("controlType") or ""):
+ return False
+ if name_of(node) != str(saved.get("name") or ""):
+ return False
+ if accessible_id(node) != str(saved.get("automationId") or ""):
+ return False
+ saved_actions = [str(action) for action in saved.get("actions") or []]
+ return action_labels(node) == saved_actions
+
+
+def preferred_action(node):
+ if node is None:
+ return None
+ priority = {"click", "press", "activate", "invoke", "select", "toggle", "open"}
+ fallback = None
+ for index in range(int(attempt(node.get_n_actions, 0) or 0)):
+ label = str(attempt(lambda i=index: node.get_action_name(i), "") or "").lower()
+ if label in priority:
+ return index
+ if fallback is None and any(term in label for term in ("click", "press", "activate")):
+ fallback = index
+ return fallback
+
+
+def perform_action(node, index):
+ return bool(index is not None and attempt(lambda: node.do_action(int(index)), False))
+
+
+def screen_point(window_rect, saved_element=None, x=None, y=None, node=None):
+ rect = screen_rect(node) if node is not None else None
+ if rect is not None:
+ return rect.x + rect.width / 2, rect.y + rect.height / 2
+ if saved_element is not None:
+ raise RuntimeError("stale element frame; run get-app-state again and use a fresh element index")
+ if window_rect is None or x is None or y is None:
+ raise RuntimeError("coordinate action requires a visible window and coordinates")
+ return window_rect.x + float(x), window_rect.y + float(y)
+
+
+def click_at(x, y, button, count):
+ button = (button or "left").lower()
+ down, up = {"right": ("b3p", "b3r"), "middle": ("b2p", "b2r")}.get(button, ("b1p", "b1r"))
+ for _ in range(max(1, int(count or 1))):
+ Atspi.generate_mouse_event(round(x), round(y), "abs")
+ Atspi.generate_mouse_event(round(x), round(y), down)
+ time.sleep(0.03)
+ Atspi.generate_mouse_event(round(x), round(y), up)
+
+
+def scroll_at(x, y, direction, pages):
+ down, up = {
+ "up": ("b4p", "b4r"),
+ "down": ("b5p", "b5r"),
+ "left": ("b6p", "b6r"),
+ "right": ("b7p", "b7r"),
+ }.get(str(direction or "down").lower(), ("b5p", "b5r"))
+ for _ in range(max(1, math.ceil(float(pages or 1)))):
+ Atspi.generate_mouse_event(round(x), round(y), "abs")
+ Atspi.generate_mouse_event(round(x), round(y), down)
+ time.sleep(0.03)
+ Atspi.generate_mouse_event(round(x), round(y), up)
+
+
+def drag_between(start, end):
+ Atspi.generate_mouse_event(round(start[0]), round(start[1]), "abs")
+ Atspi.generate_mouse_event(round(start[0]), round(start[1]), "b1p")
+ for step in range(1, 13):
+ x = start[0] + (end[0] - start[0]) * step / 12
+ y = start[1] + (end[1] - start[1]) * step / 12
+ Atspi.generate_mouse_event(round(x), round(y), "abs")
+ time.sleep(0.02)
+ Atspi.generate_mouse_event(round(end[0]), round(end[1]), "b1r")
+
+
+def key_name(raw):
+ aliases = {
+ "return": "Return", "enter": "Return", "tab": "Tab", "escape": "Escape", "esc": "Escape",
+ "backspace": "BackSpace", "delete": "Delete", "space": "space", "left": "Left", "right": "Right",
+ "up": "Up", "down": "Down", "page_up": "Page_Up", "page_down": "Page_Down",
+ }
+ return aliases.get(str(raw).lower(), str(raw))
+
+
+def press_key(raw):
+ name = key_name(raw)
+ if len(name) == 1:
+ Atspi.generate_keyboard_event(0, name, Atspi.KeySynthType.STRING)
+ return
+ if Gdk is None:
+ raise RuntimeError("GDK is required for non-character key synthesis")
+ Atspi.generate_keyboard_event(Gdk.keyval_from_name(name), None, Atspi.KeySynthType.PRESSRELEASE)
+
+
+def hotkey(raw):
+ key_spec = re.sub(r"(?i)commandorcontrol|cmdorctrl", "ctrl", str(raw))
+ xdotool = shutil.which("xdotool")
+ if xdotool:
+ subprocess.run([xdotool, "key", key_spec], check=True)
+ return
+ if "+" in key_spec:
+ raise RuntimeError("hotkey combinations require xdotool")
+ press_key(key_spec)
+
+
+def type_text(value):
+ Atspi.generate_keyboard_event(0, str(value), Atspi.KeySynthType.STRING)
+
+
+def paste_text(value):
+ text = str(value)
+ previous = read_clipboard()
+ try:
+ write_clipboard(text)
+ hotkey("ctrl+v")
+ finally:
+ if previous is not None:
+ write_clipboard(previous)
+
+
+def read_clipboard():
+ for command in (["wl-paste"], ["xclip", "-selection", "clipboard", "-o"], ["xsel", "--clipboard", "--output"]):
+ if shutil.which(command[0]):
+ result = subprocess.run(command, check=False, capture_output=True, text=True)
+ if result.returncode == 0:
+ return result.stdout
+ return None
+
+
+def write_clipboard(value):
+ for command in (["wl-copy"], ["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"]):
+ if shutil.which(command[0]):
+ subprocess.run(command, input=value, check=True, text=True)
+ return
+ raise RuntimeError("paste_text requires wl-copy, xclip, or xsel")
+
+
+def set_value(node, value):
+ if node is not None and bool(attempt(node.is_editable_text, False)):
+ editable = attempt(node.get_editable_text_iface)
+ if editable is not None and attempt(lambda: Atspi.EditableText.set_text_contents(editable, str(value)), False):
+ return True
+ value_iface = attempt(node.get_value_iface) if node is not None else None
+ if value_iface is not None:
+ return bool(attempt(lambda: Atspi.Value.set_current_value(value_iface, float(value)), False))
+ return False
+
+
+def run_operation(operation):
+ tool = operation.get("tool")
+ include_screenshot = not bool(operation.get("noScreenshot"))
+ if tool == "handshake":
+ return {"ok": True, "capabilities": handshake_response()}
+ if tool == "list_apps":
+ return {"ok": True, "apps": list_apps_response()}
+ if tool == "list_windows":
+ return {"ok": True, **list_windows_response(operation.get("app", ""))}
+ if tool == "get_app_state":
+ return {
+ "ok": True,
+ "snapshot": make_snapshot(
+ operation.get("app", ""),
+ include_screenshot,
+ operation.get("windowId"),
+ operation.get("windowIndex"),
+ bool(operation.get("restoreWindow")),
+ ),
+ }
+
+ app = find_app(operation.get("app", ""))
+ if operation.get("restoreWindow"):
+ restore_window(app)
+ _, window = choose_window(app, operation.get("windowId"), operation.get("windowIndex"))
+ if tool in {"type_text", "press_key", "hotkey", "paste_text"}:
+ require_keyboard_focus(window, operation)
+ bounds = screen_rect(window)
+ saved = operation.get("element")
+ node = find_element(app, saved)
+ from_node = find_element(app, operation.get("fromElement"))
+ to_node = find_element(app, operation.get("toElement"))
+ action = None
+
+ if tool == "click":
+ preferred = preferred_action(node)
+ click_count = int(operation.get("click_count", 1) or 1)
+ handled = operation.get("mouse_button", "left") == "left" and click_count <= 1 and perform_action(node, preferred)
+ if not handled:
+ click_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node), operation.get("mouse_button", "left"), operation.get("click_count", 1))
+ action = {"path": "synthetic", "actionName": None, "fallbackReason": "actionUnsupported"}
+ else:
+ labels = action_labels(node)
+ action = {"path": "accessibility", "actionName": labels[preferred] if preferred is not None and preferred < len(labels) else "action", "fallbackReason": None}
+ elif tool == "perform_secondary_action":
+ wanted = str(operation.get("action", "")).lower()
+ for index, label in enumerate(action_labels(node)):
+ if label.lower() == wanted and perform_action(node, index):
+ action = {"path": "accessibility", "actionName": label, "fallbackReason": None}
+ break
+ else:
+ raise RuntimeError(f'{operation.get("action", "")} is not a valid secondary action')
+ elif tool == "scroll":
+ scroll_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node), operation.get("direction"), operation.get("pages"))
+ action = {"path": "synthetic", "actionName": "scroll", "fallbackReason": None}
+ elif tool == "drag":
+ drag_between(
+ screen_point(bounds, operation.get("fromElement"), operation.get("from_x"), operation.get("from_y"), from_node),
+ screen_point(bounds, operation.get("toElement"), operation.get("to_x"), operation.get("to_y"), to_node),
+ )
+ action = {"path": "synthetic", "actionName": "drag", "fallbackReason": None}
+ elif tool == "type_text":
+ type_text(operation.get("text", ""))
+ action = {"path": "synthetic", "actionName": "typeText", "fallbackReason": None}
+ elif tool == "press_key":
+ press_key(operation.get("key", ""))
+ action = {"path": "synthetic", "actionName": "pressKey", "fallbackReason": None}
+ elif tool == "hotkey":
+ hotkey(operation.get("key", ""))
+ action = {"path": "synthetic", "actionName": "hotkey", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}
+ elif tool == "paste_text":
+ paste_text(operation.get("text", ""))
+ action = {"path": "clipboard", "actionName": "paste", "fallbackReason": None, "verification": {"state": "unverified", "reason": "clipboard_paste"}}
+ elif tool == "set_value":
+ if not set_value(node, operation.get("value", "")):
+ raise RuntimeError("element value is not settable")
+ action = {"path": "accessibility", "actionName": "setValue", "fallbackReason": None}
+ else:
+ raise RuntimeError("unknown tool: " + str(tool))
+
+ try:
+ snapshot = make_snapshot(
+ operation.get("app", ""),
+ include_screenshot,
+ operation.get("windowId"),
+ operation.get("windowIndex"),
+ )
+ except Exception:
+ if operation.get("windowId") is None and operation.get("windowIndex") is None:
+ raise
+ action.setdefault("verification", {"state": "unverified", "reason": "window_changed"})
+ snapshot = make_snapshot(operation.get("app", ""), include_screenshot, None, None)
+
+ return {"ok": True, "action": action, "snapshot": snapshot}
+
+
+def main():
+ try:
+ ensure_desktop_bus()
+ with open(sys.argv[1], "r", encoding="utf-8") as handle:
+ operation = json.load(handle)
+ print(json.dumps(run_operation(operation), separators=(",", ":")))
+ except Exception as exc:
+ print(json.dumps({"ok": False, "error": str(exc)}, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/native/computer-use-macos/Package.swift b/native/computer-use-macos/Package.swift
new file mode 100644
index 00000000000..9e7da76790f
--- /dev/null
+++ b/native/computer-use-macos/Package.swift
@@ -0,0 +1,36 @@
+// swift-tools-version: 6.0
+
+import PackageDescription
+
+let package = Package(
+ name: "OrcaComputerUseMacOS",
+ platforms: [
+ .macOS(.v14)
+ ],
+ products: [
+ .library(
+ name: "OrcaComputerUseMacOSCore",
+ targets: ["OrcaComputerUseMacOSCore"]
+ ),
+ .executable(
+ name: "orca-computer-use-macos",
+ targets: ["OrcaComputerUseMacOS"]
+ )
+ ],
+ targets: [
+ .target(
+ name: "OrcaComputerUseMacOSCore",
+ path: "Sources/OrcaComputerUseMacOSCore"
+ ),
+ .executableTarget(
+ name: "OrcaComputerUseMacOS",
+ dependencies: ["OrcaComputerUseMacOSCore"],
+ path: "Sources/OrcaComputerUseMacOS"
+ ),
+ .testTarget(
+ name: "OrcaComputerUseMacOSTests",
+ dependencies: ["OrcaComputerUseMacOSCore"],
+ path: "Tests/OrcaComputerUseMacOSTests"
+ )
+ ]
+)
diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift
new file mode 100644
index 00000000000..2f5800e3dfa
--- /dev/null
+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift
@@ -0,0 +1,3021 @@
+import AppKit
+import ApplicationServices
+import CoreGraphics
+import Darwin
+import Foundation
+import ImageIO
+import OrcaComputerUseMacOSCore
+import ScreenCaptureKit
+
+private let providerName = "orca-computer-use-macos"
+private let providerVersion = "1.0.0"
+private let providerProtocolVersion = 1
+
+struct Request: Decodable {
+ let id: Int
+ let method: String
+ let params: [String: JSONValue]?
+ let token: String?
+}
+
+enum JSONValue: Decodable {
+ case string(String)
+ case number(Double)
+ case bool(Bool)
+ case object([String: JSONValue])
+ case array([JSONValue])
+ case null
+
+ init(from decoder: Decoder) throws {
+ let container = try decoder.singleValueContainer()
+ if container.decodeNil() {
+ self = .null
+ } else if let value = try? container.decode(Bool.self) {
+ self = .bool(value)
+ } else if let value = try? container.decode(Double.self) {
+ self = .number(value)
+ } else if let value = try? container.decode(String.self) {
+ self = .string(value)
+ } else if let value = try? container.decode([JSONValue].self) {
+ self = .array(value)
+ } else {
+ self = .object(try container.decode([String: JSONValue].self))
+ }
+ }
+
+ var string: String? {
+ if case let .string(value) = self { return value }
+ return nil
+ }
+
+ var number: Double? {
+ if case let .number(value) = self { return value }
+ return nil
+ }
+
+ var bool: Bool? {
+ if case let .bool(value) = self { return value }
+ return nil
+ }
+}
+
+enum ProviderError: Error {
+ case coded(String, String)
+
+ var code: String {
+ switch self {
+ case let .coded(code, _):
+ return code
+ }
+ }
+
+ var message: String {
+ switch self {
+ case let .coded(_, message):
+ return message
+ }
+ }
+}
+
+struct AppDescriptor {
+ let name: String
+ let bundleId: String?
+ let pid: pid_t
+ let app: NSRunningApplication
+
+ var needsManualAccessibilityMode: Bool {
+ // Chromium/Electron apps often need this private AX mode, but applying it
+ // broadly can corrupt native Cocoa app trees into app-root-only nodes.
+ guard let bundleId = bundleId?.lowercased() else {
+ return false
+ }
+ return bundleId.hasPrefix("com.google.chrome") ||
+ bundleId.hasPrefix("com.microsoft.edgemac") ||
+ bundleId.hasPrefix("com.brave.browser") ||
+ bundleId.hasPrefix("com.operasoftware.opera") ||
+ bundleId.hasPrefix("com.vivaldi.vivaldi") ||
+ bundleId == "com.github.electron" ||
+ bundleId == "com.tinyspeck.slackmacgap" ||
+ bundleId == "com.spotify.client" ||
+ bundleId == "com.hnc.discord" ||
+ bundleId == "com.microsoft.teams2" ||
+ bundleId == "notion.id"
+ }
+}
+
+final class ElementRecord {
+ let index: Int
+ let element: AXUIElement
+ let localFrame: CGRect?
+ let actions: [String]
+ let signature: String
+
+ init(index: Int, element: AXUIElement, localFrame: CGRect?, actions: [String], signature: String) {
+ self.index = index
+ self.element = element
+ self.localFrame = localFrame
+ self.actions = actions
+ self.signature = signature
+ }
+}
+
+struct Snapshot {
+ let id: String
+ let app: AppDescriptor
+ let windowTitle: String
+ let windowBounds: CGRect
+ let windowId: CGWindowID
+ let windowLayer: Int
+ let treeText: String
+ let focusedElementId: Int?
+ let screenshot: ScreenshotPayload?
+ let screenshotStatus: ScreenshotStatus
+ let screenshotScale: CGSize
+ let screenshotEngine: String?
+ let elements: [Int: ElementRecord]
+ let truncated: Bool
+ let maxDepthReached: Bool
+}
+
+struct ScreenshotPayload {
+ let data: String
+ let width: Int
+ let height: Int
+ let scale: Double
+}
+
+struct CapturedImage {
+ let image: CGImage
+ let engine: String
+}
+
+enum ScreenshotStatus {
+ case captured
+ case skipped
+ case failed(String)
+}
+
+final class Provider {
+ private var snapshots: [String: Snapshot] = [:]
+ private var hasPromptedForAccessibility = false
+
+ func handle(method: String, params: [String: JSONValue]) throws -> Any {
+ switch method {
+ case "handshake":
+ return providerHandshake()
+ case "listApps":
+ return ["apps": listApps().map(renderListedApp)]
+ case "listWindows":
+ return try listWindows(params: params)
+ case "getAppState":
+ return try renderSnapshot(observe(params: params))
+ case "click":
+ return try actionResult(params: params) { try click(params: params) }
+ case "performSecondaryAction":
+ return try actionResult(params: params) { try performSecondaryAction(params: params) }
+ case "setValue":
+ return try actionResult(params: params) { try setValue(params: params) }
+ case "typeText":
+ return try actionResult(params: params) { try typeText(params: params) }
+ case "pressKey":
+ return try actionResult(params: params) { try pressKey(params: params) }
+ case "hotkey":
+ return try actionResult(params: params) { try hotkey(params: params) }
+ case "pasteText":
+ return try actionResult(params: params) { try pasteText(params: params) }
+ case "scroll":
+ return try actionResult(params: params) { try scroll(params: params) }
+ case "drag":
+ return try actionResult(params: params) { try drag(params: params) }
+ default:
+ throw ProviderError.coded("invalid_argument", "unknown method '\(method)'")
+ }
+ }
+
+ private func actionResult(params: [String: JSONValue], action runAction: () throws -> [String: Any]) throws -> [String: Any] {
+ var action = try runAction()
+ do {
+ return try renderActionResult(action: action, snapshot: observe(params: params))
+ } catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && (requestedWindowId(params) != nil || requestedWindowIndex(params) != nil) {
+ var fallbackParams = params
+ fallbackParams.removeValue(forKey: "windowId")
+ fallbackParams.removeValue(forKey: "windowIndex")
+ if action["verification"] == nil {
+ action["verification"] = ["state": "unverified", "reason": "window_changed"]
+ }
+ return try renderActionResult(action: action, snapshot: observe(params: fallbackParams))
+ }
+ }
+
+ private func observe(params: [String: JSONValue]) throws -> Snapshot {
+ let query = try requiredString(params, "app")
+ let app = try resolveApp(query)
+ if params["restoreWindow"]?.bool == true {
+ recoverWindow(app)
+ }
+ let snapshot = try buildSnapshot(
+ app: app,
+ includeScreenshot: params["noScreenshot"]?.bool != true,
+ windowId: requestedWindowId(params),
+ windowIndex: requestedWindowIndex(params),
+ restoreWindow: params["restoreWindow"]?.bool == true
+ )
+ let keys = [query, app.name, app.bundleId ?? ""].filter { !$0.isEmpty }.map { $0.lowercased() }
+ let namespace = snapshotNamespace(params)
+ for key in keys {
+ if !isExplicitSnapshotNamespace(namespace) {
+ snapshots[key] = snapshot
+ snapshots[snapshotWindowKey(key, snapshot.windowId)] = snapshot
+ if let windowIndex = requestedWindowIndex(params) {
+ snapshots[snapshotWindowIndexKey(key, windowIndex)] = snapshot
+ }
+ }
+ snapshots[namespacedSnapshotKey(namespace, key)] = snapshot
+ snapshots[namespacedSnapshotKey(namespace, snapshotWindowKey(key, snapshot.windowId))] = snapshot
+ if let windowIndex = requestedWindowIndex(params) {
+ snapshots[namespacedSnapshotKey(namespace, snapshotWindowIndexKey(key, windowIndex))] = snapshot
+ }
+ }
+ return snapshot
+ }
+
+ private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot {
+ let cached = cachedSnapshot(params: params)
+ // Why: cached AX frames can be stale after a window move or resize, and
+ // stale geometry can turn an intended action into a misclick.
+ let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
+ try validateRequestedElements(cached: cached, current: snapshot, params: params)
+ return snapshot
+ }
+
+ private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot {
+ let snapshot = try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
+ if params["restoreWindow"]?.bool != true && !isTargetWindowFocused(snapshot) {
+ throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements")
+ }
+ return snapshot
+ }
+
+ private func cachedSnapshot(params: [String: JSONValue]) -> Snapshot? {
+ guard let query = params["app"]?.string, !query.isEmpty else { return nil }
+ let namespace = snapshotNamespace(params)
+ if let targetWindowId = requestedWindowId(params) {
+ let windowKey = snapshotWindowKey(query.lowercased(), targetWindowId)
+ return snapshots[namespacedSnapshotKey(namespace, windowKey)] ??
+ (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[windowKey])
+ }
+ if let targetWindowIndex = requestedWindowIndex(params) {
+ let windowKey = snapshotWindowIndexKey(query.lowercased(), targetWindowIndex)
+ return snapshots[namespacedSnapshotKey(namespace, windowKey)] ??
+ (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[windowKey])
+ }
+ let key = query.lowercased()
+ return snapshots[namespacedSnapshotKey(namespace, key)] ??
+ (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key])
+ }
+
+ private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws {
+ let requestedKeys = ["elementIndex", "fromElementIndex", "toElementIndex"].filter {
+ params[$0]?.number != nil
+ }
+ guard !requestedKeys.isEmpty else { return }
+ guard let cached else {
+ throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window")
+ }
+ for key in requestedKeys {
+ guard let value = params[key]?.number else { continue }
+ let index = Int(value)
+ guard let expected = cached.elements[index], let actual = current.elements[index] else {
+ throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index")
+ }
+ guard expected.signature == actual.signature else {
+ throw ProviderError.coded("element_not_found", "element \(index) changed since the last snapshot; run get-app-state again and use a fresh element index")
+ }
+ }
+ }
+
+ private func ensureWindowStillAvailable(_ snapshot: Snapshot) throws {
+ guard WindowCapture.candidates(pid: snapshot.app.pid).contains(where: { $0.windowId == snapshot.windowId }) else {
+ throw ProviderError.coded("window_stale", "window \(Int(snapshot.windowId)) is no longer available; run get-app-state again to refresh the target window")
+ }
+ }
+
+ private func promptForAccessibilityOnce() {
+ guard !hasPromptedForAccessibility else {
+ return
+ }
+ hasPromptedForAccessibility = true
+ _ = promptForAccessibility()
+ }
+
+ private func listApps() -> [AppDescriptor] {
+ var seen = Set()
+ return NSWorkspace.shared.runningApplications
+ .filter { !$0.isTerminated && $0.activationPolicy == .regular }
+ .compactMap { app in
+ guard let name = app.localizedName, !name.isEmpty else { return nil }
+ let pid = app.processIdentifier
+ guard pid > 0, pidIsLive(pid) else { return nil }
+ let key = (app.bundleIdentifier ?? "pid:\(pid)").lowercased()
+ guard seen.insert(key).inserted else { return nil }
+ return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app)
+ }
+ .sorted { lhs, rhs in
+ if lhs.app.isActive != rhs.app.isActive {
+ return lhs.app.isActive && !rhs.app.isActive
+ }
+ return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending
+ }
+ }
+
+ private func renderListedApp(_ app: AppDescriptor) -> [String: Any] {
+ [
+ "name": app.name,
+ "bundleId": jsonNullable(app.bundleId),
+ "pid": Int(app.pid),
+ "isRunning": true,
+ "lastUsedAt": NSNull(),
+ "useCount": NSNull(),
+ ]
+ }
+
+ private func providerHandshake() -> [String: Any] {
+ [
+ "platform": "darwin",
+ "provider": providerName,
+ "providerVersion": providerVersion,
+ "protocolVersion": providerProtocolVersion,
+ "supports": [
+ "apps": [
+ "list": true,
+ "bundleIds": true,
+ "pids": true,
+ ],
+ "windows": [
+ "list": true,
+ "targetById": true,
+ "targetByIndex": true,
+ "focus": false,
+ "moveResize": false,
+ ],
+ "observation": [
+ "screenshot": true,
+ "annotatedScreenshot": false,
+ "elementFrames": true,
+ "ocr": false,
+ ],
+ "actions": [
+ "click": true,
+ "typeText": true,
+ "pressKey": true,
+ "hotkey": true,
+ "pasteText": true,
+ "scroll": true,
+ "drag": true,
+ "setValue": true,
+ "performAction": true,
+ ],
+ "surfaces": [
+ "menus": false,
+ "dialogs": false,
+ "dock": false,
+ "menubar": false,
+ ],
+ ],
+ ]
+ }
+
+ private func resolveApp(_ query: String) throws -> AppDescriptor {
+ let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !trimmed.isEmpty else {
+ throw ProviderError.coded("invalid_argument", "app query must not be empty")
+ }
+ if let pid = parsePid(trimmed) {
+ if let app = appByPid(pid) {
+ try rejectBlockedApp(app)
+ return app
+ }
+ throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found")
+ }
+ if blockedBundleIds.contains(trimmed) {
+ throw ProviderError.coded("app_blocked", "app '\(trimmed)' is blocked for safety")
+ }
+ if let app = listApps().first(where: { matches($0, query: trimmed) }) {
+ try rejectBlockedApp(app)
+ return app
+ }
+ throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found")
+ }
+
+ private func rejectBlockedApp(_ app: AppDescriptor) throws {
+ if let bundle = app.bundleId, blockedBundleIds.contains(bundle) {
+ throw ProviderError.coded("app_blocked", "app '\(bundle)' is blocked for safety")
+ }
+ }
+
+ private func listWindows(params: [String: JSONValue]) throws -> [String: Any] {
+ let app = try resolveApp(try requiredString(params, "app"))
+ let windows = WindowCapture.candidates(pid: app.pid)
+ .filter { $0.layer == 0 }
+ .enumerated()
+ .map { index, candidate -> [String: Any] in
+ [
+ "index": index,
+ "app": [
+ "name": app.name,
+ "bundleId": jsonNullable(app.bundleId),
+ "pid": Int(app.pid),
+ ],
+ "id": Int(candidate.windowId),
+ "title": candidate.title ?? "",
+ "x": Int(candidate.bounds.origin.x.rounded()),
+ "y": Int(candidate.bounds.origin.y.rounded()),
+ "width": Int(candidate.bounds.width.rounded()),
+ "height": Int(candidate.bounds.height.rounded()),
+ "isMinimized": false,
+ "isOffscreen": !candidate.isOnScreen,
+ "screenIndex": jsonNullable(screenIndex(for: candidate.bounds)),
+ "isMain": NSNull(),
+ "platform": [
+ "layer": candidate.layer,
+ "alpha": candidate.alpha,
+ ],
+ ]
+ }
+ return [
+ "app": renderListedApp(app),
+ "windows": windows,
+ ]
+ }
+
+ private func appByPid(_ pid: pid_t) -> AppDescriptor? {
+ guard let app = NSRunningApplication(processIdentifier: pid),
+ !app.isTerminated,
+ let name = app.localizedName
+ else {
+ return nil
+ }
+ return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app)
+ }
+
+ private func buildSnapshot(
+ app: AppDescriptor,
+ includeScreenshot: Bool,
+ windowId: CGWindowID?,
+ windowIndex: Int?,
+ restoreWindow: Bool
+ ) throws -> Snapshot {
+ guard accessibilityTrusted() else {
+ promptForAccessibilityOnce()
+ throw ProviderError.coded("permission_denied", "Accessibility permission is required for Orca Computer Use.")
+ }
+ let appElement = AXUIElementCreateApplication(app.pid)
+ enableManualAccessibilityIfNeeded(appElement, app: app)
+ let windowCandidates = WindowCapture.candidates(pid: app.pid)
+ let focused = try focusedWindow(
+ appElement: appElement,
+ app: app,
+ visibleWindowCount: windowCandidates.count,
+ allowRecovery: restoreWindow
+ )
+ let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name
+ guard let capture = WindowCapture.resolve(candidates: windowCandidates, titleHint: focusedTitle, windowId: windowId, windowIndex: windowIndex) else {
+ throw ProviderError.coded("window_not_found", "app '\(app.name)' has no on-screen window")
+ }
+ guard let window = matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else {
+ throw ProviderError.coded("window_not_found", "could not match accessibility window to requested window; run get-app-state again or retry without a window selector")
+ }
+ let title = stringAttribute(window, kAXTitleAttribute as String) ?? capture.title ?? app.name
+ let renderer = TreeRenderer(windowBounds: capture.bounds, focused: focusedElement(appElement: appElement))
+ renderer.render(window)
+ let screenshot = includeScreenshot ? capture.screenshotPayload() : nil
+ let screenshotStatus: ScreenshotStatus = if screenshot != nil {
+ .captured
+ } else if includeScreenshot && !screenCaptureTrusted() {
+ .failed("Screen Recording permission is required for Orca Computer Use; grant permission or pass --no-screenshot to inspect accessibility state only.")
+ } else if includeScreenshot {
+ .failed("window screenshot capture returned no image; retry with --no-screenshot if accessibility state is sufficient.")
+ } else {
+ .skipped
+ }
+ return Snapshot(
+ id: UUID().uuidString,
+ app: app,
+ windowTitle: title,
+ windowBounds: capture.bounds,
+ windowId: capture.windowId,
+ windowLayer: capture.layer,
+ treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary),
+ focusedElementId: renderer.focusedElementId,
+ screenshot: screenshot,
+ screenshotStatus: screenshotStatus,
+ screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds),
+ screenshotEngine: capture.image?.engine,
+ elements: renderer.records,
+ truncated: renderer.truncated,
+ maxDepthReached: renderer.maxDepthReached
+ )
+ }
+
+ private func renderSnapshot(_ snapshot: Snapshot) -> [String: Any] {
+ var screenshot: Any = NSNull()
+ if let payload = snapshot.screenshot {
+ screenshot = [
+ "data": payload.data,
+ "format": "png",
+ "width": payload.width,
+ "height": payload.height,
+ "scale": payload.scale,
+ ]
+ }
+ return [
+ "snapshot": [
+ "id": snapshot.id,
+ "app": [
+ "name": snapshot.app.name,
+ "bundleId": jsonNullable(snapshot.app.bundleId),
+ "pid": Int(snapshot.app.pid),
+ ],
+ "window": [
+ "id": Int(snapshot.windowId),
+ "title": snapshot.windowTitle,
+ "x": Int(snapshot.windowBounds.origin.x.rounded()),
+ "y": Int(snapshot.windowBounds.origin.y.rounded()),
+ "width": Int(snapshot.windowBounds.width.rounded()),
+ "height": Int(snapshot.windowBounds.height.rounded()),
+ "isMinimized": false,
+ "isOffscreen": false,
+ "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)),
+ "platform": [
+ "layer": snapshot.windowLayer,
+ ],
+ ],
+ "coordinateSpace": "window",
+ "treeText": snapshot.treeText,
+ "elementCount": snapshot.elements.count,
+ "focusedElementId": snapshot.focusedElementId as Any,
+ "truncation": [
+ "truncated": snapshot.truncated,
+ "maxNodes": TreeRenderer.maxNodes,
+ "maxDepth": TreeRenderer.maxDepth,
+ "maxDepthReached": snapshot.maxDepthReached,
+ ],
+ ],
+ "screenshot": screenshot,
+ "screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot),
+ ]
+ }
+
+ private func renderActionResult(action: [String: Any], snapshot: Snapshot) -> [String: Any] {
+ var result = renderSnapshot(snapshot)
+ var metadata = action
+ metadata["targetWindowId"] = Int(snapshot.windowId)
+ result["action"] = metadata
+ return result
+ }
+
+ private func click(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentSnapshot(params: params)
+ let button = params["mouseButton"]?.string ?? "left"
+ let count = Int(params["clickCount"]?.number ?? 1)
+ if let elementIndex = params["elementIndex"]?.number {
+ let record = try element(snapshot, Int(elementIndex))
+ if count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) {
+ return actionMetadata(path: "accessibility", actionName: actionName)
+ }
+ if let point = center(record.localFrame, in: snapshot.windowBounds) {
+ try Input.click(
+ pid: snapshot.app.pid,
+ at: point,
+ button: mouseButton(button),
+ count: count
+ )
+ return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported")
+ }
+ throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
+ }
+ let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
+ try Input.click(
+ pid: snapshot.app.pid,
+ at: point,
+ button: mouseButton(button),
+ count: count
+ )
+ return actionMetadata(path: "synthetic")
+ }
+
+ private func performClickAction(record: ElementRecord, mouseButton: String) throws -> String? {
+ if mouseButton == "right" {
+ return performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil
+ }
+ for action in ["AXPress", "AXConfirm", "AXOpen"] {
+ if performAction(record.element, action) {
+ return action
+ }
+ }
+ return nil
+ }
+
+ private func performSecondaryAction(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentSnapshot(params: params)
+ let record = try element(snapshot, Int(try requiredNumber(params, "elementIndex")))
+ let requested = try requiredString(params, "action")
+ let action = record.actions.first { SnapshotRenderHeuristics.prettyAction($0).caseInsensitiveCompare(requested) == .orderedSame || $0.caseInsensitiveCompare(requested) == .orderedSame }
+ guard let action else {
+ throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)")
+ }
+ guard performAction(record.element, action) else {
+ throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed")
+ }
+ return actionMetadata(path: "accessibility", actionName: action)
+ }
+
+ private func setValue(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentSnapshot(params: params)
+ let record = try element(snapshot, Int(try requiredNumber(params, "elementIndex")))
+ guard isSettable(record.element, kAXValueAttribute as String) else {
+ throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable")
+ }
+ let result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, try requiredStringAllowingEmpty(params, "value") as CFString)
+ guard result == .success else {
+ throw ProviderError.coded("accessibility_error", "AXUIElementSetAttributeValue failed with \(result.rawValue)")
+ }
+ return actionMetadata(path: "accessibility", actionName: "AXSetValue")
+ }
+
+ private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentKeyboardSnapshot(params: params)
+ try Input.typeText(try requiredString(params, "text"), pid: snapshot.app.pid)
+ return actionMetadata(path: "synthetic")
+ }
+
+ private func pressKey(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentKeyboardSnapshot(params: params)
+ try Input.pressKey(try requiredString(params, "key"), pid: snapshot.app.pid)
+ return actionMetadata(path: "synthetic")
+ }
+
+ private func hotkey(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentKeyboardSnapshot(params: params)
+ let key = try requiredString(params, "key")
+ if isSelectAllHotkey(key), let focused = focusedRecord(snapshot), TextInput.selectAll(focused.element) {
+ return actionMetadata(
+ path: "accessibility",
+ actionName: "AXSelectAll",
+ verification: TextInput.selectionVerification(focused.element)
+ )
+ }
+ try Input.pressKey(key, pid: snapshot.app.pid)
+ return actionMetadata(
+ path: "synthetic",
+ actionName: "hotkey",
+ verification: unverifiedAction(reason: "synthetic_input")
+ )
+ }
+
+ private func pasteText(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentKeyboardSnapshot(params: params)
+ let text = try requiredString(params, "text")
+ if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {
+ return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
+ }
+ try Input.pasteText(text, pid: snapshot.app.pid)
+ return actionMetadata(
+ path: "clipboard",
+ actionName: "paste",
+ verification: unverifiedAction(reason: "clipboard_paste")
+ )
+ }
+
+ private func scroll(params: [String: JSONValue]) throws -> [String: Any] {
+ let snapshot = try currentSnapshot(params: params)
+ let direction = try requiredString(params, "direction")
+ let pages = params["pages"]?.number ?? 1
+ if let elementIndex = params["elementIndex"]?.number {
+ let record = try element(snapshot, Int(elementIndex))
+ let action = "AXScroll\(direction.capitalized)ByPage"
+ if pages.rounded() == pages, record.actions.contains(action) {
+ for _ in 0.. [String: Any] {
+ let snapshot = try currentSnapshot(params: params)
+ let start: CGPoint
+ let end: CGPoint
+ if let fromIndex = params["fromElementIndex"]?.number, let toIndex = params["toElementIndex"]?.number {
+ let from = try element(snapshot, Int(fromIndex))
+ let to = try element(snapshot, Int(toIndex))
+ guard let fromPoint = center(from.localFrame, in: snapshot.windowBounds),
+ let toPoint = center(to.localFrame, in: snapshot.windowBounds)
+ else {
+ throw ProviderError.coded("element_not_found", "drag element has no frame")
+ }
+ start = fromPoint
+ end = toPoint
+ } else {
+ start = try coordinatePoint(params: params, xKey: "fromX", yKey: "fromY", snapshot: snapshot)
+ end = try coordinatePoint(params: params, xKey: "toX", yKey: "toY", snapshot: snapshot)
+ }
+ try Input.drag(pid: snapshot.app.pid, from: start, to: end)
+ return actionMetadata(path: "synthetic")
+ }
+
+ private func element(_ snapshot: Snapshot, _ index: Int) throws -> ElementRecord {
+ guard let record = snapshot.elements[index] else {
+ throw ProviderError.coded("element_not_found", "element \(index) is not in the current cached snapshot for \(snapshot.app.name); run get-app-state again and use a fresh element index")
+ }
+ return record
+ }
+
+ private func focusedRecord(_ snapshot: Snapshot) -> ElementRecord? {
+ guard let focusedElementId = snapshot.focusedElementId else {
+ return nil
+ }
+ return snapshot.elements[focusedElementId]
+ }
+}
+
+private let blockedBundleIds: Set = [
+ "com.1password.1password",
+ "com.1password.safari",
+ "com.bitwarden.desktop",
+ "com.dashlane.dashlanephonefinal",
+ "com.lastpass.LastPass",
+ "com.nordsec.nordpass",
+ "me.proton.pass.electron",
+ "me.proton.pass.catalyst",
+]
+
+private func requiredString(_ params: [String: JSONValue], _ key: String) throws -> String {
+ guard let value = params[key]?.string, !value.isEmpty else {
+ throw ProviderError.coded("invalid_argument", "missing \(key)")
+ }
+ return value
+}
+
+private func requiredStringAllowingEmpty(_ params: [String: JSONValue], _ key: String) throws -> String {
+ guard let value = params[key]?.string else {
+ throw ProviderError.coded("invalid_argument", "missing \(key)")
+ }
+ return value
+}
+
+private func requiredNumber(_ params: [String: JSONValue], _ key: String) throws -> Double {
+ guard let value = params[key]?.number, value.isFinite else {
+ throw ProviderError.coded("invalid_argument", "missing \(key)")
+ }
+ return value
+}
+
+private func parsePid(_ query: String) -> pid_t? {
+ guard query.hasPrefix("pid:") else { return nil }
+ guard let pid = Int32(query.dropFirst(4)), pid > 0 else { return nil }
+ return pid
+}
+
+private func matches(_ app: AppDescriptor, query: String) -> Bool {
+ app.name.caseInsensitiveCompare(query) == .orderedSame ||
+ app.bundleId?.caseInsensitiveCompare(query) == .orderedSame
+}
+
+private func pidIsLive(_ pid: pid_t) -> Bool {
+ kill(pid, 0) == 0
+}
+
+private func accessibilityTrusted() -> Bool {
+ AXIsProcessTrusted()
+}
+
+private func promptForAccessibility() -> Bool {
+ let options = ["AXTrustedCheckOptionPrompt": true] as CFDictionary
+ return AXIsProcessTrustedWithOptions(options)
+}
+
+private func screenCaptureTrusted() -> Bool {
+ CGPreflightScreenCaptureAccess()
+}
+
+private func requestScreenCaptureAccess() -> Bool {
+ CGRequestScreenCaptureAccess()
+}
+
+private func openAccessibilitySettings() {
+ openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility")
+}
+
+private func openScreenRecordingSettings() {
+ openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture")
+}
+
+private func openSystemSettings(_ value: String) {
+ guard let url = URL(string: value) else { return }
+ NSWorkspace.shared.open(url)
+}
+
+private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: AppDescriptor) {
+ guard app.needsManualAccessibilityMode else {
+ return
+ }
+ _ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
+ _ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
+}
+
+private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement {
+ let systemWide = AXUIElementCreateSystemWide()
+ if let window = focusedSystemWindow(systemWide: systemWide, app: app) {
+ return window
+ }
+ if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) {
+ return window
+ }
+ if let windows = copyArray(appElement, kAXWindowsAttribute as String) {
+ if let window = windows.first(where: usableWindow) {
+ return window
+ }
+ }
+ if allowRecovery {
+ recoverWindow(app)
+ if let window = focusedSystemWindow(systemWide: systemWide, app: app) {
+ return window
+ }
+ if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) {
+ return window
+ }
+ if let windows = copyArray(appElement, kAXWindowsAttribute as String) {
+ if let window = windows.first(where: usableWindow) {
+ return window
+ }
+ }
+ }
+ let permissionHint = visibleWindowCount > 0
+ ? " The app has visible windows, so macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings."
+ : ""
+ throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; make sure the app has a visible window, then retry with --restore-window.\(permissionHint)")
+}
+
+private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) -> AXUIElement? {
+ guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String),
+ pidAttribute(focusedApp) == app.pid
+ else {
+ return nil
+ }
+ if let window = copyElement(systemWide, kAXFocusedWindowAttribute as String), usableWindow(window) {
+ return window
+ }
+ if let window = copyElement(focusedApp, kAXFocusedWindowAttribute as String), usableWindow(window) {
+ return window
+ }
+ if let windows = copyArray(focusedApp, kAXWindowsAttribute as String) {
+ return windows.first(where: usableWindow)
+ }
+ return nil
+}
+
+private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool {
+ guard let focusedWindow = focusedSystemWindow(systemWide: AXUIElementCreateSystemWide(), app: snapshot.app) else {
+ return false
+ }
+ if windowNumber(focusedWindow) == snapshot.windowId {
+ return true
+ }
+ guard let frame = absoluteFrame(focusedWindow) else {
+ return false
+ }
+ let intersection = frame.intersection(snapshot.windowBounds)
+ return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75
+}
+
+private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? {
+ guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else {
+ return nil
+ }
+ if let byNumber = windows.first(where: { windowNumber($0) == capture.windowId }) {
+ return byNumber
+ }
+ if let byBounds = windows.first(where: { window in
+ guard usableWindow(window), let frame = absoluteFrame(window) else { return false }
+ let intersection = frame.intersection(capture.bounds)
+ return !intersection.isNull && intersection.area >= min(frame.area, capture.bounds.area) * 0.75
+ }) {
+ return byBounds
+ }
+ if explicitTarget {
+ return nil
+ }
+ guard let titleHint = capture.title, !titleHint.isEmpty else {
+ return focused
+ }
+ return windows.first {
+ usableWindow($0) && stringAttribute($0, kAXTitleAttribute as String) == titleHint
+ } ?? focused
+}
+
+private func recoverWindow(_ app: AppDescriptor) {
+ _ = app.app.unhide()
+ _ = app.app.activate(options: [.activateAllWindows])
+ if let bundleId = app.bundleId {
+ openBundle(bundleId)
+ }
+ let appElement = AXUIElementCreateApplication(app.pid)
+ if let window = copyElement(appElement, kAXFocusedWindowAttribute as String) ?? copyArray(appElement, kAXWindowsAttribute as String)?.first {
+ _ = AXUIElementSetAttributeValue(window, kAXMinimizedAttribute as CFString, kCFBooleanFalse)
+ _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
+ _ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
+ _ = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
+ }
+ Thread.sleep(forTimeInterval: 0.4)
+}
+
+private func openBundle(_ bundleId: String) {
+ let process = Process()
+ process.executableURL = URL(fileURLWithPath: "/usr/bin/open")
+ process.arguments = ["-b", bundleId]
+ process.standardOutput = Pipe()
+ process.standardError = Pipe()
+ try? process.run()
+ process.waitUntilExit()
+}
+
+private func requestedWindowId(_ params: [String: JSONValue]) -> CGWindowID? {
+ guard let value = params["windowId"]?.number, value >= 0 else { return nil }
+ return CGWindowID(UInt32(value))
+}
+
+private func snapshotWindowKey(_ query: String, _ windowId: CGWindowID) -> String {
+ "\(query.lowercased())#window:\(Int(windowId))"
+}
+
+private func snapshotWindowIndexKey(_ query: String, _ windowIndex: Int) -> String {
+ "\(query.lowercased())#windowIndex:\(windowIndex)"
+}
+
+private func snapshotNamespace(_ params: [String: JSONValue]) -> String {
+ if let session = params["session"]?.string, !session.isEmpty {
+ return "session:\(session)"
+ }
+ if let worktree = params["worktree"]?.string, !worktree.isEmpty {
+ return "worktree:\(worktree)"
+ }
+ return "default"
+}
+
+private func namespacedSnapshotKey(_ namespace: String, _ key: String) -> String {
+ "\(namespace):\(key.lowercased())"
+}
+
+private func isExplicitSnapshotNamespace(_ namespace: String) -> Bool {
+ namespace != "default"
+}
+
+private func requestedWindowIndex(_ params: [String: JSONValue]) -> Int? {
+ guard let value = params["windowIndex"]?.number, value >= 0 else { return nil }
+ return Int(value)
+}
+
+private func usableWindow(_ element: AXUIElement) -> Bool {
+ stringAttribute(element, kAXRoleAttribute as String) == kAXWindowRole as String &&
+ boolAttribute(element, kAXMinimizedAttribute as String) != true
+}
+
+private func focusedElement(appElement: AXUIElement) -> AXUIElement? {
+ copyElement(appElement, kAXFocusedUIElementAttribute as String)
+}
+
+private func copyElement(_ element: AXUIElement, _ attribute: String) -> AXUIElement? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
+ return nil
+ }
+ return (value as! AXUIElement)
+}
+
+private func copyArray(_ element: AXUIElement, _ attribute: String) -> [AXUIElement]? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
+ return nil
+ }
+ return value as? [AXUIElement]
+}
+
+private func pidAttribute(_ element: AXUIElement) -> pid_t? {
+ var pid: pid_t = 0
+ guard AXUIElementGetPid(element, &pid) == .success else {
+ return nil
+ }
+ return pid
+}
+
+private func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {
+ return nil
+ }
+ if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String {
+ let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines)
+ return trimmed.isEmpty ? nil : trimmed
+ }
+ if CFGetTypeID(value) == CFURLGetTypeID(), let url = value as? URL {
+ return url.absoluteString
+ }
+ return nil
+}
+
+private func rawStringAttribute(_ element: AXUIElement, _ attribute: String) -> String? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+ let value,
+ CFGetTypeID(value) == CFStringGetTypeID()
+ else {
+ return nil
+ }
+ return value as? String
+}
+
+private func boolAttribute(_ element: AXUIElement, _ attribute: String) -> Bool? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
+ return nil
+ }
+ return value as? Bool
+}
+
+private func numberAttribute(_ element: AXUIElement, _ attribute: String) -> NSNumber? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
+ return nil
+ }
+ return value as? NSNumber
+}
+
+private func windowNumber(_ element: AXUIElement) -> CGWindowID? {
+ guard let number = numberAttribute(element, "AXWindowNumber") else {
+ return nil
+ }
+ return CGWindowID(number.uint32Value)
+}
+
+private func absoluteFrame(_ element: AXUIElement) -> CGRect? {
+ var positionValue: CFTypeRef?
+ var sizeValue: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
+ AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
+ let positionValue,
+ let sizeValue
+ else {
+ return nil
+ }
+ var point = CGPoint.zero
+ var size = CGSize.zero
+ guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point),
+ AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
+ else {
+ return nil
+ }
+ return CGRect(origin: point, size: size)
+}
+
+private extension CGRect {
+ var area: CGFloat {
+ max(width, 0) * max(height, 0)
+ }
+}
+
+private func actions(_ element: AXUIElement) -> [String] {
+ var value: CFArray?
+ guard AXUIElementCopyActionNames(element, &value) == .success, let value else {
+ return []
+ }
+ return value as? [String] ?? []
+}
+
+private func performAction(_ element: AXUIElement, _ action: String) -> Bool {
+ actions(element).contains(where: { $0.caseInsensitiveCompare(action) == .orderedSame }) &&
+ AXUIElementPerformAction(element, action as CFString) == .success
+}
+
+private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
+ var settable = DarwinBoolean(false)
+ return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
+}
+
+private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {
+ var positionValue: CFTypeRef?
+ var sizeValue: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
+ AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
+ let positionValue,
+ let sizeValue
+ else {
+ return nil
+ }
+ var point = CGPoint.zero
+ var size = CGSize.zero
+ guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &point),
+ AXValueGetValue(sizeValue as! AXValue, .cgSize, &size)
+ else {
+ return nil
+ }
+ return CGRect(x: point.x - windowBounds.minX, y: point.y - windowBounds.minY, width: size.width, height: size.height)
+}
+
+private func elementSignature(_ node: SnapshotRenderNode) -> String {
+ [
+ node.role,
+ node.roleDescription ?? "",
+ node.title ?? "",
+ node.label ?? "",
+ node.linkText ?? "",
+ node.value ?? "",
+ node.placeholder ?? "",
+ node.url ?? "",
+ node.summary ?? "",
+ node.rowSummary ?? "",
+ SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role).joined(separator: ","),
+ ].joined(separator: "\u{1f}")
+}
+
+private func center(_ localFrame: CGRect?, in windowBounds: CGRect) -> CGPoint? {
+ guard let localFrame else { return nil }
+ return CGPoint(x: windowBounds.minX + localFrame.midX, y: windowBounds.minY + localFrame.midY)
+}
+
+private func screenIndex(for bounds: CGRect) -> Int? {
+ guard let index = NSScreen.screens.firstIndex(where: { $0.frame.intersects(bounds) }) else {
+ return nil
+ }
+ return index
+}
+
+private func coordinatePoint(params: [String: JSONValue], xKey: String, yKey: String, snapshot: Snapshot) throws -> CGPoint {
+ let x = try requiredNumber(params, xKey)
+ let y = try requiredNumber(params, yKey)
+ return CGPoint(
+ x: snapshot.windowBounds.minX + x,
+ y: snapshot.windowBounds.minY + y
+ )
+}
+
+private func screenshotScale(screenshot: ScreenshotPayload?, bounds: CGRect) -> CGSize {
+ guard let screenshot, bounds.width > 0, bounds.height > 0 else {
+ return CGSize(width: 1, height: 1)
+ }
+ return CGSize(
+ width: CGFloat(screenshot.width) / bounds.width,
+ height: CGFloat(screenshot.height) / bounds.height
+ )
+}
+
+private enum MouseButton {
+ case left
+ case right
+
+ var cgButton: CGMouseButton {
+ switch self {
+ case .left:
+ return .left
+ case .right:
+ return .right
+ }
+ }
+
+ var downEvent: CGEventType {
+ switch self {
+ case .left:
+ return .leftMouseDown
+ case .right:
+ return .rightMouseDown
+ }
+ }
+
+ var upEvent: CGEventType {
+ switch self {
+ case .left:
+ return .leftMouseUp
+ case .right:
+ return .rightMouseUp
+ }
+ }
+}
+
+private func mouseButton(_ raw: String?) throws -> MouseButton {
+ switch raw ?? "left" {
+ case "left":
+ return .left
+ case "right":
+ return .right
+ case "middle":
+ throw ProviderError.coded("invalid_argument", "middle-click is not yet supported")
+ case let value:
+ throw ProviderError.coded("invalid_argument", "unsupported mouse button '\(value)'")
+ }
+}
+
+private func renderTreeText(app: AppDescriptor, title: String, bounds: CGRect, lines: [String], focused: String?) -> String {
+ var output = [
+ "App=\(app.bundleId ?? app.name.replacingOccurrences(of: " ", with: "_")) (pid \(app.pid))",
+ "Window: \"\(sanitize(title))\", App: \(sanitize(app.name)).",
+ "",
+ ]
+ output.append(contentsOf: lines)
+ output.append("")
+ output.append(focused.map { "The focused UI element is \($0)." } ?? "No UI element is currently focused.")
+ return output.joined(separator: "\n")
+}
+
+private func renderScreenshotStatus(_ status: ScreenshotStatus, snapshot: Snapshot) -> [String: Any] {
+ let metadata: [String: Any] = [
+ "engine": snapshot.screenshotEngine ?? "unknown",
+ "windowId": Int(snapshot.windowId),
+ ]
+ switch status {
+ case .captured:
+ return ["state": "captured", "metadata": metadata]
+ case .skipped:
+ return ["state": "skipped", "reason": "no_screenshot_flag"]
+ case let .failed(message):
+ return ["state": "failed", "code": "screenshot_failed", "message": message, "metadata": metadata]
+ }
+}
+
+private final class TreeRenderer {
+ let windowBounds: CGRect
+ let focused: AXUIElement?
+ var lines: [String] = []
+ var records: [Int: ElementRecord] = [:]
+ var focusedSummary: String?
+ var focusedElementId: Int?
+ var truncated = false
+ var maxDepthReached = false
+ private var nextIndex = 0
+ static let maxNodes = 1200
+ static let maxDepth = 64
+
+ init(windowBounds: CGRect, focused: AXUIElement?) {
+ self.windowBounds = windowBounds
+ self.focused = focused
+ }
+
+ func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) {
+ guard nextIndex < Self.maxNodes else {
+ truncated = true
+ return
+ }
+ guard depth < Self.maxDepth else {
+ truncated = true
+ maxDepthReached = true
+ return
+ }
+ guard !ancestors.contains(where: { CFEqual($0, element) }) else { return }
+
+ let role = stringAttribute(element, kAXRoleAttribute as String) ?? "AXUnknown"
+ let children = primaryChildren(element, role: role, windowBounds: windowBounds)
+ let value = valueString(element)
+ let placeholder = placeholderString(element)
+ let rawActions = actions(element)
+ let rowSummary = rowTextSummary(element, role: role)
+ let linkText = role == "AXLink" ? descendantTextSnippets(element, limit: 2, maxDepth: 3).first : nil
+ let baseNode = SnapshotRenderNode(
+ role: role,
+ roleDescription: stringAttribute(element, kAXRoleDescriptionAttribute as String),
+ title: stringAttribute(element, kAXTitleAttribute as String),
+ label: stringAttribute(element, kAXDescriptionAttribute as String),
+ linkText: linkText,
+ value: value,
+ placeholder: placeholder,
+ url: stringAttribute(element, kAXURLAttribute as String),
+ traits: [],
+ rawActions: rawActions,
+ childCount: children.count,
+ rowSummary: rowSummary
+ )
+ let name = SnapshotRenderHeuristics.displayName(baseNode)
+ let meaningful = SnapshotRenderHeuristics.meaningfulActions(rawActions, role: role)
+ let localFrame = frame(element, windowBounds: windowBounds)
+ let traits = traitsFor(element, role: role)
+ let webAreaDepth = webAreaDepth(role: role, ancestors: ancestors)
+ let summary = genericTextSummary(element, role: role, name: name, actions: meaningful, traits: traits)
+ let node = SnapshotRenderNode(
+ role: role,
+ roleDescription: stringAttribute(element, kAXRoleDescriptionAttribute as String),
+ title: stringAttribute(element, kAXTitleAttribute as String),
+ label: stringAttribute(element, kAXDescriptionAttribute as String),
+ linkText: linkText,
+ value: value,
+ placeholder: placeholder,
+ url: stringAttribute(element, kAXURLAttribute as String),
+ traits: traits,
+ rawActions: rawActions,
+ childCount: children.count,
+ summary: summary,
+ rowSummary: rowSummary,
+ webAreaDepth: webAreaDepth
+ )
+ if SnapshotRenderHeuristics.shouldElide(node) {
+ for child in children {
+ render(child, depth: depth, ancestors: ancestors + [element])
+ }
+ return
+ }
+
+ let index = nextIndex
+ nextIndex += 1
+ let line = SnapshotRenderHeuristics.line(index: index, node: node)
+ lines.append(String(repeating: "\t", count: depth) + line)
+ records[index] = ElementRecord(
+ index: index,
+ element: element,
+ localFrame: localFrame,
+ actions: rawActions,
+ signature: elementSignature(node)
+ )
+ if let focused, CFEqual(focused, element) {
+ focusedElementId = index
+ focusedSummary = line
+ }
+ if summary != nil || SnapshotRenderHeuristics.shouldSuppressChildren(node) {
+ return
+ }
+ for child in children {
+ render(child, depth: depth + 1, ancestors: ancestors + [element])
+ }
+ }
+}
+
+private func valueString(_ element: AXUIElement) -> String? {
+ if isSecureTextElement(element) {
+ return "[redacted]"
+ }
+ if let string = stringAttribute(element, kAXValueAttribute as String) {
+ return string
+ }
+ if let number = numberAttribute(element, kAXValueAttribute as String) {
+ return number.stringValue
+ }
+ return nil
+}
+
+private func isSecureTextElement(_ element: AXUIElement) -> Bool {
+ let role = stringAttribute(element, kAXRoleAttribute as String)?.lowercased() ?? ""
+ let subrole = stringAttribute(element, kAXSubroleAttribute as String)?.lowercased() ?? ""
+ let title = stringAttribute(element, kAXTitleAttribute as String)?.lowercased() ?? ""
+ let description = stringAttribute(element, kAXDescriptionAttribute as String)?.lowercased() ?? ""
+ let placeholder = placeholderString(element)?.lowercased() ?? ""
+ let haystack = [role, subrole, title, description, placeholder].joined(separator: " ")
+ return haystack.contains("secure") ||
+ haystack.contains("password") ||
+ haystack.contains("passcode") ||
+ haystack.contains("verification code") ||
+ haystack.contains("one-time code")
+}
+
+private func placeholderString(_ element: AXUIElement) -> String? {
+ stringAttribute(element, "AXPlaceholderValue") ?? stringAttribute(element, "AXPlaceholder")
+}
+
+private func traitsFor(_ element: AXUIElement, role: String) -> [String] {
+ var traits: [String] = []
+ if boolAttribute(element, kAXSelectedAttribute as String) == true { traits.append("selected") }
+ if boolAttribute(element, kAXExpandedAttribute as String) == true { traits.append("expanded") }
+ if boolAttribute(element, kAXEnabledAttribute as String) == false { traits.append("disabled") }
+ if valueSettableRoles.contains(role), isSettable(element, kAXValueAttribute as String) { traits.append("settable") }
+ return traits
+}
+
+private let valueSettableRoles: Set = [
+ kAXCheckBoxRole as String,
+ kAXComboBoxRole as String,
+ kAXRadioButtonRole as String,
+ "AXSearchField",
+ kAXSliderRole as String,
+ kAXTextAreaRole as String,
+ kAXTextFieldRole as String,
+]
+
+private func primaryChildren(_ element: AXUIElement, role: String, windowBounds: CGRect) -> [AXUIElement] {
+ if usesRowsAsPrimaryChildren(role: role), let rows = copyArray(element, kAXRowsAttribute as String), !rows.isEmpty {
+ return visibleRows(rows, parent: element, windowBounds: windowBounds)
+ }
+ return copyArray(element, kAXChildrenAttribute as String) ?? []
+}
+
+private func visibleRows(_ rows: [AXUIElement], parent: AXUIElement, windowBounds: CGRect) -> [AXUIElement] {
+ guard let parentFrame = frame(parent, windowBounds: windowBounds) else {
+ return Array(rows.prefix(20))
+ }
+ let visible = rows.filter { row in
+ guard let rowFrame = frame(row, windowBounds: windowBounds) else { return false }
+ return rowFrame.intersects(parentFrame)
+ }
+ return Array((visible.isEmpty ? rows : visible).prefix(20))
+}
+
+private func usesRowsAsPrimaryChildren(role: String) -> Bool {
+ [
+ kAXBrowserRole as String,
+ kAXListRole as String,
+ kAXOutlineRole as String,
+ kAXTableRole as String,
+ ].contains(role)
+}
+
+private func genericTextSummary(
+ _ element: AXUIElement,
+ role: String,
+ name: String?,
+ actions: [String],
+ traits: [String]
+) -> String? {
+ guard (role == kAXGroupRole as String || role == kAXUnknownRole as String),
+ name == nil,
+ actions.isEmpty,
+ traits.isEmpty,
+ isPlainTextSubtree(element, maxDepth: 4)
+ else {
+ return nil
+ }
+ let texts = descendantTextSnippets(element, limit: 8, maxDepth: 4)
+ guard texts.count >= 2 else { return nil }
+ let summary = texts.joined(separator: " ")
+ guard summary.count <= 220 else { return nil }
+ return summary
+}
+
+private func rowTextSummary(_ element: AXUIElement, role: String) -> String? {
+ guard ["AXRow", "AXCell", "AXOutlineRow"].contains(role) else { return nil }
+ let texts = descendantTextSnippets(element, limit: 6, maxDepth: 3)
+ guard !texts.isEmpty else { return nil }
+ return texts.joined(separator: " ")
+}
+
+private func isPlainTextSubtree(_ element: AXUIElement, maxDepth: Int) -> Bool {
+ var sawText = false
+ let allowedContainerRoles: Set = [
+ kAXGroupRole as String,
+ kAXUnknownRole as String,
+ kAXStaticTextRole as String,
+ "AXLink",
+ "AXImage",
+ ]
+
+ func visit(_ node: AXUIElement, depth: Int) -> Bool {
+ guard depth <= maxDepth else { return false }
+ let role = stringAttribute(node, kAXRoleAttribute as String) ?? "AXUnknown"
+ guard allowedContainerRoles.contains(role) else { return false }
+ if role == kAXStaticTextRole as String || role == "AXLink" {
+ sawText = true
+ }
+ guard SnapshotRenderHeuristics.meaningfulActions(actions(node), role: role).isEmpty else { return false }
+ for child in copyArray(node, kAXChildrenAttribute as String) ?? [] {
+ guard visit(child, depth: depth + 1) else { return false }
+ }
+ return true
+ }
+
+ return visit(element, depth: 0) && sawText
+}
+
+private func descendantTextSnippets(_ element: AXUIElement, limit: Int, maxDepth: Int) -> [String] {
+ var values: [String] = []
+ var seen = Set()
+
+ func collect(_ node: AXUIElement, depth: Int) {
+ guard values.count < limit, depth <= maxDepth else { return }
+ let role = stringAttribute(node, kAXRoleAttribute as String) ?? ""
+ if role == kAXStaticTextRole as String || role == "AXLink" {
+ for candidate in [
+ stringAttribute(node, kAXValueAttribute as String),
+ stringAttribute(node, kAXTitleAttribute as String),
+ stringAttribute(node, kAXDescriptionAttribute as String),
+ ] {
+ guard let candidate else { continue }
+ let text = preview(candidate, maxLength: 80)
+ guard !text.isEmpty, seen.insert(text).inserted else { continue }
+ values.append(text)
+ if values.count >= limit { return }
+ }
+ }
+ for child in copyArray(node, kAXChildrenAttribute as String) ?? [] {
+ collect(child, depth: depth + 1)
+ if values.count >= limit { return }
+ }
+ }
+
+ collect(element, depth: 0)
+ return values
+}
+
+private func webAreaDepth(role: String, ancestors: [AXUIElement]) -> Int? {
+ if role == "AXWebArea" { return 0 }
+ guard let index = ancestors.firstIndex(where: { stringAttribute($0, kAXRoleAttribute as String) == "AXWebArea" }) else {
+ return nil
+ }
+ return ancestors.count - index
+}
+
+private func sanitize(_ value: String) -> String {
+ value.replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\r", with: " ")
+}
+
+private func preview(_ value: String, maxLength: Int = 120) -> String {
+ let clean = sanitize(value)
+ if clean.count <= maxLength {
+ return clean
+ }
+ return String(clean.prefix(maxLength)) + "..."
+}
+
+private func actionMetadata(
+ path: String,
+ actionName: String? = nil,
+ fallbackReason: String? = nil,
+ verification: [String: Any]? = nil
+) -> [String: Any] {
+ var metadata: [String: Any] = [
+ "path": path,
+ "actionName": jsonNullable(actionName),
+ "fallbackReason": jsonNullable(fallbackReason),
+ ]
+ if let verification {
+ metadata["verification"] = verification
+ }
+ return metadata
+}
+
+private func verifiedAction(property: String, expected: String? = nil, actualPreview: String? = nil) -> [String: Any] {
+ [
+ "state": "verified",
+ "property": property,
+ "expected": jsonNullable(expected),
+ "actualPreview": jsonNullable(actualPreview),
+ ]
+}
+
+private func unverifiedAction(reason: String) -> [String: Any] {
+ [
+ "state": "unverified",
+ "reason": reason,
+ ]
+}
+
+private func jsonNullable(_ value: T?) -> Any {
+ value ?? NSNull()
+}
+
+private struct WindowCandidate {
+ let windowId: CGWindowID
+ let layer: Int
+ let bounds: CGRect
+ let title: String?
+ let alpha: CGFloat
+ let isOnScreen: Bool
+ let sharingState: Int?
+
+ var score: Int {
+ var value = Int(bounds.width * bounds.height)
+ if layer == 0 { value += 1_000_000_000 }
+ if title != nil && title?.isEmpty == false { value += 10_000_000 }
+ if isOnScreen { value += 1_000_000 }
+ if alpha >= 0.99 { value += 100_000 }
+ return value
+ }
+}
+
+private struct WindowCapture {
+ let windowId: CGWindowID
+ let layer: Int
+ let bounds: CGRect
+ let title: String?
+ let image: CapturedImage?
+
+ static func resolve(pid: pid_t, titleHint: String?, windowId: CGWindowID?, windowIndex: Int?) -> WindowCapture? {
+ resolve(candidates: candidates(pid: pid), titleHint: titleHint, windowId: windowId, windowIndex: windowIndex)
+ }
+
+ static func resolve(candidates: [WindowCandidate], titleHint: String?, windowId: CGWindowID?, windowIndex: Int?) -> WindowCapture? {
+ if let windowId {
+ guard let candidate = candidates.first(where: { $0.windowId == windowId }) else { return nil }
+ return WindowCapture(candidate: candidate)
+ }
+ if let windowIndex {
+ let visibleWindows = candidates.filter { $0.layer == 0 }
+ guard visibleWindows.indices.contains(windowIndex) else { return nil }
+ return WindowCapture(candidate: visibleWindows[windowIndex])
+ }
+ guard let best = candidates.sorted(by: { lhs, rhs in
+ if let titleHint, lhs.title == titleHint, rhs.title != titleHint { return true }
+ if let titleHint, rhs.title == titleHint, lhs.title != titleHint { return false }
+ return lhs.score > rhs.score
+ }).first else {
+ return nil
+ }
+ return WindowCapture(candidate: best)
+ }
+
+ private init(candidate: WindowCandidate) {
+ self.windowId = candidate.windowId
+ self.layer = candidate.layer
+ self.bounds = candidate.bounds
+ self.title = candidate.title
+ self.image = Self.captureImage(windowId: candidate.windowId, bounds: candidate.bounds)
+ }
+
+ static func candidates(pid: pid_t) -> [WindowCandidate] {
+ guard let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else {
+ return []
+ }
+ return infos.compactMap { info in
+ guard let ownerPid = info[kCGWindowOwnerPID as String] as? pid_t, ownerPid == pid,
+ let number = info[kCGWindowNumber as String] as? NSNumber,
+ let layer = info[kCGWindowLayer as String] as? Int,
+ let boundsDictionary = info[kCGWindowBounds as String] as? NSDictionary,
+ let bounds = CGRect(dictionaryRepresentation: boundsDictionary),
+ bounds.width >= 48,
+ bounds.height >= 48
+ else {
+ return nil
+ }
+ let alpha = info[kCGWindowAlpha as String] as? CGFloat ?? 1
+ guard alpha > 0.01 else { return nil }
+ let sharing = (info[kCGWindowSharingState as String] as? NSNumber).map { $0.intValue }
+ if sharing == 0 { return nil }
+ let isOnScreen = (info[kCGWindowIsOnscreen as String] as? Bool) ?? true
+ return WindowCandidate(
+ windowId: CGWindowID(number.uint32Value),
+ layer: layer,
+ bounds: bounds,
+ title: info[kCGWindowName as String] as? String,
+ alpha: alpha,
+ isOnScreen: isOnScreen,
+ sharingState: sharing
+ )
+ }
+ .sorted { lhs, rhs in
+ lhs.score > rhs.score
+ }
+ }
+
+ func screenshotPayload() -> ScreenshotPayload? {
+ guard let image, let bounded = boundedPngData(image.image) else { return nil }
+ return ScreenshotPayload(
+ data: bounded.data.base64EncodedString(),
+ width: bounded.width,
+ height: bounded.height,
+ scale: Double(bounded.width) / max(Double(bounds.width), 1)
+ )
+ }
+
+ private static func captureImage(windowId: CGWindowID, bounds: CGRect) -> CapturedImage? {
+ if ProcessInfo.processInfo.environment["ORCA_COMPUTER_USE_SCK_SCREENSHOTS"] == "1",
+ let image = captureImageWithScreenCaptureKit(windowId: windowId, bounds: bounds) {
+ return CapturedImage(image: image, engine: "screenCaptureKit")
+ }
+ if let image = CGWindowListCreateImage(.null, [.optionIncludingWindow], windowId, [.boundsIgnoreFraming, .bestResolution]) {
+ return CapturedImage(image: image, engine: "cgWindowList")
+ }
+ return nil
+ }
+
+ private static func captureImageWithScreenCaptureKit(windowId: CGWindowID, bounds: CGRect) -> CGImage? {
+ try? BlockingAsync.run(timeout: 3) {
+ let content = try await SCShareableContent.current
+ guard let window = content.windows.first(where: { $0.windowID == windowId }) else {
+ return nil
+ }
+ let filter = SCContentFilter(desktopIndependentWindow: window)
+ let configuration = SCStreamConfiguration()
+ let scale = NSScreen.screens.first(where: { $0.frame.intersects(bounds) })?.backingScaleFactor ?? NSScreen.main?.backingScaleFactor ?? 2
+ configuration.width = size_t(max(1, Int((bounds.width * scale).rounded(.up))))
+ configuration.height = size_t(max(1, Int((bounds.height * scale).rounded(.up))))
+ configuration.scalesToFit = true
+ configuration.preservesAspectRatio = true
+ configuration.showsCursor = false
+ configuration.ignoreShadowsSingleWindow = true
+ configuration.ignoreGlobalClipSingleWindow = true
+ return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
+ }
+ }
+}
+
+private final class AsyncBox: @unchecked Sendable {
+ var result: Result?
+}
+
+private enum BlockingAsync {
+ static func run(timeout: TimeInterval, operation: @escaping @Sendable () async throws -> T) throws -> T {
+ let semaphore = DispatchSemaphore(value: 0)
+ let box = AsyncBox()
+ let task = Task.detached {
+ do {
+ box.result = .success(try await operation())
+ } catch {
+ box.result = .failure(error)
+ }
+ semaphore.signal()
+ }
+ guard semaphore.wait(timeout: .now() + timeout) == .success else {
+ task.cancel()
+ throw ProviderError.coded("action_timeout", "screenshot capture timed out")
+ }
+ return try box.result!.get()
+ }
+}
+
+private struct BoundedPNG {
+ let data: Data
+ let width: Int
+ let height: Int
+}
+
+private func boundedPngData(_ image: CGImage) -> BoundedPNG? {
+ let rep = NSBitmapImageRep(cgImage: image)
+ guard let data = rep.representation(using: .png, properties: [:]) else { return nil }
+ if data.count <= 900_000 {
+ return BoundedPNG(data: data, width: image.width, height: image.height)
+ }
+ var scale = min(1, 1280 / CGFloat(max(image.width, image.height)))
+ var best = BoundedPNG(data: data, width: image.width, height: image.height)
+ while scale >= 0.25 {
+ guard let resized = resizePng(image, scale: scale) else {
+ break
+ }
+ best = resized
+ if resized.data.count <= 900_000 {
+ return resized
+ }
+ scale *= 0.85
+ }
+ return best
+}
+
+private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? {
+ let width = max(1, Int(CGFloat(image.width) * scale))
+ let height = max(1, Int(CGFloat(image.height) * scale))
+ guard let context = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {
+ return nil
+ }
+ context.interpolationQuality = .medium
+ context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
+ guard let resized = context.makeImage() else { return nil }
+ guard let data = NSBitmapImageRep(cgImage: resized).representation(using: .png, properties: [:]) else {
+ return nil
+ }
+ return BoundedPNG(data: data, width: width, height: height)
+}
+
+private enum Input {
+ static func click(pid: pid_t, at point: CGPoint, button: MouseButton, count: Int) throws {
+ guard let source = CGEventSource(stateID: .combinedSessionState) else {
+ throw ProviderError.coded("accessibility_error", "failed to create event source")
+ }
+ for _ in 0.. [String: Any]? {
+ guard isSettable(element, kAXValueAttribute as String),
+ let current = rawStringAttribute(element, kAXValueAttribute as String)
+ else {
+ return nil
+ }
+ let selectedRange = selectedTextRange(element) ?? CFRange(location: current.utf16.count, length: 0)
+ let startOffset = max(0, min(selectedRange.location, current.utf16.count))
+ let endOffset = max(startOffset, min(startOffset + selectedRange.length, current.utf16.count))
+ let start = String.Index(utf16Offset: startOffset, in: current)
+ let end = String.Index(utf16Offset: endOffset, in: current)
+ let next = String(current[.. Bool {
+ guard let current = rawStringAttribute(element, kAXValueAttribute as String) else {
+ return false
+ }
+ return setSelectedTextRange(element, CFRange(location: 0, length: current.utf16.count))
+ }
+
+ static func selectionVerification(_ element: AXUIElement) -> [String: Any] {
+ guard let current = rawStringAttribute(element, kAXValueAttribute as String),
+ let selectedRange = selectedTextRange(element),
+ selectedRange.location == 0,
+ selectedRange.length == current.utf16.count
+ else {
+ return unverifiedAction(reason: "provider_unavailable")
+ }
+ return verifiedAction(property: "selection", actualPreview: preview(current))
+ }
+
+ private static func selectedTextRange(_ element: AXUIElement) -> CFRange? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &value) == .success,
+ let value,
+ CFGetTypeID(value) == AXValueGetTypeID()
+ else {
+ return nil
+ }
+ var range = CFRange(location: 0, length: 0)
+ guard AXValueGetValue(value as! AXValue, .cfRange, &range) else {
+ return nil
+ }
+ return range
+ }
+
+ @discardableResult
+ private static func setSelectedTextRange(_ element: AXUIElement, _ range: CFRange) -> Bool {
+ var mutableRange = range
+ guard let value = AXValueCreate(.cfRange, &mutableRange) else {
+ return false
+ }
+ return AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, value) == .success
+ }
+}
+
+private func isSelectAllHotkey(_ key: String) -> Bool {
+ let parts = key
+ .lowercased()
+ .replacingOccurrences(of: " ", with: "")
+ .replacingOccurrences(of: "-", with: "+")
+ .split(separator: "+")
+ .map(String.init)
+ guard parts.last == "a" else {
+ return false
+ }
+ let modifiers = Set(parts.dropLast())
+ return modifiers.contains("cmd") ||
+ modifiers.contains("command") ||
+ modifiers.contains("meta") ||
+ modifiers.contains("cmdorctrl") ||
+ modifiers.contains("commandorcontrol")
+}
+
+private struct KeyModifier {
+ let keyCode: CGKeyCode
+ let flag: CGEventFlags
+}
+
+private struct ParsedKey {
+ let keyCode: CGKeyCode
+ let modifiers: [KeyModifier]
+}
+
+private enum KeyMap {
+ static func parse(_ spec: String) throws -> ParsedKey {
+ let parts = spec.split(separator: "+").map { String($0).lowercased() }
+ var modifiers: [KeyModifier] = []
+ var keyName: String?
+ for part in parts {
+ switch part {
+ case "cmd", "command", "meta", "super", "cmdorctrl", "commandorcontrol":
+ modifiers.append(KeyModifier(keyCode: 55, flag: .maskCommand))
+ case "ctrl", "control":
+ modifiers.append(KeyModifier(keyCode: 59, flag: .maskControl))
+ case "alt", "option":
+ modifiers.append(KeyModifier(keyCode: 58, flag: .maskAlternate))
+ case "shift":
+ modifiers.append(KeyModifier(keyCode: 56, flag: .maskShift))
+ default:
+ keyName = part
+ }
+ }
+ guard let keyName, let keyCode = codes[keyName] else {
+ throw ProviderError.coded("invalid_argument", "unsupported key '\(spec)'")
+ }
+ return ParsedKey(keyCode: keyCode, modifiers: modifiers)
+ }
+
+ private static let codes: [String: CGKeyCode] = [
+ "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9,
+ "b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19,
+ "3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25, "7": 26, "-": 27, "8": 28,
+ "0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36,
+ "enter": 36, "l": 37, "j": 38, "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43,
+ "/": 44, "n": 45, "m": 46, ".": 47, "tab": 48, "space": 49, "`": 50,
+ "backspace": 51, "delete": 51, "escape": 53, "esc": 53, "left": 123, "right": 124,
+ "down": 125, "up": 126,
+ ]
+}
+
+private final class AgentRuntime: NSObject, NSApplicationDelegate {
+ private let socketPath: String
+ private let token: String?
+ private var listener: SocketListener?
+
+ init(socketPath: String, token: String?) {
+ self.socketPath = socketPath
+ self.token = token
+ }
+
+ func applicationDidFinishLaunching(_ notification: Notification) {
+ do {
+ let listener = try SocketListener(socketPath: socketPath, token: token)
+ self.listener = listener
+ listener.start()
+ } catch {
+ fputs("failed to start computer-use socket: \(error)\n", stderr)
+ NSApp.terminate(nil)
+ }
+ }
+
+ func applicationWillTerminate(_ notification: Notification) {
+ listener?.stop()
+ }
+}
+
+private final class PermissionRuntime: NSObject, NSApplicationDelegate {
+ private let initialPermission: PermissionKind?
+ private var windowController: PermissionWindowController?
+
+ init(initialPermission: PermissionKind?) {
+ self.initialPermission = initialPermission
+ }
+
+ func applicationDidFinishLaunching(_ notification: Notification) {
+ windowController = PermissionWindowController(
+ terminateWhenDragAssistantCloses: initialPermission != nil
+ )
+ if let initialPermission {
+ windowController?.openPermission(initialPermission)
+ } else {
+ windowController?.showWindow(nil)
+ windowController?.window?.makeKeyAndOrderFront(nil)
+ NSApp.activate(ignoringOtherApps: true)
+ }
+ }
+
+ func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
+ initialPermission == nil
+ }
+}
+
+private final class PermissionWindowController: NSWindowController {
+ private var dragAssistant: PermissionDragAssistantController?
+ private let terminateWhenDragAssistantCloses: Bool
+
+ convenience init(terminateWhenDragAssistantCloses: Bool = false) {
+ let window = NSWindow(
+ contentRect: NSRect(x: 0, y: 0, width: 300, height: 315),
+ styleMask: [.titled, .closable, .miniaturizable, .fullSizeContentView],
+ backing: .buffered,
+ defer: false
+ )
+ window.title = "Enable Orca Computer Use"
+ window.titleVisibility = .hidden
+ window.titlebarAppearsTransparent = true
+ window.backgroundColor = PermissionPalette.background
+ window.center()
+ window.isReleasedWhenClosed = false
+ self.init(window: window, terminateWhenDragAssistantCloses: terminateWhenDragAssistantCloses)
+ window.contentView = PermissionView(frame: window.contentView?.bounds ?? .zero) { [weak self] permission in
+ self?.showDragAssistant(for: permission)
+ }
+ }
+
+ init(window: NSWindow?, terminateWhenDragAssistantCloses: Bool) {
+ self.terminateWhenDragAssistantCloses = terminateWhenDragAssistantCloses
+ super.init(window: window)
+ }
+
+ required init?(coder: NSCoder) {
+ fatalError("init(coder:) has not been implemented")
+ }
+
+ private func showDragAssistant(for permission: PermissionKind) {
+ dragAssistant?.close()
+ dragAssistant = PermissionDragAssistantController(
+ permission: permission,
+ fallbackVisibleFrame: window?.screen?.visibleFrame,
+ onClose: { [weak self] in
+ if self?.terminateWhenDragAssistantCloses == true {
+ NSApp.terminate(nil)
+ }
+ }
+ )
+ dragAssistant?.showWhenReady()
+ }
+
+ func openPermission(_ permission: PermissionKind) {
+ permission.requestAndOpenSettings()
+ showDragAssistant(for: permission)
+ }
+}
+
+private enum PermissionKind {
+ case accessibility
+ case screenshots
+
+ static func parse(_ value: String?) -> PermissionKind? {
+ switch value {
+ case "accessibility":
+ .accessibility
+ case "screenshots", "screen", "screen-recording":
+ .screenshots
+ default:
+ nil
+ }
+ }
+
+ var dragInstruction: String {
+ switch self {
+ case .accessibility:
+ "Drag Orca Computer Use into the list above to allow Accessibility."
+ case .screenshots:
+ "Drag Orca Computer Use into the list above to allow Screenshots."
+ }
+ }
+
+ func requestAndOpenSettings() {
+ switch self {
+ case .accessibility:
+ openAccessibilitySettings()
+ case .screenshots:
+ _ = requestScreenCaptureAccess()
+ openScreenRecordingSettings()
+ }
+ }
+}
+
+private final class PermissionView: NSView {
+ private let appURL = Bundle.main.bundleURL
+ private let showDragAssistant: (PermissionKind) -> Void
+
+ init(frame frameRect: NSRect, showDragAssistant: @escaping (PermissionKind) -> Void) {
+ self.showDragAssistant = showDragAssistant
+ super.init(frame: frameRect)
+ wantsLayer = true
+ layer?.backgroundColor = PermissionPalette.background.cgColor
+ build()
+ }
+
+ required init?(coder: NSCoder) {
+ fatalError("init(coder:) has not been implemented")
+ }
+
+ private func build() {
+ let stack = NSStackView()
+ stack.orientation = .vertical
+ stack.alignment = .leading
+ stack.spacing = 10
+ stack.distribution = .gravityAreas
+ stack.translatesAutoresizingMaskIntoConstraints = false
+ addSubview(stack)
+
+ let icon = NSImageView(image: NSWorkspace.shared.icon(forFile: appURL.path))
+ icon.imageScaling = .scaleProportionallyUpOrDown
+ icon.translatesAutoresizingMaskIntoConstraints = false
+ NSLayoutConstraint.activate([
+ icon.widthAnchor.constraint(equalToConstant: 58),
+ icon.heightAnchor.constraint(equalToConstant: 58)
+ ])
+
+ let title = label("Enable Orca Computer Use", size: 22, weight: .bold)
+ let subtitle = label(
+ "Grant permissions so Orca can use apps when you ask.",
+ size: 12,
+ weight: .regular
+ )
+ subtitle.textColor = PermissionPalette.secondaryText
+ subtitle.alignment = .center
+ subtitle.maximumNumberOfLines = 3
+
+ let header = NSStackView(views: [icon, title, subtitle])
+ header.orientation = .vertical
+ header.alignment = .centerX
+ header.spacing = 6
+ header.translatesAutoresizingMaskIntoConstraints = false
+ stack.addArrangedSubview(header)
+ header.widthAnchor.constraint(equalTo: stack.widthAnchor).isActive = true
+ subtitle.widthAnchor.constraint(equalTo: stack.widthAnchor, constant: -10).isActive = true
+
+ stack.addArrangedSubview(permissionRow(
+ icon: NSImage(systemSymbolName: "figure", accessibilityDescription: "Accessibility"),
+ title: "Accessibility",
+ detail: "Read and control app interfaces",
+ buttonTitle: "Allow"
+ ) {
+ PermissionKind.accessibility.requestAndOpenSettings()
+ self.showDragAssistant(.accessibility)
+ })
+
+ stack.addArrangedSubview(permissionRow(
+ icon: NSImage(systemSymbolName: "camera.viewfinder", accessibilityDescription: "Screen Recording"),
+ title: "Screenshots",
+ detail: "Capture windows for visual state",
+ buttonTitle: "Allow"
+ ) {
+ PermissionKind.screenshots.requestAndOpenSettings()
+ self.showDragAssistant(.screenshots)
+ })
+
+ NSLayoutConstraint.activate([
+ stack.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 18),
+ stack.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -18),
+ stack.topAnchor.constraint(equalTo: topAnchor, constant: 22),
+ stack.bottomAnchor.constraint(lessThanOrEqualTo: bottomAnchor, constant: -20)
+ ])
+ }
+
+ private func permissionRow(
+ icon: NSImage?,
+ title: String,
+ detail: String,
+ buttonTitle: String,
+ action: @escaping () -> Void
+ ) -> NSView {
+ let row = NSView()
+ row.wantsLayer = true
+ row.layer?.cornerRadius = 14
+ row.layer?.borderWidth = 1
+ row.layer?.borderColor = PermissionPalette.border.cgColor
+ row.layer?.backgroundColor = PermissionPalette.card.cgColor
+ row.translatesAutoresizingMaskIntoConstraints = false
+
+ let iconView = NSImageView(image: icon ?? NSImage())
+ iconView.symbolConfiguration = NSImage.SymbolConfiguration(pointSize: 30, weight: .regular)
+ iconView.contentTintColor = .controlAccentColor
+ iconView.translatesAutoresizingMaskIntoConstraints = false
+
+ let titleLabel = label(title, size: 13, weight: .bold)
+ let detailLabel = label(detail, size: 11, weight: .regular)
+ detailLabel.textColor = PermissionPalette.secondaryText
+ let textStack = NSStackView(views: [titleLabel, detailLabel])
+ textStack.orientation = .vertical
+ textStack.alignment = .leading
+ textStack.spacing = 4
+ textStack.translatesAutoresizingMaskIntoConstraints = false
+
+ let button = NSButton(title: buttonTitle, target: nil, action: nil)
+ button.bezelStyle = .rounded
+ button.controlSize = .regular
+ button.font = NSFont.systemFont(ofSize: 13, weight: .semibold)
+ button.contentTintColor = .white
+ button.bezelColor = .controlAccentColor
+ let buttonTitleAttributes: [NSAttributedString.Key: Any] = [
+ .foregroundColor: NSColor.white,
+ .font: NSFont.systemFont(ofSize: 13, weight: .semibold)
+ ]
+ button.attributedTitle = NSAttributedString(string: buttonTitle, attributes: buttonTitleAttributes)
+ button.attributedAlternateTitle = NSAttributedString(string: buttonTitle, attributes: buttonTitleAttributes)
+ let target = ButtonTarget(action)
+ button.target = target
+ button.action = #selector(ButtonTarget.run)
+ objc_setAssociatedObject(button, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC)
+ button.translatesAutoresizingMaskIntoConstraints = false
+
+ row.addSubview(iconView)
+ row.addSubview(textStack)
+ row.addSubview(button)
+ NSLayoutConstraint.activate([
+ row.heightAnchor.constraint(equalToConstant: 62),
+ iconView.leadingAnchor.constraint(equalTo: row.leadingAnchor, constant: 12),
+ iconView.centerYAnchor.constraint(equalTo: row.centerYAnchor),
+ iconView.widthAnchor.constraint(equalToConstant: 34),
+ iconView.heightAnchor.constraint(equalToConstant: 34),
+ textStack.leadingAnchor.constraint(equalTo: iconView.trailingAnchor, constant: 10),
+ textStack.centerYAnchor.constraint(equalTo: row.centerYAnchor),
+ button.widthAnchor.constraint(greaterThanOrEqualToConstant: 52),
+ button.heightAnchor.constraint(equalToConstant: 30),
+ button.trailingAnchor.constraint(equalTo: row.trailingAnchor, constant: -12),
+ button.centerYAnchor.constraint(equalTo: row.centerYAnchor),
+ textStack.trailingAnchor.constraint(lessThanOrEqualTo: button.leadingAnchor, constant: -12)
+ ])
+ return row
+ }
+
+ private func label(_ text: String, size: CGFloat, weight: NSFont.Weight) -> NSTextField {
+ let label = NSTextField(labelWithString: text)
+ label.font = NSFont.systemFont(ofSize: size, weight: weight)
+ label.lineBreakMode = .byWordWrapping
+ label.textColor = PermissionPalette.primaryText
+ label.translatesAutoresizingMaskIntoConstraints = false
+ return label
+ }
+}
+
+private final class PermissionDragAssistantController: NSWindowController {
+ private let fallbackVisibleFrame: NSRect?
+ private let onClose: () -> Void
+ private var hasSeenSettingsWindow = false
+ private var followTimer: Timer?
+
+ convenience init(permission: PermissionKind, fallbackVisibleFrame: NSRect?, onClose: @escaping () -> Void) {
+ let window = NSPanel(
+ contentRect: NSRect(x: 0, y: 0, width: 390, height: 92),
+ styleMask: [.borderless, .nonactivatingPanel],
+ backing: .buffered,
+ defer: false
+ )
+ window.title = "Drag Orca Computer Use"
+ window.backgroundColor = .clear
+ window.isOpaque = false
+ window.isReleasedWhenClosed = false
+ window.level = .floating
+ window.hidesOnDeactivate = false
+ window.isFloatingPanel = true
+ window.becomesKeyOnlyIfNeeded = true
+ window.isMovableByWindowBackground = true
+ window.hasShadow = true
+ self.init(window: window, fallbackVisibleFrame: fallbackVisibleFrame, onClose: onClose)
+ window.contentView = PermissionDragAssistantView(permission: permission, appURL: Bundle.main.bundleURL) { [weak self, weak window] in
+ window?.close()
+ self?.onClose()
+ }
+ }
+
+ init(window: NSWindow?, fallbackVisibleFrame: NSRect?, onClose: @escaping () -> Void) {
+ self.fallbackVisibleFrame = fallbackVisibleFrame
+ self.onClose = onClose
+ super.init(window: window)
+ }
+
+ required init?(coder: NSCoder) {
+ fatalError("init(coder:) has not been implemented")
+ }
+
+ func showWhenReady() {
+ startFollowingSettingsWindow()
+ schedulePositionAndShow()
+ }
+
+ override func close() {
+ followTimer?.invalidate()
+ followTimer = nil
+ super.close()
+ }
+
+ private func schedulePositionAndShow() {
+ let delays = [0.12, 0.25, 0.4, 0.65, 0.95, 1.35, 1.8, 2.5]
+ for (index, delay) in delays.enumerated() {
+ DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in
+ guard let self, self.window?.isVisible != true else { return }
+ if self.systemSettingsIsFrontmost() && self.positionNearSettingsWindow() {
+ self.showWindow(nil)
+ self.window?.orderFrontRegardless()
+ } else if index == delays.count - 1 && self.systemSettingsIsFrontmost() {
+ self.positionFallback()
+ self.showWindow(nil)
+ self.window?.orderFrontRegardless()
+ }
+ }
+ }
+ }
+
+ private func startFollowingSettingsWindow() {
+ followTimer?.invalidate()
+ followTimer = Timer.scheduledTimer(withTimeInterval: 0.35, repeats: true) { [weak self] _ in
+ Task { @MainActor in
+ self?.syncVisibilityWithSettingsWindow()
+ }
+ }
+ }
+
+ private func syncVisibilityWithSettingsWindow() {
+ guard let window else {
+ followTimer?.invalidate()
+ followTimer = nil
+ return
+ }
+ let settingsWindowExists = systemSettingsWindowFrame() != nil
+ if settingsWindowExists {
+ hasSeenSettingsWindow = true
+ } else if hasSeenSettingsWindow {
+ NSApp.terminate(nil)
+ return
+ }
+ if systemSettingsIsFrontmost() && settingsWindowExists && positionNearSettingsWindow() {
+ if !window.isVisible {
+ showWindow(nil)
+ }
+ window.orderFrontRegardless()
+ } else if window.isVisible {
+ window.orderOut(nil)
+ }
+ }
+
+ private func systemSettingsIsFrontmost() -> Bool {
+ let bundleId = NSWorkspace.shared.frontmostApplication?.bundleIdentifier
+ return bundleId == "com.apple.systempreferences" || bundleId == "com.apple.SystemSettings"
+ }
+
+ private func positionNearSettingsWindow() -> Bool {
+ guard let window else { return false }
+ guard let settingsFrame = systemSettingsWindowFrame() else { return false }
+ let visibleFrame = visibleFrameContaining(settingsFrame)
+ let x = settingsFrame.maxX - window.frame.width - 18
+ let y = settingsFrame.minY + 18
+ window.setFrameOrigin(clampedOrigin(x: x, y: y, window: window, visibleFrame: visibleFrame))
+ return true
+ }
+
+ private func positionFallback() {
+ guard let window else { return }
+ let visibleFrame = fallbackVisibleFrame ?? NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 900, height: 700)
+ let origin = NSPoint(
+ x: visibleFrame.midX - window.frame.width / 2,
+ y: visibleFrame.minY + 24
+ )
+ window.setFrameOrigin(clampedOrigin(x: origin.x, y: origin.y, window: window, visibleFrame: visibleFrame))
+ }
+
+ private func clampedOrigin(x: CGFloat, y: CGFloat, window: NSWindow, visibleFrame: NSRect) -> NSPoint {
+ let inset: CGFloat = 10
+ let minX = visibleFrame.minX + inset
+ let maxX = visibleFrame.maxX - window.frame.width - inset
+ let minY = visibleFrame.minY + inset
+ let maxY = visibleFrame.maxY - window.frame.height - inset
+ return NSPoint(
+ x: min(max(x, minX), maxX),
+ y: min(max(y, minY), maxY)
+ )
+ }
+
+ private func systemSettingsWindowFrame() -> NSRect? {
+ guard let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
+ return nil
+ }
+ for windowInfo in windows {
+ guard
+ let ownerName = windowInfo[kCGWindowOwnerName as String] as? String,
+ ownerName == "System Settings" || ownerName == "System Preferences",
+ (windowInfo[kCGWindowLayer as String] as? Int) == 0,
+ let bounds = windowInfo[kCGWindowBounds as String] as? [String: CGFloat],
+ let x = bounds["X"],
+ let y = bounds["Y"],
+ let width = bounds["Width"],
+ let height = bounds["Height"],
+ width > 520,
+ height > 360
+ else {
+ continue
+ }
+ return appKitFrameForCGWindowFrame(NSRect(x: x, y: y, width: width, height: height))
+ }
+ return nil
+ }
+
+ private func appKitFrameForCGWindowFrame(_ cgFrame: NSRect) -> NSRect {
+ guard let screen = screenContainingCGFrame(cgFrame),
+ let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber
+ else {
+ return cgFrame
+ }
+
+ let displayBounds = CGDisplayBounds(CGDirectDisplayID(screenNumber.uint32Value))
+ return NSRect(
+ x: cgFrame.minX,
+ y: screen.frame.maxY - (cgFrame.minY - displayBounds.minY) - cgFrame.height,
+ width: cgFrame.width,
+ height: cgFrame.height
+ )
+ }
+
+ private func screenContainingCGFrame(_ frame: NSRect) -> NSScreen? {
+ NSScreen.screens.max { lhs, rhs in
+ cgDisplayBounds(for: lhs).intersection(frame).width * cgDisplayBounds(for: lhs).intersection(frame).height <
+ cgDisplayBounds(for: rhs).intersection(frame).width * cgDisplayBounds(for: rhs).intersection(frame).height
+ }
+ }
+
+ private func cgDisplayBounds(for screen: NSScreen) -> NSRect {
+ guard let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber else {
+ return screen.frame
+ }
+ return CGDisplayBounds(CGDirectDisplayID(screenNumber.uint32Value))
+ }
+
+ private func visibleFrameContaining(_ frame: NSRect) -> NSRect {
+ let screen = NSScreen.screens.max { lhs, rhs in
+ lhs.frame.intersection(frame).width * lhs.frame.intersection(frame).height <
+ rhs.frame.intersection(frame).width * rhs.frame.intersection(frame).height
+ }
+ return screen?.visibleFrame ?? NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 900, height: 700)
+ }
+}
+
+private final class PermissionDragAssistantView: NSView {
+ private let permission: PermissionKind
+ private let appURL: URL
+ private let close: () -> Void
+
+ init(permission: PermissionKind, appURL: URL, close: @escaping () -> Void) {
+ self.permission = permission
+ self.appURL = appURL
+ self.close = close
+ super.init(frame: .zero)
+ wantsLayer = true
+ layer?.backgroundColor = PermissionPalette.background.cgColor
+ layer?.cornerRadius = 14
+ layer?.borderWidth = 1
+ layer?.borderColor = PermissionPalette.border.cgColor
+ layer?.masksToBounds = true
+ build()
+ }
+
+ required init?(coder: NSCoder) {
+ fatalError("init(coder:) has not been implemented")
+ }
+
+ private func build() {
+ let closeButton = NSButton(title: "", target: nil, action: nil)
+ closeButton.bezelStyle = .shadowlessSquare
+ closeButton.isBordered = false
+ closeButton.image = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")
+ closeButton.imagePosition = .imageOnly
+ closeButton.contentTintColor = PermissionPalette.secondaryText
+ closeButton.controlSize = .small
+ closeButton.translatesAutoresizingMaskIntoConstraints = false
+ let target = ButtonTarget(close)
+ closeButton.target = target
+ closeButton.action = #selector(ButtonTarget.run)
+ objc_setAssociatedObject(closeButton, "orca-action", target, .OBJC_ASSOCIATION_RETAIN_NONATOMIC)
+
+ let instruction = label(permission.dragInstruction, size: 12, weight: .semibold)
+ instruction.textColor = PermissionPalette.primaryText
+ instruction.maximumNumberOfLines = 2
+
+ let dragTile = DraggableAppTile(appURL: appURL)
+ dragTile.translatesAutoresizingMaskIntoConstraints = false
+
+ addSubview(closeButton)
+ addSubview(instruction)
+ addSubview(dragTile)
+
+ NSLayoutConstraint.activate([
+ closeButton.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 10),
+ closeButton.topAnchor.constraint(equalTo: topAnchor, constant: 10),
+ closeButton.widthAnchor.constraint(equalToConstant: 18),
+ closeButton.heightAnchor.constraint(equalToConstant: 18),
+ instruction.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 36),
+ instruction.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16),
+ instruction.centerYAnchor.constraint(equalTo: closeButton.centerYAnchor),
+ dragTile.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16),
+ dragTile.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16),
+ dragTile.topAnchor.constraint(equalTo: instruction.bottomAnchor, constant: 8),
+ dragTile.heightAnchor.constraint(equalToConstant: 42)
+ ])
+ }
+
+ private func label(_ text: String, size: CGFloat, weight: NSFont.Weight) -> NSTextField {
+ let label = NSTextField(labelWithString: text)
+ label.font = NSFont.systemFont(ofSize: size, weight: weight)
+ label.lineBreakMode = .byWordWrapping
+ label.textColor = PermissionPalette.primaryText
+ label.translatesAutoresizingMaskIntoConstraints = false
+ return label
+ }
+}
+
+private final class DraggableAppTile: NSView, NSDraggingSource {
+ private let appURL: URL
+
+ init(appURL: URL) {
+ self.appURL = appURL
+ super.init(frame: .zero)
+ wantsLayer = true
+ layer?.cornerRadius = 12
+ layer?.borderWidth = 1
+ layer?.borderColor = PermissionPalette.border.cgColor
+ layer?.backgroundColor = PermissionPalette.card.cgColor
+ build()
+ }
+
+ required init?(coder: NSCoder) {
+ fatalError("init(coder:) has not been implemented")
+ }
+
+ override func mouseDragged(with event: NSEvent) {
+ let item = NSPasteboardItem()
+ item.setString(appURL.absoluteString, forType: .fileURL)
+ item.setString(appURL.path, forType: .string)
+
+ let draggingItem = NSDraggingItem(pasteboardWriter: item)
+ let iconSize: CGFloat = 64
+ let location = convert(event.locationInWindow, from: nil)
+ let dragFrame = NSRect(
+ x: location.x - iconSize / 2,
+ y: location.y - iconSize / 2,
+ width: iconSize,
+ height: iconSize
+ )
+ draggingItem.setDraggingFrame(dragFrame, contents: appIcon(size: iconSize))
+ beginDraggingSession(with: [draggingItem], event: event, source: self)
+ }
+
+ func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation {
+ .copy
+ }
+
+ private func build() {
+ let icon = NSImageView(image: appIcon(size: 34))
+ icon.imageScaling = .scaleProportionallyUpOrDown
+ icon.translatesAutoresizingMaskIntoConstraints = false
+
+ let title = NSTextField(labelWithString: "Orca Computer Use")
+ title.font = NSFont.systemFont(ofSize: 15, weight: .semibold)
+ title.textColor = PermissionPalette.primaryText
+ title.translatesAutoresizingMaskIntoConstraints = false
+
+ addSubview(icon)
+ addSubview(title)
+ NSLayoutConstraint.activate([
+ icon.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16),
+ icon.centerYAnchor.constraint(equalTo: centerYAnchor),
+ icon.widthAnchor.constraint(equalToConstant: 34),
+ icon.heightAnchor.constraint(equalToConstant: 34),
+ title.leadingAnchor.constraint(equalTo: icon.trailingAnchor, constant: 14),
+ title.centerYAnchor.constraint(equalTo: centerYAnchor),
+ title.trailingAnchor.constraint(lessThanOrEqualTo: trailingAnchor, constant: -20)
+ ])
+ }
+
+ private func appIcon(size: CGFloat) -> NSImage {
+ let icon = NSWorkspace.shared.icon(forFile: appURL.path)
+ icon.size = NSSize(width: size, height: size)
+ return icon
+ }
+}
+
+private enum PermissionPalette {
+ static let background = NSColor(calibratedWhite: 0.16, alpha: 0.98)
+ static let card = NSColor(calibratedWhite: 0.22, alpha: 0.98)
+ static let border = NSColor(calibratedWhite: 0.36, alpha: 0.9)
+ static let primaryText = NSColor(calibratedWhite: 0.94, alpha: 1)
+ static let secondaryText = NSColor(calibratedWhite: 0.66, alpha: 1)
+}
+
+private final class ButtonTarget: NSObject {
+ private let actionBlock: () -> Void
+
+ init(_ actionBlock: @escaping () -> Void) {
+ self.actionBlock = actionBlock
+ }
+
+ @objc func run() {
+ actionBlock()
+ }
+}
+
+private final class SocketListener: @unchecked Sendable {
+ private let socketPath: String
+ private let token: String?
+ private let provider = Provider()
+ private let providerLock = NSLock()
+ private var socketFd: Int32 = -1
+ private var isStopped = false
+
+ init(socketPath: String, token: String?) throws {
+ self.socketPath = socketPath
+ self.token = token
+ try bindSocket()
+ }
+
+ func start() {
+ Thread.detachNewThread { [weak self] in
+ self?.acceptLoop()
+ }
+ }
+
+ func stop() {
+ isStopped = true
+ if socketFd >= 0 {
+ close(socketFd)
+ socketFd = -1
+ }
+ unlink(socketPath)
+ }
+
+ private func bindSocket() throws {
+ unlink(socketPath)
+ socketFd = socket(AF_UNIX, SOCK_STREAM, 0)
+ guard socketFd >= 0 else {
+ throw ProviderError.coded("accessibility_error", "failed to create computer-use socket")
+ }
+
+ var address = sockaddr_un()
+ address.sun_family = sa_family_t(AF_UNIX)
+ let maxPathLength = MemoryLayout.size(ofValue: address.sun_path)
+ guard socketPath.utf8.count < maxPathLength else {
+ throw ProviderError.coded("invalid_argument", "computer-use socket path is too long")
+ }
+ _ = withUnsafeMutablePointer(to: &address.sun_path) { pointer in
+ socketPath.withCString { source in
+ strncpy(UnsafeMutableRawPointer(pointer).assumingMemoryBound(to: CChar.self), source, maxPathLength)
+ }
+ }
+
+ let result = withUnsafePointer(to: &address) { pointer -> Int32 in
+ pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in
+ bind(socketFd, sockaddrPointer, socklen_t(MemoryLayout.size))
+ }
+ }
+ guard result == 0 else {
+ let message = String(cString: strerror(errno))
+ close(socketFd)
+ socketFd = -1
+ throw ProviderError.coded("accessibility_error", "failed to bind computer-use socket: \(message)")
+ }
+ chmod(socketPath, 0o600)
+
+ guard listen(socketFd, 8) == 0 else {
+ let message = String(cString: strerror(errno))
+ close(socketFd)
+ socketFd = -1
+ throw ProviderError.coded("accessibility_error", "failed to listen on computer-use socket: \(message)")
+ }
+ }
+
+ private func acceptLoop() {
+ while !isStopped {
+ let fd = accept(socketFd, nil, nil)
+ if fd < 0 {
+ if !isStopped {
+ fputs("computer-use socket accept failed: \(String(cString: strerror(errno)))\n", stderr)
+ }
+ continue
+ }
+ Thread.detachNewThread { [weak self] in
+ self?.handleConnection(fd)
+ }
+ }
+ }
+
+ private func handleConnection(_ fd: Int32) {
+ defer { close(fd) }
+ let authorizedPeer = peerProcessId(fd).map(isAuthorizedAgentPeer) == true
+ let decoder = JSONDecoder()
+ while let line = readLine(from: fd) {
+ guard let data = line.data(using: .utf8),
+ let request = try? decoder.decode(Request.self, from: data)
+ else {
+ continue
+ }
+ let response = handleRequest(
+ provider: provider,
+ lock: providerLock,
+ request: request,
+ expectedToken: token,
+ authorizedPeer: authorizedPeer
+ )
+ writeJSON(response, to: fd)
+ }
+ }
+}
+
+private func peerProcessId(_ fd: Int32) -> pid_t? {
+ var pid = pid_t(0)
+ var length = socklen_t(MemoryLayout.size)
+ let result = withUnsafeMutablePointer(to: &pid) { pointer in
+ getsockopt(fd, 0, 2, pointer, &length)
+ }
+ return result == 0 && pid > 0 ? pid : nil
+}
+
+private func isAuthorizedAgentPeer(_ pid: pid_t) -> Bool {
+ guard let command = processCommand(pid),
+ command.contains("/out/main/computer-sidecar.js")
+ || command.contains("/Contents/Resources/app.asar.unpacked/out/main/computer-sidecar.js")
+ else {
+ return false
+ }
+ if isTrustedOrcaApplication(pid) {
+ return true
+ }
+ guard let parentPid = parentProcessId(pid) else { return false }
+ return isTrustedOrcaApplication(parentPid)
+}
+
+private func isTrustedOrcaApplication(_ pid: pid_t) -> Bool {
+ guard let app = NSRunningApplication(processIdentifier: pid),
+ let bundleId = app.bundleIdentifier
+ else {
+ return false
+ }
+ return bundleId == "com.stablyai.orca" || bundleId == "com.github.Electron"
+}
+
+private func parentProcessId(_ pid: pid_t) -> pid_t? {
+ guard let output = processField(pid: pid, field: "ppid=") else {
+ return nil
+ }
+ let trimmed = output.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard let parentPid = pid_t(trimmed), parentPid > 1 else {
+ return nil
+ }
+ return parentPid
+}
+
+private func processCommand(_ pid: pid_t) -> String? {
+ return processField(pid: pid, field: "command=")
+}
+
+private func processField(pid: pid_t, field: String) -> String? {
+ let process = Process()
+ let pipe = Pipe()
+ process.executableURL = URL(fileURLWithPath: "/bin/ps")
+ process.arguments = ["-p", "\(pid)", "-o", field]
+ process.standardOutput = pipe
+ process.standardError = Pipe()
+ do {
+ try process.run()
+ process.waitUntilExit()
+ guard process.terminationStatus == 0 else { return nil }
+ let data = pipe.fileHandleForReading.readDataToEndOfFile()
+ return String(data: data, encoding: .utf8)
+ } catch {
+ return nil
+ }
+}
+
+@MainActor
+private func runAgent(socketPath: String, token: String?) {
+ let app = NSApplication.shared
+ let delegate = AgentRuntime(socketPath: socketPath, token: token)
+ app.delegate = delegate
+ // Why: SCK is reliable once this code runs as a signed app with a real TCC identity.
+ setenv("ORCA_COMPUTER_USE_SCK_SCREENSHOTS", "1", 1)
+ app.run()
+}
+
+@MainActor
+private func runPermissionCheck(initialPermission: PermissionKind? = nil) {
+ let app = NSApplication.shared
+ let delegate = PermissionRuntime(initialPermission: initialPermission)
+ app.delegate = delegate
+ // Why: setup must foreground reliably; the long-running agent path stays accessory-only.
+ app.setActivationPolicy(.regular)
+ app.run()
+}
+
+private func printPermissionStatus() {
+ let accessibility = accessibilityTrusted() ? "granted" : "not-granted"
+ let screenshots = screenCaptureTrusted() ? "granted" : "not-granted"
+ print(#"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"#)
+}
+
+private func writePermissionStatus(to path: String) {
+ let accessibility = accessibilityTrusted() ? "granted" : "not-granted"
+ let screenshots = screenCaptureTrusted() ? "granted" : "not-granted"
+ let text = #"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"#
+ do {
+ try text.write(toFile: path, atomically: true, encoding: .utf8)
+ } catch {
+ fputs("failed to write permission status: \(error)\n", stderr)
+ exit(1)
+ }
+}
+
+private func runStdio() {
+ fputs("Orca Computer Use provider must be launched by Orca in app-agent mode.\n", stderr)
+ exit(13)
+}
+
+private func handleRequest(
+ provider: Provider,
+ lock: NSLock,
+ request: Request,
+ expectedToken: String?,
+ authorizedPeer: Bool
+) -> Any {
+ if let expectedToken, request.token != expectedToken {
+ return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "invalid computer-use agent token"]]
+ }
+ if expectedToken != nil && !authorizedPeer {
+ return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "computer-use agent peer is not authorized"]]
+ }
+ if request.method == "terminate" {
+ DispatchQueue.main.async {
+ NSApp.terminate(nil)
+ }
+ return ["id": request.id, "ok": true, "result": ["ok": true]]
+ }
+
+ do {
+ lock.lock()
+ defer { lock.unlock() }
+ let result = try provider.handle(method: request.method, params: request.params ?? [:])
+ return ["id": request.id, "ok": true, "result": result]
+ } catch let error as ProviderError {
+ return ["id": request.id, "ok": false, "error": ["code": error.code, "message": error.message]]
+ } catch {
+ return ["id": request.id, "ok": false, "error": ["code": "accessibility_error", "message": String(describing: error)]]
+ }
+}
+
+private func readLine(from fd: Int32) -> String? {
+ var bytes: [UInt8] = []
+ var byte: UInt8 = 0
+ while true {
+ let count = read(fd, &byte, 1)
+ if count == 0 {
+ return bytes.isEmpty ? nil : String(bytes: bytes, encoding: .utf8)
+ }
+ if count < 0 {
+ return nil
+ }
+ if byte == 10 {
+ return String(bytes: bytes, encoding: .utf8)
+ }
+ bytes.append(byte)
+ }
+}
+
+private func writeJSON(_ object: Any, to fd: Int32?) {
+ guard JSONSerialization.isValidJSONObject(object),
+ let data = try? JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes])
+ else {
+ return
+ }
+ if let fd {
+ _ = writeAll(data, to: fd)
+ _ = writeAll(Data([10]), to: fd)
+ } else {
+ guard let text = String(data: data, encoding: .utf8) else {
+ return
+ }
+ print(text)
+ fflush(stdout)
+ }
+}
+
+private func writeAll(_ data: Data, to fd: Int32) -> Bool {
+ data.withUnsafeBytes { rawBuffer in
+ guard let baseAddress = rawBuffer.baseAddress else {
+ return true
+ }
+ var offset = 0
+ while offset < rawBuffer.count {
+ let written = write(fd, baseAddress.advanced(by: offset), rawBuffer.count - offset)
+ if written < 0 {
+ if errno == EINTR {
+ continue
+ }
+ return false
+ }
+ if written == 0 {
+ return false
+ }
+ offset += written
+ }
+ return true
+ }
+}
+
+let arguments = Array(CommandLine.arguments.dropFirst())
+if arguments.first == "--agent" {
+ guard arguments.count >= 2 else {
+ fputs("usage: orca-computer-use-macos --agent --token-file \n", stderr)
+ exit(2)
+ }
+ let tokenFileIndex = arguments.firstIndex(of: "--token-file")
+ let token = tokenFileIndex.flatMap { index -> String? in
+ let valueIndex = index + 1
+ guard valueIndex < arguments.count else { return nil }
+ let tokenPath = arguments[valueIndex]
+ defer { unlink(tokenPath) }
+ return try? String(contentsOfFile: tokenPath, encoding: .utf8)
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ }
+ guard let token, !token.isEmpty else {
+ fputs("orca-computer-use-macos --agent requires a non-empty --token-file\n", stderr)
+ exit(2)
+ }
+ runAgent(socketPath: arguments[1], token: token)
+} else if arguments.first == "--permissions" {
+ runPermissionCheck()
+} else if arguments.first == "--permission" {
+ runPermissionCheck(initialPermission: PermissionKind.parse(arguments.dropFirst().first))
+} else if arguments.first == "--permission-status" {
+ printPermissionStatus()
+} else if arguments.first == "--permission-status-file" {
+ guard arguments.count >= 2 else {
+ fputs("usage: orca-computer-use-macos --permission-status-file \n", stderr)
+ exit(2)
+ }
+ writePermissionStatus(to: arguments[1])
+} else {
+ runStdio()
+}
diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SnapshotRendering.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SnapshotRendering.swift
new file mode 100644
index 00000000000..b5299cbf54b
--- /dev/null
+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SnapshotRendering.swift
@@ -0,0 +1,236 @@
+import Foundation
+
+public struct SnapshotRenderNode: Equatable {
+ public var role: String
+ public var roleDescription: String?
+ public var title: String?
+ public var label: String?
+ public var linkText: String?
+ public var value: String?
+ public var placeholder: String?
+ public var url: String?
+ public var traits: [String]
+ public var rawActions: [String]
+ public var childCount: Int
+ public var summary: String?
+ public var rowSummary: String?
+ public var webAreaDepth: Int?
+
+ public init(
+ role: String,
+ roleDescription: String? = nil,
+ title: String? = nil,
+ label: String? = nil,
+ linkText: String? = nil,
+ value: String? = nil,
+ placeholder: String? = nil,
+ url: String? = nil,
+ traits: [String] = [],
+ rawActions: [String] = [],
+ childCount: Int = 0,
+ summary: String? = nil,
+ rowSummary: String? = nil,
+ webAreaDepth: Int? = nil
+ ) {
+ self.role = role
+ self.roleDescription = roleDescription
+ self.title = title
+ self.label = label
+ self.linkText = linkText
+ self.value = value
+ self.placeholder = placeholder
+ self.url = url
+ self.traits = traits
+ self.rawActions = rawActions
+ self.childCount = childCount
+ self.summary = summary
+ self.rowSummary = rowSummary
+ self.webAreaDepth = webAreaDepth
+ }
+}
+
+public enum SnapshotRenderHeuristics {
+ public static func displayName(_ node: SnapshotRenderNode) -> String? {
+ if let title = clean(node.title) {
+ return title
+ }
+ if node.role == "AXLink", let url = clean(node.url), let text = clean(node.linkText ?? node.label ?? node.value) {
+ return "[\(markdownEscaped(text))](\(url))"
+ }
+ if node.role == "AXWebArea" {
+ return clean(node.label) ?? clean(node.value)
+ }
+ if ["AXButton", "AXPopUpButton", "AXImage"].contains(node.role) {
+ return clean(node.label)
+ }
+ if ["AXRow", "AXCell", "AXOutlineRow"].contains(node.role) {
+ return clean(node.rowSummary)
+ }
+ return clean(node.label)
+ }
+
+ public static func meaningfulActions(_ rawActions: [String], role: String) -> [String] {
+ let noisy: Set = [
+ "AXPress",
+ "AXShowDefaultUI",
+ "AXShowAlternateUI",
+ "AXShowMenu",
+ "AXScrollToVisible",
+ "AXConfirm",
+ "AXRaise",
+ ]
+ return rawActions.filter { action in
+ if noisy.contains(action) { return false }
+ if role == "AXMenu" || role == "AXMenuItem" {
+ return action != "AXCancel" && action != "AXPick"
+ }
+ if role == "AXScrollArea",
+ (rawActions.contains("AXScrollUpByPage") || rawActions.contains("AXScrollDownByPage")),
+ action == "AXScrollLeftByPage" || action == "AXScrollRightByPage" {
+ return false
+ }
+ return true
+ }
+ }
+
+ public static func shouldElide(_ node: SnapshotRenderNode) -> Bool {
+ guard node.role == "AXGroup" || node.role == "AXUnknown" else { return false }
+ guard displayName(node) == nil,
+ node.traits.isEmpty,
+ meaningfulActions(node.rawActions, role: node.role).isEmpty,
+ clean(node.summary) == nil
+ else {
+ return false
+ }
+ if node.webAreaDepth != nil, node.childCount > 1 {
+ return false
+ }
+ return true
+ }
+
+ public static func shouldSuppressChildren(_ node: SnapshotRenderNode) -> Bool {
+ if node.role == "AXMenuBarItem" {
+ return true
+ }
+ let name = displayName(node)
+ if node.role == "AXLink" && name?.hasPrefix("[") == true {
+ return true
+ }
+ let hasCompactLabel = name != nil || clean(node.value) != nil || clean(node.summary) != nil
+ return hasCompactLabel && compactControlRoles.contains(node.role)
+ }
+
+ public static func shouldSuppressChildren(role: String, name: String? = nil) -> Bool {
+ shouldSuppressChildren(SnapshotRenderNode(role: role, title: name))
+ }
+
+ public static func line(index: Int, node: SnapshotRenderNode) -> String {
+ let name = displayName(node)
+ let roleText = roleText(node)
+ let meaningful = meaningfulActions(node.rawActions, role: node.role)
+ var line = roleText.isEmpty ? "\(index)" : "\(index) \(roleText)"
+ if !node.traits.isEmpty { line += " (\(node.traits.joined(separator: ", ")))" }
+ if let name { line += " \(sanitize(name))" }
+ if node.role != "AXLink", let description = clean(node.label), description != name {
+ line += ", Description: \(sanitize(description))"
+ }
+ if let valueSegment = formattedValueSegment(roleText: roleText, name: name, value: node.value) {
+ line += valueSegment
+ }
+ if let placeholder = clean(node.placeholder), placeholder != name, placeholder != node.value {
+ line += name == nil && clean(node.value) == nil ? " Placeholder: \(sanitize(placeholder))" : ", Placeholder: \(sanitize(placeholder))"
+ }
+ if let summary = clean(node.summary), summary != name {
+ line += ", Text: \(sanitize(summary))"
+ } else if let rowSummary = clean(node.rowSummary), rowSummary != name {
+ line += ", Text: \(sanitize(rowSummary))"
+ }
+ if !meaningful.isEmpty {
+ line += ", Secondary Actions: \(meaningful.map(prettyAction).joined(separator: ", "))"
+ }
+ return line
+ }
+
+ public static func roleText(_ node: SnapshotRenderNode) -> String {
+ if node.role == "AXGroup" || node.role == "AXUnknown" {
+ return "container"
+ }
+ if node.role == "AXLink" {
+ return "link"
+ }
+ if node.role == "AXWebArea" {
+ return clean(node.roleDescription) ?? "html content"
+ }
+ if node.role == "AXMenuBarItem" {
+ return ""
+ }
+ if let value = clean(node.roleDescription) {
+ return value.lowercased()
+ }
+ if node.role.hasPrefix("AX") {
+ return splitCamelCase(String(node.role.dropFirst(2))).lowercased()
+ }
+ return node.role
+ }
+
+ public static func prettyAction(_ action: String) -> String {
+ if action == "AXZoomWindow" {
+ return "zoom the window"
+ }
+ let stripped = action.hasPrefix("AX") ? String(action.dropFirst(2)) : action
+ return splitCamelCase(stripped.replacingOccurrences(of: "ByPage", with: "")).lowercased()
+ }
+
+ public static func sanitize(_ value: String) -> String {
+ value.replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\r", with: " ")
+ }
+
+ private static func clean(_ value: String?) -> String? {
+ guard let value else { return nil }
+ let sanitized = sanitize(value)
+ return sanitized.isEmpty ? nil : sanitized
+ }
+
+ private static func formattedValueSegment(roleText: String, name: String?, value: String?) -> String? {
+ guard let value = clean(value), value != name else { return nil }
+ if roleText == "heading", Int(value) != nil {
+ return nil
+ }
+ let clean = sanitize(value)
+ if roleText == "text" || roleText == "text entry area" || roleText == "scroll bar" || roleText == "value indicator" {
+ return " \(clean)"
+ }
+ return ", Value: \(clean)"
+ }
+
+ private static func markdownEscaped(_ value: String) -> String {
+ sanitize(value)
+ .replacingOccurrences(of: "\\", with: "\\\\")
+ .replacingOccurrences(of: "[", with: "\\[")
+ .replacingOccurrences(of: "]", with: "\\]")
+ }
+
+ private static func splitCamelCase(_ value: String) -> String {
+ var result = ""
+ for character in value {
+ if character.isUppercase, !result.isEmpty {
+ result.append(" ")
+ }
+ result.append(character)
+ }
+ return result
+ }
+
+ private static let compactControlRoles: Set = [
+ "AXButton",
+ "AXCheckBox",
+ "AXComboBox",
+ "AXDisclosureTriangle",
+ "AXHeading",
+ "AXMenuItem",
+ "AXPopUpButton",
+ "AXRadioButton",
+ "AXStaticText",
+ "AXTab",
+ ]
+}
diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SnapshotRenderingTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SnapshotRenderingTests.swift
new file mode 100644
index 00000000000..3c28e044901
--- /dev/null
+++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SnapshotRenderingTests.swift
@@ -0,0 +1,85 @@
+import XCTest
+@testable import OrcaComputerUseMacOSCore
+
+final class SnapshotRenderingTests: XCTestCase {
+ func testElidesAnonymousWrappers() {
+ let node = SnapshotRenderNode(role: "AXGroup", childCount: 1)
+
+ XCTAssertTrue(SnapshotRenderHeuristics.shouldElide(node))
+ }
+
+ func testPreservesWebAreaContainersWithMultipleChildren() {
+ let node = SnapshotRenderNode(role: "AXGroup", childCount: 3, webAreaDepth: 1)
+
+ XCTAssertFalse(SnapshotRenderHeuristics.shouldElide(node))
+ }
+
+ func testRendersMarkdownLinksAndSuppressesTheirChildren() {
+ let node = SnapshotRenderNode(role: "AXLink", linkText: "Skip [main]", url: "https://example.com/path")
+
+ XCTAssertEqual(SnapshotRenderHeuristics.line(index: 7, node: node), "7 link [Skip \\[main\\]](https://example.com/path)")
+ XCTAssertTrue(SnapshotRenderHeuristics.shouldSuppressChildren(node))
+ }
+
+ func testSuppressesChildrenForNamedCompactControls() {
+ let button = SnapshotRenderNode(role: "AXButton", roleDescription: "button", label: "Install GitHub", childCount: 1)
+ let heading = SnapshotRenderNode(role: "AXHeading", roleDescription: "heading", label: "Repository navigation", value: "2", childCount: 1)
+
+ XCTAssertTrue(SnapshotRenderHeuristics.shouldSuppressChildren(button))
+ XCTAssertTrue(SnapshotRenderHeuristics.shouldSuppressChildren(heading))
+ XCTAssertEqual(SnapshotRenderHeuristics.line(index: 5, node: heading), "5 heading Repository navigation")
+ }
+
+ func testKeepsChildrenForRowsWithNestedControls() {
+ let row = SnapshotRenderNode(role: "AXRow", roleDescription: "row", childCount: 3, rowSummary: "Liked Songs")
+
+ XCTAssertFalse(SnapshotRenderHeuristics.shouldSuppressChildren(row))
+ }
+
+ func testFiltersNoisyActionsAndFormatsSecondaryActions() {
+ let node = SnapshotRenderNode(
+ role: "AXWindow",
+ title: "Document",
+ rawActions: ["AXPress", "AXShowMenu", "AXScrollToVisible", "AXZoomWindow"]
+ )
+
+ XCTAssertEqual(SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role), ["AXZoomWindow"])
+ XCTAssertEqual(SnapshotRenderHeuristics.line(index: 1, node: node), "1 window Document, Secondary Actions: zoom the window")
+ }
+
+ func testSuppressesHorizontalScrollWhenVerticalScrollExists() {
+ let node = SnapshotRenderNode(
+ role: "AXScrollArea",
+ rawActions: ["AXScrollUpByPage", "AXScrollDownByPage", "AXScrollLeftByPage", "AXScrollRightByPage"]
+ )
+
+ XCTAssertEqual(SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role), ["AXScrollUpByPage", "AXScrollDownByPage"])
+ }
+
+ func testTextFieldsKeepDistinctValueAndPlaceholder() {
+ let node = SnapshotRenderNode(
+ role: "AXTextField",
+ roleDescription: "text field",
+ label: "Address",
+ value: "https://example.com",
+ placeholder: "Search"
+ )
+
+ XCTAssertEqual(
+ SnapshotRenderHeuristics.line(index: 3, node: node),
+ "3 text field Address, Value: https://example.com, Placeholder: Search"
+ )
+ }
+
+ func testStaticTextUsesCompactValueFormatting() {
+ let node = SnapshotRenderNode(role: "AXStaticText", roleDescription: "text", value: "Home")
+
+ XCTAssertEqual(SnapshotRenderHeuristics.line(index: 4, node: node), "4 text Home")
+ }
+
+ func testRowSummaryBecomesName() {
+ let node = SnapshotRenderNode(role: "AXRow", roleDescription: "row", rowSummary: "General Settings Enabled")
+
+ XCTAssertEqual(SnapshotRenderHeuristics.line(index: 9, node: node), "9 row General Settings Enabled")
+ }
+}
diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1
new file mode 100644
index 00000000000..f87daa87e5c
--- /dev/null
+++ b/native/computer-use-windows/runtime.ps1
@@ -0,0 +1,879 @@
+param(
+ [Parameter(Mandatory = $true)]
+ [string]$OperationPath
+)
+
+$ErrorActionPreference = "Stop"
+
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+Add-Type -AssemblyName System.Drawing
+Add-Type -AssemblyName System.Windows.Forms
+
+Add-Type -TypeDefinition @"
+using System;
+using System.Runtime.InteropServices;
+
+public static class OrcaDesktopWin32 {
+ [StructLayout(LayoutKind.Sequential)]
+ public struct RECT {
+ public int Left;
+ public int Top;
+ public int Right;
+ public int Bottom;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public struct POINT {
+ public int X;
+ public int Y;
+ }
+
+ [DllImport("user32.dll")]
+ public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);
+
+ [DllImport("user32.dll")]
+ public static extern bool ScreenToClient(IntPtr hwnd, ref POINT point);
+
+ [DllImport("user32.dll")]
+ public static extern bool PostMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam);
+
+ [DllImport("user32.dll")]
+ public static extern bool ShowWindow(IntPtr hwnd, int nCmdShow);
+
+ [DllImport("user32.dll")]
+ public static extern bool SetForegroundWindow(IntPtr hwnd);
+
+ [DllImport("user32.dll")]
+ public static extern IntPtr GetForegroundWindow();
+}
+"@
+
+$MaxNodes = 1200
+$MaxDepth = 64
+$TextLimit = 500
+$BlockedAppFragments = @(
+ "1password",
+ "bitwarden",
+ "dashlane",
+ "lastpass",
+ "nordpass",
+ "proton pass"
+)
+
+$WindowsMessages = @{
+ Char = 0x0102
+ KeyDown = 0x0100
+ KeyUp = 0x0101
+ MouseMove = 0x0200
+ LeftDown = 0x0201
+ LeftUp = 0x0202
+ RightDown = 0x0204
+ RightUp = 0x0205
+ MiddleDown = 0x0207
+ MiddleUp = 0x0208
+ Wheel = 0x020A
+}
+
+function Write-OrcaJson($Payload) {
+ $Payload | ConvertTo-Json -Depth 100 -Compress
+}
+
+function New-OrcaFrame([double]$X, [double]$Y, [double]$Width, [double]$Height) {
+ if ($Width -le 0 -or $Height -le 0) { return $null }
+ [pscustomobject]@{ x = $X; y = $Y; width = $Width; height = $Height }
+}
+
+function Read-OrcaOperation([string]$Path) {
+ Get-Content -Raw -Path $Path | ConvertFrom-Json
+}
+
+function ConvertTo-OrcaLParam([int]$X, [int]$Y) {
+ [IntPtr]((($Y -band 0xffff) -shl 16) -bor ($X -band 0xffff))
+}
+
+function ConvertTo-OrcaWheelParam([int]$Delta) {
+ [IntPtr](($Delta -band 0xffff) -shl 16)
+}
+
+function Get-OrcaWindowProcesses {
+ @(Get-Process | Where-Object { $_.MainWindowHandle -ne 0 } | Sort-Object ProcessName, Id)
+}
+
+function Find-OrcaProcess([string]$Query) {
+ $needle = ""
+ if ($null -ne $Query) { $needle = $Query.Trim() }
+ if ([string]::IsNullOrWhiteSpace($needle)) { throw 'appNotFound("")' }
+ if ($needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase)) {
+ $needle = $needle.Substring(4)
+ }
+
+ $pid = 0
+ $processes = Get-OrcaWindowProcesses
+ if ([int]::TryParse($needle, [ref]$pid)) {
+ $match = $processes | Where-Object { $_.Id -eq $pid } | Select-Object -First 1
+ if ($null -ne $match) {
+ Assert-OrcaProcessAllowed $match
+ return $match
+ }
+ }
+
+ $processNeedle = $needle
+ if ($processNeedle.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) {
+ $processNeedle = $processNeedle.Substring(0, $processNeedle.Length - 4)
+ }
+
+ $match = $processes | Where-Object {
+ $_.ProcessName -ieq $processNeedle -or
+ "$($_.ProcessName).exe" -ieq $needle -or
+ $_.MainWindowTitle -ieq $needle -or
+ $_.MainWindowTitle -ilike "*$needle*"
+ } | Select-Object -First 1
+ if ($null -ne $match) {
+ Assert-OrcaProcessAllowed $match
+ return $match
+ }
+
+ throw "appNotFound(`"$Query`")"
+}
+
+function Assert-OrcaProcessAllowed($Process) {
+ $values = @($Process.ProcessName, $Process.MainWindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() }
+ foreach ($fragment in $BlockedAppFragments) {
+ foreach ($value in $values) {
+ if ($value.Contains($fragment)) {
+ throw "appBlocked(`"$($Process.ProcessName)`")"
+ }
+ }
+ }
+}
+
+function Get-OrcaRootElement($Process) {
+ if ($Process.MainWindowHandle -eq 0) {
+ throw "No top-level UI Automation window is available for $($Process.ProcessName)."
+ }
+ [Windows.Automation.AutomationElement]::FromHandle([IntPtr]$Process.MainWindowHandle)
+}
+
+function Get-OrcaWindowFrame($Process, $RootElement) {
+ $rect = New-Object OrcaDesktopWin32+RECT
+ if ([OrcaDesktopWin32]::GetWindowRect([IntPtr]$Process.MainWindowHandle, [ref]$rect)) {
+ return New-OrcaFrame $rect.Left $rect.Top ($rect.Right - $rect.Left) ($rect.Bottom - $rect.Top)
+ }
+
+ try {
+ $bounds = $RootElement.Current.BoundingRectangle
+ if (-not $bounds.IsEmpty) {
+ return New-OrcaFrame $bounds.X $bounds.Y $bounds.Width $bounds.Height
+ }
+ } catch {}
+ $null
+}
+
+function Get-OrcaWindowId($Process) {
+ [int64]$Process.MainWindowHandle
+}
+
+function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) {
+ if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) {
+ throw "windowNotFound(`"$WindowIndex`")"
+ }
+ if ($null -ne $WindowId -and [int64]$WindowId -ne (Get-OrcaWindowId $Process)) {
+ throw "windowNotFound(`"$WindowId`")"
+ }
+}
+
+function Restore-OrcaWindow($Process) {
+ if ($Process.MainWindowHandle -eq 0) { return }
+ [void][OrcaDesktopWin32]::ShowWindow([IntPtr]$Process.MainWindowHandle, 9)
+ [void][OrcaDesktopWin32]::SetForegroundWindow([IntPtr]$Process.MainWindowHandle)
+}
+
+function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation) {
+ if ([bool]$Operation.restoreWindow) { return }
+ if ([OrcaDesktopWin32]::GetForegroundWindow() -eq $WindowHandle) { return }
+ throw "window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window"
+}
+
+function Get-OrcaElementFrame($Element, $WindowFrame) {
+ try {
+ $bounds = $Element.Current.BoundingRectangle
+ if ($bounds.IsEmpty) { return $null }
+ if ($null -eq $WindowFrame) {
+ return New-OrcaFrame $bounds.X $bounds.Y $bounds.Width $bounds.Height
+ }
+ New-OrcaFrame ($bounds.X - $WindowFrame.x) ($bounds.Y - $WindowFrame.y) $bounds.Width $bounds.Height
+ } catch {
+ $null
+ }
+}
+
+function Get-OrcaProperty($Element, [string]$Name) {
+ try { [string]$Element.Current.$Name } catch { "" }
+}
+
+function Get-OrcaRuntimeId($Element) {
+ try { @($Element.GetRuntimeId()) } catch { @() }
+}
+
+function Get-OrcaValueText($Element) {
+ try {
+ if ($Element.Current.IsPassword) { return "[redacted]" }
+ $pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
+ $rawValue = $pattern.Current.Value
+ $text = if ($null -eq $rawValue) { "" } else { [string]$rawValue }
+ if ($text.Length -gt $TextLimit) { return $text.Substring(0, $TextLimit) + "..." }
+ $text
+ } catch {
+ ""
+ }
+}
+
+function Get-OrcaActions($Element) {
+ $actions = New-Object System.Collections.Generic.List[string]
+ foreach ($pattern in $Element.GetSupportedPatterns()) {
+ $name = [string]$pattern.ProgrammaticName
+ if ($name -like "InvokePatternIdentifiers.Pattern") { $actions.Add("Invoke") }
+ elseif ($name -like "TogglePatternIdentifiers.Pattern") { $actions.Add("Toggle") }
+ elseif ($name -like "SelectionItemPatternIdentifiers.Pattern") { $actions.Add("Select") }
+ elseif ($name -like "ScrollPatternIdentifiers.Pattern") { $actions.Add("Scroll") }
+ elseif ($name -like "ValuePatternIdentifiers.Pattern") { $actions.Add("SetValue") }
+ }
+ @($actions | Select-Object -Unique)
+}
+
+function Get-OrcaMeaningfulActions($Actions) {
+ $noisy = @("Invoke", "ScrollToVisible", "ShowMenu")
+ @($Actions | Where-Object { $noisy -notcontains $_ })
+}
+
+function Format-OrcaSnapshotText([string]$Text) {
+ if ([string]::IsNullOrWhiteSpace($Text)) { return "" }
+ (($Text -replace "\s+", " ").Trim())
+}
+
+function Format-OrcaValueSegment([string]$RoleKey, [string]$Title, [string]$Value) {
+ $clean = Format-OrcaSnapshotText $Value
+ if ([string]::IsNullOrWhiteSpace($clean) -or $clean -eq $Title) { return "" }
+ if ($RoleKey -eq "heading" -and $clean -match "^\d+$") { return "" }
+ if ($RoleKey -in @("text", "edit", "document", "scroll bar", "progress bar")) {
+ return " $clean"
+ }
+ ", Value: $clean"
+}
+
+function Test-OrcaSuppressChildren([string]$RoleKey, [string]$Title, [string]$Value, [string]$Summary) {
+ $hasCompactLabel = -not [string]::IsNullOrWhiteSpace($Title) -or -not [string]::IsNullOrWhiteSpace((Format-OrcaSnapshotText $Value)) -or -not [string]::IsNullOrWhiteSpace((Format-OrcaSnapshotText $Summary))
+ $hasCompactLabel -and $RoleKey -in @(
+ "button",
+ "check box",
+ "combo box",
+ "heading",
+ "hyperlink",
+ "link",
+ "menu item",
+ "radio button",
+ "tab item"
+ )
+}
+
+function Get-OrcaTextSnippets($Element, [int]$Limit = 6, [int]$MaxDepth = 3) {
+ $values = New-Object System.Collections.Generic.List[string]
+ $seen = New-Object System.Collections.Generic.HashSet[string]
+
+ function Visit-OrcaText($Node, [int]$Depth) {
+ if ($values.Count -ge $Limit -or $Depth -gt $MaxDepth) { return }
+ $role = try { [string]$Node.Current.LocalizedControlType } catch { "" }
+ if ($role -match "text|link|label") {
+ foreach ($raw in @((Get-OrcaProperty $Node "Name"), (Get-OrcaValueText $Node))) {
+ $value = (($raw -replace "\s+", " ").Trim())
+ if (-not [string]::IsNullOrWhiteSpace($value) -and $seen.Add($value)) {
+ if ($value.Length -gt 80) { $value = $value.Substring(0, 80) + "..." }
+ $values.Add($value)
+ if ($values.Count -ge $Limit) { return }
+ }
+ }
+ }
+ try {
+ $children = $Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition)
+ for ($i = 0; $i -lt $children.Count; $i++) {
+ Visit-OrcaText $children.Item($i) ($Depth + 1)
+ if ($values.Count -ge $Limit) { return }
+ }
+ } catch {}
+ }
+
+ Visit-OrcaText $Element 0
+ @($values.ToArray())
+}
+
+function Test-OrcaPlainTextSubtree($Element, [int]$MaxDepth = 4) {
+ $script:sawOrcaText = $false
+ $allowed = @("pane", "group", "custom", "unknown", "text", "link", "image")
+
+ function Visit-OrcaPlainText($Node, [int]$Depth) {
+ if ($Depth -gt $MaxDepth) { return $false }
+ $role = try { [string]$Node.Current.LocalizedControlType } catch { "" }
+ $roleKey = $role.ToLowerInvariant()
+ if ($allowed -notcontains $roleKey) { return $false }
+ if ($roleKey -match "text|link") { $script:sawOrcaText = $true }
+ if (@(Get-OrcaMeaningfulActions @(Get-OrcaActions $Node)).Count -gt 0) { return $false }
+ try {
+ $children = $Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition)
+ for ($i = 0; $i -lt $children.Count; $i++) {
+ if (-not (Visit-OrcaPlainText $children.Item($i) ($Depth + 1))) { return $false }
+ }
+ } catch {}
+ return $true
+ }
+
+ (Visit-OrcaPlainText $Element 0) -and $script:sawOrcaText
+}
+
+function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
+ $controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
+ $nativeWindowHandle = try { [int64]$Element.Current.NativeWindowHandle } catch { 0 }
+ [pscustomobject]@{
+ index = $Index
+ runtimeId = @(Get-OrcaRuntimeId $Element)
+ automationId = Get-OrcaProperty $Element "AutomationId"
+ name = Get-OrcaProperty $Element "Name"
+ controlType = $controlType
+ localizedControlType = Get-OrcaProperty $Element "LocalizedControlType"
+ className = Get-OrcaProperty $Element "ClassName"
+ value = Get-OrcaValueText $Element
+ nativeWindowHandle = $nativeWindowHandle
+ frame = Get-OrcaElementFrame $Element $WindowFrame
+ actions = @(Get-OrcaActions $Element)
+ }
+}
+
+function Render-OrcaTree($RootElement, $WindowFrame) {
+ $records = New-Object System.Collections.Generic.List[object]
+ $lines = New-Object System.Collections.Generic.List[string]
+ $seen = New-Object System.Collections.Generic.HashSet[string]
+ $truncation = [pscustomobject]@{
+ truncated = $false
+ maxNodes = $MaxNodes
+ maxDepth = $MaxDepth
+ maxDepthReached = $false
+ }
+
+ function Visit-OrcaNode($Node, [int]$Depth) {
+ if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) {
+ $truncation.truncated = $true
+ if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true }
+ return
+ }
+ $identity = try { (@($Node.GetRuntimeId()) -join ".") } catch { [Guid]::NewGuid().ToString() }
+ if (-not $seen.Add($identity)) { return }
+
+ $record = New-OrcaElementRecord $Node $records.Count $WindowFrame
+ $children = @()
+ try {
+ $children = @($Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition))
+ } catch {}
+ $meaningfulActions = @(Get-OrcaMeaningfulActions $record.actions)
+ $title = if ([string]::IsNullOrWhiteSpace($record.name)) { $record.automationId } else { $record.name }
+ $role = if ([string]::IsNullOrWhiteSpace($record.localizedControlType)) { $record.controlType } else { $record.localizedControlType }
+ $roleKey = $role.ToLowerInvariant()
+ $snippets = @(Get-OrcaTextSnippets $Node 8 4)
+ $genericSummary = $null
+ if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) {
+ $genericSummary = ($snippets -join " ")
+ }
+ if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) {
+ for ($i = 0; $i -lt $children.Count; $i++) {
+ Visit-OrcaNode $children.Item($i) $Depth
+ }
+ return
+ }
+
+ $records.Add($record)
+
+ $line = "$($record.index) $role $(Format-OrcaSnapshotText $title)".TrimEnd()
+ $line += Format-OrcaValueSegment $roleKey $title $record.value
+ if (-not [string]::IsNullOrWhiteSpace($genericSummary) -and $genericSummary -ne $title) {
+ $line += ", Text: " + (Format-OrcaSnapshotText $genericSummary)
+ } elseif ($roleKey -in @("row", "data item", "list item")) {
+ $rowSummary = @((Get-OrcaTextSnippets $Node 6 3)) -join " "
+ if (-not [string]::IsNullOrWhiteSpace($rowSummary) -and $rowSummary -ne $title) {
+ $line += ", Text: " + (Format-OrcaSnapshotText $rowSummary)
+ }
+ }
+ if ($meaningfulActions.Count -gt 0) {
+ $line += ", Secondary Actions: " + ($meaningfulActions -join ", ")
+ }
+ $lines.Add(("`t" * $Depth) + $line)
+
+ if (-not [string]::IsNullOrWhiteSpace($genericSummary) -or (Test-OrcaSuppressChildren $roleKey $title $record.value $genericSummary)) { return }
+ for ($i = 0; $i -lt $children.Count; $i++) {
+ Visit-OrcaNode $children.Item($i) ($Depth + 1)
+ }
+ }
+
+ Visit-OrcaNode $RootElement 0
+ [pscustomobject]@{ elements = @($records.ToArray()); lines = @($lines.ToArray()); truncation = $truncation }
+}
+
+function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame) {
+ if (-not $IncludeScreenshot -or $null -eq $WindowFrame) { return $null }
+ try {
+ $width = [int][Math]::Max(1, [Math]::Round($WindowFrame.width))
+ $height = [int][Math]::Max(1, [Math]::Round($WindowFrame.height))
+ $bitmap = New-Object System.Drawing.Bitmap $width, $height
+ $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+ $graphics.CopyFromScreen([int][Math]::Round($WindowFrame.x), [int][Math]::Round($WindowFrame.y), 0, 0, $bitmap.Size)
+ $stream = New-Object System.IO.MemoryStream
+ $bitmap.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png)
+ $bytes = $stream.ToArray()
+ $graphics.Dispose()
+ $bitmap.Dispose()
+ $stream.Dispose()
+ [Convert]::ToBase64String($bytes)
+ } catch {
+ $null
+ }
+}
+
+function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false) {
+ $process = Find-OrcaProcess $Query
+ if ($RestoreWindow) { Restore-OrcaWindow $process }
+ Assert-OrcaWindowTarget $process $WindowId $WindowIndex
+ $root = Get-OrcaRootElement $process
+ $windowFrame = Get-OrcaWindowFrame $process $root
+ $tree = Render-OrcaTree $root $windowFrame
+
+ [pscustomobject]@{
+ snapshotId = [guid]::NewGuid().ToString()
+ app = [pscustomobject]@{
+ name = $process.ProcessName
+ bundleIdentifier = $process.ProcessName
+ bundleId = $process.ProcessName
+ pid = [int]$process.Id
+ }
+ windowTitle = $process.MainWindowTitle
+ windowId = Get-OrcaWindowId $process
+ windowBounds = $windowFrame
+ screenshotPngBase64 = Get-OrcaScreenshot $IncludeScreenshot $windowFrame
+ coordinateSpace = "window"
+ truncation = $tree.truncation
+ treeLines = @($tree.lines)
+ focusedSummary = $null
+ focusedElementId = $null
+ selectedText = $null
+ elements = @($tree.elements)
+ }
+}
+
+function Get-OrcaAppList {
+ @(Get-OrcaWindowProcesses | ForEach-Object {
+ [pscustomobject]@{
+ name = $_.ProcessName
+ bundleIdentifier = $_.ProcessName
+ bundleId = $_.ProcessName
+ pid = [int]$_.Id
+ }
+ })
+}
+
+function Get-OrcaWindowList([string]$Query) {
+ $process = Find-OrcaProcess $Query
+ $root = Get-OrcaRootElement $process
+ $windowFrame = Get-OrcaWindowFrame $process $root
+ $x = $null
+ $y = $null
+ $width = 0
+ $height = 0
+ if ($null -ne $windowFrame) {
+ $x = [int][Math]::Round($windowFrame.x)
+ $y = [int][Math]::Round($windowFrame.y)
+ $width = [int][Math]::Max(0, [Math]::Round($windowFrame.width))
+ $height = [int][Math]::Max(0, [Math]::Round($windowFrame.height))
+ }
+ $app = [pscustomobject]@{
+ name = $process.ProcessName
+ bundleIdentifier = $process.ProcessName
+ bundleId = $process.ProcessName
+ pid = [int]$process.Id
+ }
+ [pscustomobject]@{
+ app = $app
+ windows = @([pscustomobject]@{
+ index = 0
+ app = $app
+ id = Get-OrcaWindowId $process
+ title = $process.MainWindowTitle
+ x = $x
+ y = $y
+ width = $width
+ height = $height
+ isMinimized = $false
+ isOffscreen = $false
+ screenIndex = $null
+ platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = Get-OrcaWindowId $process }
+ })
+ }
+}
+
+function Get-OrcaHandshake {
+ [pscustomobject]@{
+ platform = "win32"
+ provider = "orca-computer-use-windows"
+ providerVersion = "1.0.0"
+ protocolVersion = 1
+ supports = [pscustomobject]@{
+ apps = [pscustomobject]@{ list = $true; bundleIds = $false; pids = $true }
+ windows = [pscustomobject]@{ list = $true; targetById = $true; targetByIndex = $true; focus = $false; moveResize = $false }
+ observation = [pscustomobject]@{ screenshot = $true; annotatedScreenshot = $false; elementFrames = $true; ocr = $false }
+ actions = [pscustomobject]@{
+ click = $true
+ typeText = $true
+ pressKey = $true
+ hotkey = $true
+ pasteText = $true
+ scroll = $true
+ drag = $true
+ setValue = $true
+ performAction = $true
+ }
+ surfaces = [pscustomobject]@{ menus = $false; dialogs = $false; dock = $false; menubar = $false }
+ }
+ }
+}
+
+function Test-OrcaSameRuntimeId($Left, $Right) {
+ if ($null -eq $Left -or $null -eq $Right -or $Left.Count -ne $Right.Count) { return $false }
+ for ($i = 0; $i -lt $Left.Count; $i++) {
+ if ([int]$Left[$i] -ne [int]$Right[$i]) { return $false }
+ }
+ $true
+}
+
+function Find-OrcaElement($RootElement, $Record) {
+ if ($null -eq $Record) { return $null }
+ if ($Record.index -eq 0) { return $RootElement }
+
+ try {
+ $descendants = $RootElement.FindAll([Windows.Automation.TreeScope]::Descendants, [Windows.Automation.Condition]::TrueCondition)
+ for ($i = 0; $i -lt $descendants.Count; $i++) {
+ $candidate = $descendants.Item($i)
+ if (Test-OrcaSameRuntimeId @($candidate.GetRuntimeId()) @($Record.runtimeId)) {
+ return $candidate
+ }
+ }
+ } catch {}
+ $null
+}
+
+function Invoke-OrcaPrimaryAction($Element) {
+ foreach ($pattern in @(
+ [Windows.Automation.InvokePattern]::Pattern,
+ [Windows.Automation.SelectionItemPattern]::Pattern,
+ [Windows.Automation.TogglePattern]::Pattern
+ )) {
+ try {
+ $instance = $Element.GetCurrentPattern($pattern)
+ if ($pattern -eq [Windows.Automation.InvokePattern]::Pattern) { $instance.Invoke(); return $true }
+ if ($pattern -eq [Windows.Automation.SelectionItemPattern]::Pattern) { $instance.Select(); return $true }
+ if ($pattern -eq [Windows.Automation.TogglePattern]::Pattern) { $instance.Toggle(); return $true }
+ } catch {}
+ }
+ $false
+}
+
+function Invoke-OrcaNamedAction($Element, [string]$Action) {
+ $wanted = ""
+ if ($null -ne $Action) { $wanted = $Action.Trim().ToLowerInvariant() }
+ switch ($wanted) {
+ "invoke" {
+ $pattern = $Element.GetCurrentPattern([Windows.Automation.InvokePattern]::Pattern)
+ $pattern.Invoke()
+ return $true
+ }
+ "select" {
+ $pattern = $Element.GetCurrentPattern([Windows.Automation.SelectionItemPattern]::Pattern)
+ $pattern.Select()
+ return $true
+ }
+ "toggle" {
+ $pattern = $Element.GetCurrentPattern([Windows.Automation.TogglePattern]::Pattern)
+ $pattern.Toggle()
+ return $true
+ }
+ default {
+ return $false
+ }
+ }
+}
+
+function Set-OrcaElementValue($Element, [string]$Value) {
+ try {
+ $pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
+ if (-not $pattern.Current.IsReadOnly) {
+ $pattern.SetValue($Value)
+ return $true
+ }
+ } catch {}
+ $false
+}
+
+function Get-OrcaScreenPoint($Operation, $WindowFrame) {
+ if ($null -ne $Operation.element) {
+ throw "stale element frame; run get-app-state again and use a fresh element index"
+ }
+ @{
+ x = [int][Math]::Round($WindowFrame.x + [double]$Operation.x)
+ y = [int][Math]::Round($WindowFrame.y + [double]$Operation.y)
+ }
+}
+
+function Get-OrcaElementScreenPoint($Element) {
+ if ($null -eq $Element) { return $null }
+ try {
+ $rect = $Element.Current.BoundingRectangle
+ if ($rect.Width -gt 0 -and $rect.Height -gt 0) {
+ return @{
+ x = [int][Math]::Round($rect.X + ($rect.Width / 2))
+ y = [int][Math]::Round($rect.Y + ($rect.Height / 2))
+ }
+ }
+ } catch {}
+ $null
+}
+
+function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count) {
+ $point = New-Object OrcaDesktopWin32+POINT
+ $point.X = $ScreenX
+ $point.Y = $ScreenY
+ [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$point)
+
+ $down = $WindowsMessages.LeftDown
+ $up = $WindowsMessages.LeftUp
+ $flag = 1
+ if ($Button -eq "right") {
+ $down = $WindowsMessages.RightDown
+ $up = $WindowsMessages.RightUp
+ $flag = 2
+ } elseif ($Button -eq "middle") {
+ $down = $WindowsMessages.MiddleDown
+ $up = $WindowsMessages.MiddleUp
+ $flag = 16
+ }
+
+ $position = ConvertTo-OrcaLParam $point.X $point.Y
+ for ($i = 0; $i -lt [Math]::Max(1, $Count); $i++) {
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, $position)
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $down, [IntPtr]$flag, $position)
+ Start-Sleep -Milliseconds 35
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $up, [IntPtr]::Zero, $position)
+ }
+}
+
+function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {
+ $start = New-Object OrcaDesktopWin32+POINT
+ $start.X = [int]$From.x
+ $start.Y = [int]$From.y
+ [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$start)
+
+ $end = New-Object OrcaDesktopWin32+POINT
+ $end.X = [int]$To.x
+ $end.Y = [int]$To.y
+ [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$end)
+
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, (ConvertTo-OrcaLParam $start.X $start.Y))
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftDown, [IntPtr]1, (ConvertTo-OrcaLParam $start.X $start.Y))
+ for ($step = 1; $step -le 12; $step++) {
+ $x = [int][Math]::Round($start.X + (($end.X - $start.X) * $step / 12))
+ $y = [int][Math]::Round($start.Y + (($end.Y - $start.Y) * $step / 12))
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]1, (ConvertTo-OrcaLParam $x $y))
+ Start-Sleep -Milliseconds 20
+ }
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftUp, [IntPtr]::Zero, (ConvertTo-OrcaLParam $end.X $end.Y))
+}
+
+function Send-OrcaText([IntPtr]$WindowHandle, [string]$Text) {
+ foreach ($character in $Text.ToCharArray()) {
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.Char, [IntPtr][int][char]$character, [IntPtr]::Zero)
+ Start-Sleep -Milliseconds 8
+ }
+}
+
+function Get-OrcaVirtualKey([string]$Key) {
+ $normalized = $Key.ToLowerInvariant()
+ $map = @{
+ "return" = 0x0D; "enter" = 0x0D; "tab" = 0x09; "escape" = 0x1B; "esc" = 0x1B
+ "backspace" = 0x08; "delete" = 0x2E; "space" = 0x20; "left" = 0x25
+ "up" = 0x26; "right" = 0x27; "down" = 0x28; "home" = 0x24; "end" = 0x23
+ }
+ if ($map.ContainsKey($normalized)) { return $map[$normalized] }
+ if ($normalized.Length -eq 1) { return [int][char]$normalized.ToUpperInvariant()[0] }
+ throw "Unsupported key: $Key"
+}
+
+function Send-OrcaKey([IntPtr]$WindowHandle, [string]$Key) {
+ $virtualKey = Get-OrcaVirtualKey $Key
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$virtualKey, [IntPtr]::Zero)
+ Start-Sleep -Milliseconds 25
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$virtualKey, [IntPtr]::Zero)
+}
+
+function Get-OrcaModifierVirtualKey([string]$Modifier) {
+ switch ($Modifier.ToLowerInvariant()) {
+ { $_ -in @("ctrl", "control", "cmdorctrl", "commandorcontrol") } { return 0x11 }
+ { $_ -in @("shift") } { return 0x10 }
+ { $_ -in @("alt", "option") } { return 0x12 }
+ { $_ -in @("meta", "super", "win", "cmd", "command") } { return 0x5B }
+ default { throw "Unsupported modifier: $Modifier" }
+ }
+}
+
+function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) {
+ $parts = @($KeySpec.Split("+") | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
+ if ($parts.Count -eq 0) { throw "Unsupported key: $KeySpec" }
+ $key = $parts[$parts.Count - 1]
+ $modifiers = @()
+ if ($parts.Count -gt 1) {
+ $modifiers = @($parts[0..($parts.Count - 2)] | ForEach-Object { Get-OrcaModifierVirtualKey $_ })
+ }
+ foreach ($modifier in $modifiers) {
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$modifier, [IntPtr]::Zero)
+ }
+ Send-OrcaKey $WindowHandle $key
+ for ($i = $modifiers.Count - 1; $i -ge 0; $i--) {
+ [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$modifiers[$i], [IntPtr]::Zero)
+ }
+}
+
+function Send-OrcaPasteText([IntPtr]$WindowHandle, [string]$Text) {
+ $previous = $null
+ $hadPrevious = $false
+ try { $previous = [System.Windows.Forms.Clipboard]::GetDataObject() } catch {}
+ $hadPrevious = $null -ne $previous
+ try {
+ Set-Clipboard -Value $Text
+ Send-OrcaHotkey $WindowHandle "Ctrl+v"
+ } finally {
+ if ($hadPrevious) {
+ [System.Windows.Forms.Clipboard]::SetDataObject($previous, $true)
+ } else {
+ [System.Windows.Forms.Clipboard]::Clear()
+ }
+ }
+}
+
+function Invoke-OrcaOperation($Operation) {
+ $includeScreenshot = -not [bool]$Operation.noScreenshot
+ if ($Operation.tool -eq "handshake") {
+ return [pscustomobject]@{ ok = $true; capabilities = Get-OrcaHandshake }
+ }
+ if ($Operation.tool -eq "list_apps") {
+ return [pscustomobject]@{ ok = $true; apps = @(Get-OrcaAppList) }
+ }
+ if ($Operation.tool -eq "list_windows") {
+ $list = Get-OrcaWindowList $Operation.app
+ return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) }
+ }
+ if ($Operation.tool -eq "get_app_state") {
+ return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) }
+ }
+
+ $process = Find-OrcaProcess $Operation.app
+ if ([bool]$Operation.restoreWindow) { Restore-OrcaWindow $process }
+ Assert-OrcaWindowTarget $process $Operation.windowId $Operation.windowIndex
+ $root = Get-OrcaRootElement $process
+ $windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $process $root }
+ $element = Find-OrcaElement $root $Operation.element
+ $fromElement = Find-OrcaElement $root $Operation.fromElement
+ $toElement = Find-OrcaElement $root $Operation.toElement
+ $handle = [IntPtr]$process.MainWindowHandle
+ if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text")) {
+ Assert-OrcaKeyboardFocus $handle $Operation
+ }
+ $action = $null
+
+ switch ($Operation.tool) {
+ "click" {
+ $handledByPattern = $false
+ if ($null -ne $element -and $Operation.mouse_button -ne "right" -and $Operation.mouse_button -ne "middle" -and [int]$Operation.click_count -le 1) {
+ $handledByPattern = Invoke-OrcaPrimaryAction $element
+ }
+ if (-not $handledByPattern) {
+ $point = Get-OrcaElementScreenPoint $element
+ if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
+ Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button ([int]$Operation.click_count)
+ $action = [pscustomobject]@{ path = "synthetic"; actionName = $null; fallbackReason = "actionUnsupported" }
+ } else {
+ $action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }
+ }
+ }
+ "perform_secondary_action" {
+ if ($null -eq $element) { throw "unknown element_index" }
+ if (-not (Invoke-OrcaNamedAction $element $Operation.action)) {
+ throw "$($Operation.action) is not a valid secondary action"
+ }
+ $action = [pscustomobject]@{ path = "accessibility"; actionName = $Operation.action; fallbackReason = $null }
+ }
+ "scroll" {
+ $delta = 120 * [int][Math]::Max(1, [Math]::Ceiling([double]$Operation.pages))
+ if ($Operation.direction -eq "down" -or $Operation.direction -eq "right") { $delta = -1 * $delta }
+ $point = Get-OrcaElementScreenPoint $element
+ if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
+ [void][OrcaDesktopWin32]::PostMessage($handle, $WindowsMessages.Wheel, (ConvertTo-OrcaWheelParam $delta), (ConvertTo-OrcaLParam $point.x $point.y))
+ $action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }
+ }
+ "drag" {
+ $from = Get-OrcaElementScreenPoint $fromElement
+ if ($null -eq $from -and $null -ne $Operation.fromElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
+ if ($null -eq $from) { $from = @{ x = $windowFrame.x + [double]$Operation.from_x; y = $windowFrame.y + [double]$Operation.from_y } }
+ $to = Get-OrcaElementScreenPoint $toElement
+ if ($null -eq $to -and $null -ne $Operation.toElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
+ if ($null -eq $to) { $to = @{ x = $windowFrame.x + [double]$Operation.to_x; y = $windowFrame.y + [double]$Operation.to_y } }
+ Send-OrcaDrag $handle $from $to
+ $action = [pscustomobject]@{ path = "synthetic"; actionName = "drag"; fallbackReason = $null }
+ }
+ "type_text" {
+ Send-OrcaText $handle ([string]$Operation.text)
+ $action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null }
+ }
+ "press_key" {
+ Send-OrcaKey $handle ([string]$Operation.key)
+ $action = [pscustomobject]@{ path = "synthetic"; actionName = "pressKey"; fallbackReason = $null }
+ }
+ "hotkey" {
+ Send-OrcaHotkey $handle ([string]$Operation.key)
+ $action = [pscustomobject]@{ path = "synthetic"; actionName = "hotkey"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
+ }
+ "paste_text" {
+ Send-OrcaPasteText $handle ([string]$Operation.text)
+ $action = [pscustomobject]@{ path = "clipboard"; actionName = "paste"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "clipboard_paste" } }
+ }
+ "set_value" {
+ if ($null -eq $element -or -not (Set-OrcaElementValue $element ([string]$Operation.value))) {
+ throw "element value is not settable"
+ }
+ $action = [pscustomobject]@{ path = "accessibility"; actionName = "setValue"; fallbackReason = $null }
+ }
+ default {
+ throw "unsupported tool: $($Operation.tool)"
+ }
+ }
+
+ try {
+ $snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex
+ } catch {
+ if ($null -eq $Operation.windowId -and $null -eq $Operation.windowIndex) { throw }
+ if ($null -eq $action.verification) {
+ $action | Add-Member -NotePropertyName verification -NotePropertyValue ([pscustomobject]@{ state = "unverified"; reason = "window_changed" })
+ }
+ $snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $null $null
+ }
+ [pscustomobject]@{ ok = $true; action = $action; snapshot = $snapshot }
+}
+
+try {
+ $operation = Read-OrcaOperation $OperationPath
+ Write-OrcaJson (Invoke-OrcaOperation $operation)
+} catch {
+ Write-OrcaJson ([pscustomobject]@{ ok = $false; error = [string]$_.Exception.Message })
+}
diff --git a/package.json b/package.json
index 3d8735c0c3a..ec81a4352dd 100644
--- a/package.json
+++ b/package.json
@@ -25,10 +25,13 @@
"start": "electron-vite preview",
"dev": "node config/scripts/run-electron-vite-dev.mjs",
"build:relay": "node config/scripts/build-relay.mjs",
+ "build:computer-macos": "node config/scripts/build-computer-macos.mjs",
+ "smoke:computer": "node config/scripts/computer-use-smoke.mjs",
+ "verify:computer-native": "node config/scripts/verify-computer-native.mjs",
"build:cli": "tsc -p config/tsconfig.cli.json --outDir out --composite false --incremental false && node config/scripts/install-dev-cli.mjs",
"build:electron-vite": "node config/scripts/run-electron-vite-build.mjs",
- "build": "pnpm run typecheck && pnpm run build:relay && pnpm run build:electron-vite && pnpm run build:cli",
- "build:release": "pnpm run build:relay && pnpm run build:electron-vite && pnpm run build:cli",
+ "build": "pnpm run typecheck && pnpm run build:relay && pnpm run build:computer-macos && pnpm run build:electron-vite && pnpm run build:cli",
+ "build:release": "pnpm run build:relay && pnpm run build:computer-macos && pnpm run verify:computer-native && pnpm run build:electron-vite && pnpm run build:cli",
"postinstall": "pnpm rebuild electron && node config/scripts/rebuild-native-deps.mjs",
"rebuild:electron": "node config/scripts/rebuild-native-deps.mjs",
"build:unpack": "pnpm run build && electron-builder --config config/electron-builder.config.cjs --dir",
@@ -38,7 +41,8 @@
"build:mac:release": "node config/scripts/verify-macos-release-env.mjs && ORCA_MAC_RELEASE=1 pnpm run build && ORCA_MAC_RELEASE=1 electron-builder --config config/electron-builder.config.cjs --mac",
"build:linux": "pnpm run build && electron-builder --config config/electron-builder.config.cjs --linux",
"test:e2e": "npx playwright test --config tests/playwright.config.ts --project electron-headless",
- "test:e2e:headful": "npx playwright test --config tests/playwright.config.ts --project electron-headful"
+ "test:e2e:headful": "npx playwright test --config tests/playwright.config.ts --project electron-headful",
+ "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts"
},
"dependencies": {
"@dnd-kit/core": "^6.3.1",
diff --git a/resources/build/entitlements.computer-use.mac.plist b/resources/build/entitlements.computer-use.mac.plist
new file mode 100644
index 00000000000..6631ffa6f24
--- /dev/null
+++ b/resources/build/entitlements.computer-use.mac.plist
@@ -0,0 +1,6 @@
+
+
+
+
+
+
diff --git a/skills/computer-use/SKILL.md b/skills/computer-use/SKILL.md
new file mode 100644
index 00000000000..e44e1b5d2ff
--- /dev/null
+++ b/skills/computer-use/SKILL.md
@@ -0,0 +1,192 @@
+---
+name: computer-use
+description: Use Orca's computer-use CLI to inspect and control local desktop apps through accessibility trees, screenshots, and safe UI actions. Use when an agent needs to list desktop apps, get an app state, read visible UI, click, type, press keys, scroll, drag, set values, or perform app accessibility actions. Triggers include "computer use", "orca computer", "list apps", "get app state", "read Spotify", "read Slack", "click app", "type text", "press key", "set value", "scroll app", "drag app", and desktop app interaction tasks.
+---
+
+# Computer Use
+
+Use this skill when the task should operate through Orca's desktop computer-use surface rather than native Codex computer tools, raw AppleScript, ad hoc screenshots, or direct app internals.
+
+## Preconditions
+
+- Prefer the public `orca computer ...` command.
+- In this Orca worktree, use `./config/scripts/orca-dev computer ...` when testing the local dev runtime.
+- Prefer `--json` for agent-driven calls. Screenshot image bytes are omitted from JSON and written to `screenshot.path` when present.
+- Do not push, submit forms, send messages, buy items, delete data, or change account settings unless the user explicitly asked for that specific action.
+- If an app contains sensitive content, read only what the user requested and avoid unnecessary screenshots or logs.
+
+Check runtime availability first:
+
+```bash
+orca status --json
+orca computer capabilities --json
+```
+
+For local development against this worktree:
+
+```bash
+./config/scripts/orca-dev status --json
+```
+
+## Core Workflow
+
+Use a snapshot-act-snapshot loop:
+
+1. Discover apps:
+
+```bash
+orca computer list-apps --json
+```
+
+2. Get a fresh state for the target app:
+
+```bash
+orca computer get-app-state --app com.spotify.client --json
+```
+
+3. Choose an element from that state.
+
+4. Perform one action:
+
+```bash
+orca computer click --app com.spotify.client --element-index 42 --json
+```
+
+5. Inspect the action result before deciding whether to act again. Actions return a fresh state:
+
+```bash
+orca computer click --app com.spotify.client --element-index 42 --json
+```
+
+Element indexes are scoped to the current app state. They can go stale after navigation, focus changes, scrolling, window changes, or app re-rendering. Never carry indexes across unrelated steps without refreshing state.
+
+## App Selectors
+
+Prefer bundle IDs returned by `list-apps`:
+
+```bash
+orca computer get-app-state --app com.microsoft.edgemac --json
+orca computer get-app-state --app com.spotify.client --json
+```
+
+Names are acceptable when unambiguous:
+
+```bash
+orca computer get-app-state --app Spotify --json
+```
+
+Use `pid:` only when bundle ID or name matching is ambiguous:
+
+```bash
+orca computer get-app-state --app pid:12345 --json
+```
+
+## Commands
+
+```bash
+orca computer permissions --json
+orca computer capabilities --json
+orca computer list-apps --json
+orca computer list-windows --app --json
+orca computer get-app-state --app --json
+orca computer click --app --element-index --json
+orca computer perform-secondary-action --app --element-index --action --json
+orca computer set-value --app --element-index --value "text" --json
+orca computer type-text --app --text "text" --json
+orca computer press-key --app --key Return --json
+orca computer hotkey --app --key CmdOrCtrl+A --json
+orca computer paste-text --app --text "text" --json
+orca computer scroll --app (--element-index | --x --y ) --direction down --json
+orca computer drag --app --from-x 100 --from-y 100 --to-x 300 --to-y 300 --json
+```
+
+Use `--no-screenshot` only when pixels are not needed. Screenshots are often the only useful signal for Electron, WebView, or canvas-heavy apps with shallow accessibility trees.
+
+Coordinates are window-local. Use coordinates from the latest screenshot/state for the same target window.
+
+Use `--text-stdin` or `--value-stdin` for sensitive text so payloads do not land in shell history.
+On Linux and Windows, action payloads still pass through a short-lived local operation file.
+
+```bash
+printf '%s' "$TEXT" | orca computer set-value --app --element-index --value-stdin --json
+```
+
+## Choosing Actions
+
+Prefer semantic actions over raw keyboard input:
+
+- Use `set-value` for known editable fields.
+- Use `click` for buttons, tabs, menu items, checkboxes, and other direct controls.
+- Use `perform-secondary-action` only when the state lists a concrete action name and the user intent matches it.
+- Use `type-text` after focusing a field and confirming the app has a focused text receiver.
+- Use `press-key` for navigation keys, Return, Escape, shortcuts, or submitting a field after the state confirms the right target is active.
+
+Why: keyboard input is process-targeted on macOS, but it still depends on the target app having a valid focused receiver. `set-value` targets the accessibility element directly and is more reliable when supported.
+
+## Foreground And Background
+
+Some actions work while the app is in the background. Treat this as app-dependent:
+
+- `set-value` can work in the background when the app exposes a writable accessibility value.
+- `click` and accessibility actions may work in the background for some native controls.
+- `type-text` and `press-key` are targeted to the app process on macOS, but the app may ignore them unless it owns focus or already has an active text receiver.
+
+If an action returns success but the UI did not change, do not repeat the same action blindly. Run `get-app-state` again, inspect the screenshot/tree, then switch to a more semantic action or bring/focus the target if needed.
+
+## Screenshots
+
+`get-app-state` returns an accessibility tree and, by default, a screenshot. Use both:
+
+- Trust the tree for element indexes, names, roles, values, and actions.
+- Trust the screenshot for visual confirmation, especially in Electron and WebView apps.
+- If the tree is shallow, use screenshot evidence before deciding whether any action is safe.
+- If screenshot capture fails or returns no image, the app may be hidden, minimized, off-screen, or have no visible window.
+
+Use restore only when appropriate for the task:
+
+```bash
+orca computer get-app-state --app --restore-window --json
+```
+
+## App-Specific Notes
+
+### Browsers
+
+For Edge, Chrome, and similar browsers, prefer setting the address/search field directly:
+
+```bash
+orca computer get-app-state --app com.microsoft.edgemac --json
+orca computer set-value --app com.microsoft.edgemac --element-index --value "test123" --json
+orca computer press-key --app com.microsoft.edgemac --key Return --json
+orca computer get-app-state --app com.microsoft.edgemac --json
+```
+
+Do not assume raw typing went to the address bar. Confirm the field or page changed after pressing Return.
+
+### Spotify
+
+Spotify state can update asynchronously after playback or network-backed search. After a playback click, run `get-app-state` before clicking again.
+
+For search, prefer `set-value` on the search combobox, usually named like `What do you want to play?`. `type-text` may only work when Spotify owns focus and that field is already focused.
+
+### Slack
+
+Slack may expose a shallow accessibility tree while the screenshot contains the useful information. Reading visible Slack UI is acceptable when requested, but do not send messages or trigger workflows unless explicitly asked.
+
+## Error Handling
+
+- `app_not_found`: run `list-apps` and retry with the bundle ID.
+- `element_not_found`: the index is stale; run `get-app-state` again.
+- `action_failed`: inspect the element role/actions and try a more semantic action.
+- Empty tree or no screenshot: the app may have no visible window, be minimized, or be blocked by permissions.
+- Permission errors: the user needs to grant Accessibility or Screen Recording to `Orca Computer Use`. Run `orca computer permissions --json`, use the setup UI, then retry `orca computer get-app-state --app --json`.
+
+## Safety Checks
+
+Before acting, classify the action:
+
+- Safe: read state, list apps, inspect screenshot, focus a search box, scroll, open a harmless tab.
+- Needs care: typing into a focused field, pressing Return, clicking a primary button.
+- Requires explicit user permission: sending messages, posting, purchasing, deleting, submitting forms, changing settings, signing in, or exposing secrets.
+
+When uncertain, stop after `get-app-state` and report what is visible instead of acting.
diff --git a/src/cli/args.test.ts b/src/cli/args.test.ts
new file mode 100644
index 00000000000..868e66516e6
--- /dev/null
+++ b/src/cli/args.test.ts
@@ -0,0 +1,13 @@
+import { describe, expect, it } from 'vitest'
+
+import { parseArgs } from './args'
+
+describe('parseArgs', () => {
+ it('keeps an empty string as a flag value', () => {
+ const parsed = parseArgs(['computer', 'set-value', '--value', '', '--json'])
+
+ expect(parsed.commandPath).toEqual(['computer', 'set-value'])
+ expect(parsed.flags.get('value')).toBe('')
+ expect(parsed.flags.get('json')).toBe(true)
+ })
+})
diff --git a/src/cli/args.ts b/src/cli/args.ts
index 45413a580f1..de0aff3c320 100644
--- a/src/cli/args.ts
+++ b/src/cli/args.ts
@@ -28,8 +28,9 @@ export function parseArgs(argv: string[]): ParsedArgs {
}
const flag = token.slice(2)
+ const hasNext = i + 1 < argv.length
const next = argv[i + 1]
- if (!next || next.startsWith('--')) {
+ if (!hasNext || next.startsWith('--')) {
flags.set(flag, true)
continue
}
@@ -61,7 +62,7 @@ export function supportsBrowserPageFlag(commandPath: string[]): boolean {
if (['open', 'status'].includes(commandPath[0])) {
return false
}
- if (['repo', 'worktree', 'terminal'].includes(commandPath[0])) {
+ if (['repo', 'worktree', 'terminal', 'computer'].includes(commandPath[0])) {
return false
}
return ![
@@ -90,7 +91,8 @@ export function isCommandGroup(commandPath: string[]): boolean {
'clipboard',
'dialog',
'storage',
- 'orchestration'
+ 'orchestration',
+ 'computer'
].includes(commandPath[0])) ||
(commandPath.length === 2 &&
commandPath[0] === 'storage' &&
diff --git a/src/cli/dispatch.ts b/src/cli/dispatch.ts
index 6dd41e3a554..8ea7abf7289 100644
--- a/src/cli/dispatch.ts
+++ b/src/cli/dispatch.ts
@@ -13,6 +13,7 @@ import { BROWSER_CAPTURE_HANDLERS } from './handlers/browser-capture'
import { BROWSER_ENV_HANDLERS } from './handlers/browser-env'
import { BROWSER_STORAGE_HANDLERS } from './handlers/browser-storage'
import { ORCHESTRATION_HANDLERS } from './handlers/orchestration'
+import { COMPUTER_HANDLERS } from './handlers/computer'
export type HandlerContext = {
flags: Map
@@ -38,7 +39,8 @@ function buildHandlers(): Map {
BROWSER_CAPTURE_HANDLERS,
BROWSER_ENV_HANDLERS,
BROWSER_STORAGE_HANDLERS,
- ORCHESTRATION_HANDLERS
+ ORCHESTRATION_HANDLERS,
+ COMPUTER_HANDLERS
]
for (const group of groups) {
for (const [key, handler] of Object.entries(group)) {
diff --git a/src/cli/flags.test.ts b/src/cli/flags.test.ts
new file mode 100644
index 00000000000..020d7827a20
--- /dev/null
+++ b/src/cli/flags.test.ts
@@ -0,0 +1,11 @@
+import { describe, expect, it } from 'vitest'
+
+import { getRequiredStringFlagAllowingEmpty } from './flags'
+
+describe('CLI flags', () => {
+ it('allows required string flags to be empty when the command opts in', () => {
+ const flags = new Map([['value', '']])
+
+ expect(getRequiredStringFlagAllowingEmpty(flags, 'value')).toBe('')
+ })
+})
diff --git a/src/cli/flags.ts b/src/cli/flags.ts
index 3c3383dd68d..ed8b90f9a4b 100644
--- a/src/cli/flags.ts
+++ b/src/cli/flags.ts
@@ -8,6 +8,17 @@ export function getRequiredStringFlag(flags: Map, name
throw new RuntimeClientError('invalid_argument', `Missing required --${name}`)
}
+export function getRequiredStringFlagAllowingEmpty(
+ flags: Map,
+ name: string
+): string {
+ const value = flags.get(name)
+ if (typeof value === 'string') {
+ return value
+ }
+ throw new RuntimeClientError('invalid_argument', `Missing required --${name}`)
+}
+
export function getOptionalStringFlag(
flags: Map,
name: string
diff --git a/src/cli/format.ts b/src/cli/format.ts
index 2ea429e13a1..bff5a880505 100644
--- a/src/cli/format.ts
+++ b/src/cli/format.ts
@@ -1,3 +1,7 @@
+/* eslint-disable max-lines -- Why: CLI result formatters are centralized so handlers can stay thin RPC glue. */
+import { chmodSync, lstatSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
import type {
BrowserProfileListResult,
BrowserTabCurrentResult,
@@ -7,6 +11,11 @@ import type {
BrowserTabProfileCloneResult,
BrowserTabProfileShowResult,
BrowserTabShowResult,
+ ComputerActionResult,
+ ComputerActionVerification,
+ ComputerListAppsResult,
+ ComputerListWindowsResult,
+ ComputerSnapshotResult,
CliStatusResult,
RuntimeRepoList,
RuntimeRepoSearchRefs,
@@ -33,7 +42,7 @@ export function printResult(
formatter: (value: TResult) => string
): void {
if (json) {
- console.log(JSON.stringify(response, null, 2))
+ console.log(JSON.stringify(prepareCliJsonResult(response), null, 2))
return
}
console.log(formatter(response.result))
@@ -298,3 +307,193 @@ export function formatTabProfileShow(result: BrowserTabProfileShowResult): strin
export function formatTabProfileClone(result: BrowserTabProfileCloneResult): string {
return `Cloned ${result.sourceBrowserPageId} to ${result.browserPageId} (${result.profileLabel ?? result.profileId ?? 'default'})`
}
+
+export function formatGetAppState(result: ComputerSnapshotResult): string {
+ const app = result.snapshot.app
+ const bundle = app.bundleId ? `, ${app.bundleId}` : ''
+ const focused =
+ result.snapshot.focusedElementId === null ? 'none' : `#${result.snapshot.focusedElementId}`
+ const windowId =
+ result.snapshot.window.id === null || result.snapshot.window.id === undefined
+ ? ''
+ : ` id:${result.snapshot.window.id}`
+ const origin =
+ result.snapshot.window.x === null ||
+ result.snapshot.window.x === undefined ||
+ result.snapshot.window.y === null ||
+ result.snapshot.window.y === undefined
+ ? ''
+ : ` @ ${result.snapshot.window.x},${result.snapshot.window.y}`
+ const truncation = result.snapshot.truncation?.truncated
+ ? ` Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'})`
+ : ' Truncated: no'
+ return [
+ `${app.name} (pid ${app.pid}${bundle})`,
+ ` Window:${windowId} "${result.snapshot.window.title}" (${result.snapshot.window.width}x${result.snapshot.window.height}${origin})`,
+ ` Elements: ${result.snapshot.elementCount} Focused: ${focused} Coordinates: ${result.snapshot.coordinateSpace}`,
+ truncation,
+ ` ${formatComputerScreenshotStatus(result)}`,
+ '',
+ result.snapshot.treeText
+ ].join('\n')
+}
+
+function prepareCliJsonResult(
+ response: RuntimeRpcSuccess
+): RuntimeRpcSuccess {
+ const record = response as RuntimeRpcSuccess & {
+ result?: { screenshot?: { data?: unknown; format?: unknown; path?: unknown } | null }
+ }
+ const screenshot = record.result?.screenshot
+ if (!screenshot || typeof screenshot.data !== 'string' || screenshot.data.length === 0) {
+ return response
+ }
+ const extension = screenshot.format === 'png' ? 'png' : 'img'
+ const outputDir = computerScreenshotTempDir()
+ cleanupComputerScreenshots(outputDir)
+ const outputPath = join(outputDir, `${safeCliFileStem(response.id)}-screenshot.${extension}`)
+ writeFileSync(outputPath, Buffer.from(screenshot.data, 'base64'), { mode: 0o600 })
+ const expiresAt = new Date(Date.now() + COMPUTER_SCREENSHOT_TTL_MS).toISOString()
+ return {
+ ...response,
+ result: {
+ ...record.result,
+ screenshot: {
+ ...screenshot,
+ data: undefined,
+ path: outputPath,
+ dataOmitted: true,
+ expiresAt
+ }
+ }
+ } as RuntimeRpcSuccess
+}
+
+const COMPUTER_SCREENSHOT_TTL_MS = 24 * 60 * 60 * 1000
+
+function computerScreenshotTempDir(): string {
+ const outputDir = join(tmpdir(), 'orca-computer-use')
+ mkdirSync(outputDir, { recursive: true, mode: 0o700 })
+ const stat = lstatSync(outputDir)
+ if (!stat.isDirectory() || stat.isSymbolicLink()) {
+ throw new Error(`Unsafe computer screenshot temp path: ${outputDir}`)
+ }
+ if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) {
+ throw new Error(`Computer screenshot temp path is not owned by the current user: ${outputDir}`)
+ }
+ chmodSync(outputDir, 0o700)
+ return outputDir
+}
+
+function cleanupComputerScreenshots(outputDir: string): void {
+ const cutoff = Date.now() - COMPUTER_SCREENSHOT_TTL_MS
+ for (const entry of readdirSync(outputDir)) {
+ if (!entry.endsWith('-screenshot.png') && !entry.endsWith('-screenshot.img')) {
+ continue
+ }
+ const path = join(outputDir, entry)
+ try {
+ if (statSync(path).mtimeMs < cutoff) {
+ rmSync(path, { force: true })
+ }
+ } catch {
+ // Best-effort cleanup only; formatting should not fail because a temp file raced.
+ }
+ }
+}
+
+function safeCliFileStem(value: string): string {
+ return value.replaceAll(/[^a-zA-Z0-9._-]/g, '_')
+}
+
+export function formatListApps(result: ComputerListAppsResult): string {
+ if (result.apps.length === 0) {
+ return 'No apps found.'
+ }
+ return result.apps
+ .map((app) => {
+ const bundle = app.bundleId ? ` ${app.bundleId}` : ''
+ return `${app.name} pid:${app.pid}${bundle}`
+ })
+ .join('\n')
+}
+
+export function formatListWindows(result: ComputerListWindowsResult): string {
+ if (result.windows.length === 0) {
+ return `No windows found for ${result.app.name}.`
+ }
+ return result.windows
+ .map((window) => {
+ const id = window.id === null || window.id === undefined ? 'none' : String(window.id)
+ const origin =
+ window.x === null || window.x === undefined || window.y === null || window.y === undefined
+ ? ''
+ : ` @ ${window.x},${window.y}`
+ const screen =
+ window.screenIndex === null || window.screenIndex === undefined
+ ? ''
+ : ` screen:${window.screenIndex}`
+ const state = [
+ window.isMinimized ? 'minimized' : null,
+ window.isOffscreen ? 'offscreen' : null
+ ].filter(Boolean)
+ const stateText = state.length > 0 ? ` ${state.join(',')}` : ''
+ return `[${window.index}] id:${id} "${window.title}" (${window.width}x${window.height}${origin})${screen}${stateText}`
+ })
+ .join('\n')
+}
+
+export function formatComputerAction(verb: string, result: ComputerActionResult): string {
+ const path = result.action?.path ? ` via ${result.action.path}` : ''
+ const verification = formatActionVerification(result.action?.verification)
+ const app = shellQuote(result.snapshot.app.bundleId ?? result.snapshot.app.name)
+ const windowId =
+ result.snapshot.window.id === null || result.snapshot.window.id === undefined
+ ? ''
+ : ` --window-id ${result.snapshot.window.id}`
+ return `${formatActionVerb(verb)} completed${path}${verification}; ${result.snapshot.elementCount} elements in current window. Use \`orca computer get-app-state --app ${app}${windowId}\` to inspect.`
+}
+
+function formatActionVerification(verification: ComputerActionVerification | undefined): string {
+ if (!verification) {
+ return ''
+ }
+ if (verification.state === 'verified') {
+ return `, verified ${verification.property}`
+ }
+ return `, unverified (${verification.reason.replaceAll('_', ' ')})`
+}
+
+function formatComputerScreenshotStatus(result: ComputerSnapshotResult): string {
+ if (result.screenshotStatus.state === 'captured' && result.screenshot) {
+ const bytes = result.screenshot.data
+ ? `${Math.round(result.screenshot.data.length * 0.75)} bytes`
+ : `saved to ${result.screenshot.path ?? 'temporary file'}`
+ const engine = result.screenshotStatus.metadata?.engine
+ const detail = engine
+ ? `${result.screenshot.format}, ${bytes}, ${engine}`
+ : `${result.screenshot.format}, ${bytes}`
+ return `Screenshot captured (${detail})`
+ }
+ if (result.screenshotStatus.state === 'skipped') {
+ return 'Screenshot skipped (--no-screenshot)'
+ }
+ if (result.screenshotStatus.state === 'failed') {
+ return `Screenshot failed (${result.screenshotStatus.code}): ${result.screenshotStatus.message}`
+ }
+ return 'Screenshot was not captured'
+}
+
+function shellQuote(value: string): string {
+ if (/^[a-zA-Z0-9._:/@-]+$/.test(value)) {
+ return value
+ }
+ return `'${value.replaceAll("'", "'\\''")}'`
+}
+
+function formatActionVerb(verb: string): string {
+ return verb
+ .split('-')
+ .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+ .join(' ')
+}
diff --git a/src/cli/handlers/computer.test.ts b/src/cli/handlers/computer.test.ts
new file mode 100644
index 00000000000..d7e8d07ed9b
--- /dev/null
+++ b/src/cli/handlers/computer.test.ts
@@ -0,0 +1,435 @@
+/* eslint-disable max-lines -- Why: computer CLI coverage shares one mocked runtime setup across command contracts. */
+import path from 'path'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const callMock = vi.fn()
+
+vi.mock('../runtime-client', () => {
+ class RuntimeClient {
+ call = callMock
+ getCliStatus = vi.fn()
+ openOrca = vi.fn()
+ }
+
+ class RuntimeClientError extends Error {
+ readonly code: string
+
+ constructor(code: string, message: string) {
+ super(message)
+ this.code = code
+ }
+ }
+
+ class RuntimeRpcFailureError extends RuntimeClientError {
+ readonly response: unknown
+
+ constructor(response: unknown) {
+ super('runtime_error', 'runtime_error')
+ this.response = response
+ }
+ }
+
+ return {
+ RuntimeClient,
+ RuntimeClientError,
+ RuntimeRpcFailureError
+ }
+})
+
+import { main } from '../index'
+import { buildWorktree, okFixture, queueFixtures, worktreeListFixture } from '../test-fixtures'
+
+describe('orca computer CLI handlers', () => {
+ beforeEach(() => {
+ vi.restoreAllMocks()
+ callMock.mockReset()
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ })
+
+ it('prints group help with all computer subcommands', async () => {
+ await main(['computer', '--help'], '/tmp/repo')
+
+ const output = vi.mocked(console.log).mock.calls[0][0]
+ expect(output).toContain('get-app-state')
+ expect(output).toContain('hotkey')
+ expect(output).toContain('capabilities')
+ expect(output).toContain('list-apps')
+ expect(output).toContain('list-windows')
+ expect(output).toContain('paste-text')
+ expect(output).toContain('permissions')
+ expect(output).toContain('set-value')
+ })
+
+ it('passes list-apps through with a resolved worktree', async () => {
+ queueFixtures(
+ callMock,
+ worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+ okFixture('req_apps', { apps: [] })
+ )
+
+ await main(['computer', 'list-apps', '--json'], '/tmp/repo/src')
+
+ expect(callMock).toHaveBeenNthCalledWith(1, 'worktree.list', { limit: 10_000 })
+ expect(callMock).toHaveBeenNthCalledWith(2, 'computer.listApps', {
+ worktree: `path:${path.resolve('/tmp/repo')}`
+ })
+ })
+
+ it('prints provider capabilities without resolving a worktree', async () => {
+ queueFixtures(
+ callMock,
+ okFixture('req_capabilities', {
+ platform: 'darwin',
+ provider: 'orca-computer-use-macos',
+ providerVersion: '1.0.0',
+ protocolVersion: 1,
+ supports: {
+ apps: { list: true, bundleIds: true, pids: true },
+ windows: { list: true, targetById: true, targetByIndex: true },
+ observation: { screenshot: true, elementFrames: true, annotatedScreenshot: false },
+ actions: { click: true, setValue: true, pasteText: true },
+ surfaces: {}
+ }
+ })
+ )
+
+ await main(['computer', 'capabilities'], '/tmp/repo/src')
+
+ expect(callMock).toHaveBeenCalledTimes(1)
+ expect(callMock).toHaveBeenCalledWith('computer.capabilities', {})
+ expect(vi.mocked(console.log).mock.calls[0][0]).toContain('orca-computer-use-macos')
+ })
+
+ it('opens computer permission setup without resolving a worktree', async () => {
+ queueFixtures(
+ callMock,
+ okFixture('req_permissions', {
+ platform: 'darwin',
+ helperAppPath: '/Applications/Orca Computer Use.app',
+ openedSettings: false,
+ launchedHelper: true
+ })
+ )
+
+ await main(['computer', 'permissions'], '/tmp/repo/src')
+
+ expect(callMock).toHaveBeenCalledTimes(1)
+ expect(callMock).toHaveBeenCalledWith('computer.permissions', {})
+ const output = vi.mocked(console.log).mock.calls[0][0]
+ expect(output).toContain('Opened Orca Computer Use permission setup')
+ expect(output).toContain('/Applications/Orca Computer Use.app')
+ })
+
+ it('passes get-app-state target and observe flags', async () => {
+ queueFixtures(
+ callMock,
+ worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+ okFixture('req_state', sampleSnapshot())
+ )
+
+ await main(
+ [
+ 'computer',
+ 'get-app-state',
+ '--app',
+ 'Finder',
+ '--no-screenshot',
+ '--restore-window',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenNthCalledWith(2, 'computer.getAppState', {
+ app: 'Finder',
+ worktree: `path:${path.resolve('/tmp/repo')}`,
+ noScreenshot: true,
+ restoreWindow: true
+ })
+ })
+
+ it('passes list-windows target and formats window IDs', async () => {
+ queueFixtures(
+ callMock,
+ worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+ okFixture('req_windows', {
+ app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+ windows: [
+ {
+ app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+ index: 0,
+ id: 42,
+ title: 'Recents',
+ x: 10,
+ y: 20,
+ width: 800,
+ height: 600,
+ isMinimized: false,
+ isOffscreen: false,
+ screenIndex: 0
+ }
+ ]
+ })
+ )
+
+ await main(['computer', 'list-windows', '--app', 'Finder'], '/tmp/repo/src')
+
+ expect(callMock).toHaveBeenNthCalledWith(2, 'computer.listWindows', {
+ app: 'Finder',
+ worktree: `path:${path.resolve('/tmp/repo')}`
+ })
+ const output = vi.mocked(console.log).mock.calls[0][0]
+ expect(output).toContain('[0] id:42 "Recents"')
+ })
+
+ it('does not resolve worktree when --session is explicit', async () => {
+ queueFixtures(callMock, okFixture('req_click', sampleSnapshot()))
+
+ await main(
+ [
+ 'computer',
+ 'click',
+ '--session',
+ 'manual',
+ '--app',
+ 'Finder',
+ '--element-index',
+ '3',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenCalledTimes(1)
+ expect(callMock).toHaveBeenCalledWith('computer.click', {
+ session: 'manual',
+ app: 'Finder',
+ elementIndex: 3,
+ x: undefined,
+ y: undefined,
+ clickCount: undefined,
+ mouseButton: undefined,
+ noScreenshot: undefined
+ })
+ })
+
+ it('maps action command flags to RPC payloads', async () => {
+ queueFixtures(
+ callMock,
+ worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+ okFixture('req_drag', sampleSnapshot())
+ )
+
+ await main(
+ [
+ 'computer',
+ 'drag',
+ '--app',
+ 'Finder',
+ '--from-x',
+ '1',
+ '--from-y',
+ '2',
+ '--to-x',
+ '3',
+ '--to-y',
+ '4',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenNthCalledWith(2, 'computer.drag', {
+ app: 'Finder',
+ worktree: `path:${path.resolve('/tmp/repo')}`,
+ fromElementIndex: undefined,
+ toElementIndex: undefined,
+ fromX: 1,
+ fromY: 2,
+ toX: 3,
+ toY: 4,
+ noScreenshot: undefined
+ })
+ })
+
+ it('maps coordinate scroll flags to RPC payloads', async () => {
+ queueFixtures(callMock, okFixture('req_scroll', sampleSnapshot()))
+
+ await main(
+ [
+ 'computer',
+ 'scroll',
+ '--session',
+ 'manual',
+ '--app',
+ 'Finder',
+ '--x',
+ '10',
+ '--y',
+ '20',
+ '--direction',
+ 'down',
+ '--pages',
+ '0.5',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenCalledWith('computer.scroll', {
+ session: 'manual',
+ app: 'Finder',
+ elementIndex: undefined,
+ x: 10,
+ y: 20,
+ direction: 'down',
+ pages: 0.5,
+ noScreenshot: undefined
+ })
+ })
+
+ it('maps hotkey and paste-text command flags to RPC payloads', async () => {
+ queueFixtures(callMock, okFixture('req_hotkey', sampleSnapshot()))
+ await main(
+ [
+ 'computer',
+ 'hotkey',
+ '--session',
+ 'manual',
+ '--app',
+ 'Finder',
+ '--key',
+ 'CmdOrCtrl+L',
+ '--no-screenshot',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenCalledWith('computer.hotkey', {
+ session: 'manual',
+ app: 'Finder',
+ key: 'CmdOrCtrl+L',
+ noScreenshot: true
+ })
+
+ callMock.mockReset()
+ vi.mocked(console.log).mockClear()
+ queueFixtures(callMock, okFixture('req_paste', sampleSnapshot()))
+ await main(
+ [
+ 'computer',
+ 'paste-text',
+ '--session',
+ 'manual',
+ '--app',
+ 'Finder',
+ '--text',
+ 'hello world',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenCalledWith('computer.pasteText', {
+ session: 'manual',
+ app: 'Finder',
+ text: 'hello world',
+ noScreenshot: undefined
+ })
+ })
+
+ it('formats get-app-state without printing screenshot bytes in pretty mode', async () => {
+ queueFixtures(callMock, okFixture('req_state', sampleSnapshot()))
+
+ await main(['computer', 'get-app-state', '--session', 'manual', '--app', 'Finder'], '/tmp/repo')
+
+ const output = vi.mocked(console.log).mock.calls[0][0]
+ expect(output).toContain('Finder (pid 100, com.apple.finder)')
+ expect(output).toContain('App=com.apple.finder')
+ expect(output).not.toContain('base64-data')
+ })
+
+ it('omits screenshot bytes from JSON output and writes a screenshot path', async () => {
+ queueFixtures(callMock, okFixture('req_state', sampleSnapshot()))
+
+ await main(
+ ['computer', 'get-app-state', '--session', 'manual', '--app', 'Finder', '--json'],
+ '/tmp/repo'
+ )
+
+ const output = vi.mocked(console.log).mock.calls[0][0]
+ expect(output).not.toContain('base64-data')
+ const parsed = JSON.parse(output)
+ expect(parsed.result.screenshot).toMatchObject({
+ dataOmitted: true,
+ format: 'png',
+ expiresAt: expect.any(String),
+ path: expect.stringContaining('orca-computer-use/req_state-screenshot.png')
+ })
+ })
+
+ it('shows coordinate space and truncation in pretty state output', async () => {
+ queueFixtures(callMock, okFixture('req_state', sampleSnapshot()))
+
+ await main(['computer', 'get-app-state', '--session', 'manual', '--app', 'Finder'], '/tmp/repo')
+
+ const output = vi.mocked(console.log).mock.calls[0][0]
+ expect(output).toContain('Coordinates: window')
+ expect(output).toContain('Truncated: no')
+ })
+
+ it('passes get-app-state window target flags through', async () => {
+ queueFixtures(callMock, okFixture('req_snapshot', sampleSnapshot()))
+
+ await main(
+ [
+ 'computer',
+ 'get-app-state',
+ '--session',
+ 'manual',
+ '--app',
+ 'Finder',
+ '--window-id',
+ '42',
+ '--window-index',
+ '0',
+ '--no-screenshot',
+ '--json'
+ ],
+ '/tmp/repo/src'
+ )
+
+ expect(callMock).toHaveBeenCalledWith('computer.getAppState', {
+ session: 'manual',
+ app: 'Finder',
+ noScreenshot: true,
+ restoreWindow: undefined,
+ windowId: 42,
+ windowIndex: 0
+ })
+ })
+})
+
+function sampleSnapshot() {
+ return {
+ snapshot: {
+ id: 'snap-test',
+ app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+ window: { title: 'Finder', width: 800, height: 600 },
+ coordinateSpace: 'window',
+ truncation: { truncated: false, maxNodes: 1200, maxDepth: 64, maxDepthReached: false },
+ treeText: 'App=com.apple.finder (pid 100)\n0 standard window Finder',
+ elementCount: 1,
+ focusedElementId: 0
+ },
+ screenshot: {
+ data: 'base64-data',
+ format: 'png',
+ width: 800,
+ height: 600,
+ scale: 1
+ },
+ screenshotStatus: { state: 'captured' }
+ }
+}
diff --git a/src/cli/handlers/computer.ts b/src/cli/handlers/computer.ts
new file mode 100644
index 00000000000..173509381a8
--- /dev/null
+++ b/src/cli/handlers/computer.ts
@@ -0,0 +1,278 @@
+import type {
+ ComputerActionResult,
+ ComputerListAppsResult,
+ ComputerListWindowsResult,
+ ComputerProviderCapabilities,
+ ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import type { CommandHandler } from '../dispatch'
+import {
+ getOptionalNumberFlag,
+ getOptionalNonNegativeIntegerFlag,
+ getOptionalPositiveIntegerFlag,
+ getOptionalStringFlag,
+ getRequiredStringFlagAllowingEmpty,
+ getRequiredStringFlag
+} from '../flags'
+import {
+ formatComputerAction,
+ formatGetAppState,
+ formatListApps,
+ formatListWindows,
+ printResult
+} from '../format'
+import { RuntimeClientError } from '../runtime-client'
+import { getComputerCommandTarget } from '../selectors'
+
+export const COMPUTER_HANDLERS: Record = {
+ 'computer capabilities': async ({ client, json }) => {
+ const result = await client.call('computer.capabilities', {})
+ printResult(result, json, formatComputerCapabilities)
+ },
+ 'computer list-apps': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.listApps', {
+ worktree: target.worktree
+ })
+ printResult(result, json, formatListApps)
+ },
+ 'computer permissions': async ({ client, json }) => {
+ const result = await client.call<{
+ platform: NodeJS.Platform
+ helperAppPath: string | null
+ openedSettings: boolean
+ launchedHelper: boolean
+ permissions?: { id: string; status: string }[]
+ nextStep?: string | null
+ }>('computer.permissions', {})
+ printResult(result, json, (value) => {
+ if (value.platform !== 'darwin') {
+ return 'Computer-use permission setup is only required on macOS.'
+ }
+ const firstLine = value.launchedHelper
+ ? 'Opened Orca Computer Use permission setup.'
+ : 'Computer Use permissions checked.'
+ return [
+ firstLine,
+ `Helper app: ${value.helperAppPath}`,
+ `Permissions: ${value.permissions?.map((permission) => `${permission.id}=${permission.status}`).join(', ') ?? 'unknown'}`,
+ value.nextStep
+ ? `Next: ${value.nextStep}`
+ : 'Computer Use permissions are already granted.',
+ value.launchedHelper
+ ? 'Use the Allow buttons or drag "Orca Computer Use" into the macOS permission list.'
+ : null
+ ]
+ .filter((line) => line !== null)
+ .join('\n')
+ })
+ },
+ 'computer list-windows': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.listWindows', target)
+ printResult(result, json, formatListWindows)
+ },
+ 'computer get-app-state': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.getAppState', {
+ ...target,
+ noScreenshot: flags.has('no-screenshot') ? true : undefined,
+ restoreWindow: flags.has('restore-window') ? true : undefined,
+ windowId: getOptionalNumberFlag(flags, 'window-id'),
+ windowIndex: getOptionalNonNegativeIntegerFlag(flags, 'window-index')
+ })
+ printResult(result, json, formatGetAppState)
+ },
+ 'computer click': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.click', {
+ ...target,
+ elementIndex: getOptionalNonNegativeIntegerFlag(flags, 'element-index'),
+ x: getOptionalNumberFlag(flags, 'x'),
+ y: getOptionalNumberFlag(flags, 'y'),
+ clickCount: getOptionalPositiveIntegerFlag(flags, 'click-count'),
+ mouseButton: getOptionalStringFlag(flags, 'mouse-button'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('click', value))
+ },
+ 'computer perform-secondary-action': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.performSecondaryAction', {
+ ...target,
+ elementIndex: getRequiredNonNegativeIntegerFlag(flags, 'element-index'),
+ action: getRequiredStringFlag(flags, 'action'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('perform-secondary-action', value))
+ },
+ 'computer scroll': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.scroll', {
+ ...target,
+ elementIndex: getOptionalNonNegativeIntegerFlag(flags, 'element-index'),
+ x: getOptionalNumberFlag(flags, 'x'),
+ y: getOptionalNumberFlag(flags, 'y'),
+ direction: getRequiredStringFlag(flags, 'direction'),
+ pages: getOptionalPositiveNumberFlag(flags, 'pages'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('scroll', value))
+ },
+ 'computer drag': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.drag', {
+ ...target,
+ fromElementIndex: getOptionalNonNegativeIntegerFlag(flags, 'from-element-index'),
+ toElementIndex: getOptionalNonNegativeIntegerFlag(flags, 'to-element-index'),
+ fromX: getOptionalNumberFlag(flags, 'from-x'),
+ fromY: getOptionalNumberFlag(flags, 'from-y'),
+ toX: getOptionalNumberFlag(flags, 'to-x'),
+ toY: getOptionalNumberFlag(flags, 'to-y'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('drag', value))
+ },
+ 'computer type-text': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.typeText', {
+ ...target,
+ text: await getTextPayload(flags, 'text'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('type-text', value))
+ },
+ 'computer press-key': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.pressKey', {
+ ...target,
+ key: getRequiredStringFlag(flags, 'key'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('press-key', value))
+ },
+ 'computer hotkey': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.hotkey', {
+ ...target,
+ key: getRequiredStringFlag(flags, 'key'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('hotkey', value))
+ },
+ 'computer paste-text': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.pasteText', {
+ ...target,
+ text: await getTextPayload(flags, 'text'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('paste-text', value))
+ },
+ 'computer set-value': async ({ flags, client, cwd, json }) => {
+ const target = await getComputerCommandTarget(flags, cwd, client)
+ const result = await client.call('computer.setValue', {
+ ...target,
+ elementIndex: getRequiredNonNegativeIntegerFlag(flags, 'element-index'),
+ value: await getTextPayload(flags, 'value'),
+ ...getComputerActionObserveFlags(flags)
+ })
+ printResult(result, json, (value) => formatComputerAction('set-value', value))
+ }
+}
+
+async function getTextPayload(
+ flags: Map,
+ name: 'text' | 'value'
+): Promise {
+ const stdinFlag = `${name}-stdin`
+ if (flags.has(stdinFlag)) {
+ if (flags.has(name)) {
+ throw new RuntimeClientError(
+ 'invalid_argument',
+ `Use either --${name} or --${stdinFlag}, not both`
+ )
+ }
+ return await readStdin()
+ }
+ return name === 'value'
+ ? getRequiredStringFlagAllowingEmpty(flags, name)
+ : getRequiredStringFlag(flags, name)
+}
+
+async function readStdin(): Promise {
+ if (process.stdin.isTTY) {
+ throw new RuntimeClientError('invalid_argument', 'stdin payload requested but stdin is a TTY')
+ }
+ const chunks: Buffer[] = []
+ for await (const chunk of process.stdin) {
+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)))
+ }
+ return Buffer.concat(chunks).toString('utf8')
+}
+
+function getOptionalPositiveNumberFlag(
+ flags: Map,
+ name: string
+): number | undefined {
+ const value = getOptionalNumberFlag(flags, name)
+ if (value === undefined) {
+ return undefined
+ }
+ if (value <= 0) {
+ throw new RuntimeClientError('invalid_argument', `Invalid positive number for --${name}`)
+ }
+ return value
+}
+
+function formatComputerCapabilities(value: ComputerProviderCapabilities): string {
+ return [
+ `${value.provider} (${value.platform}, protocol ${value.protocolVersion})`,
+ ` Apps: list=${value.supports.apps.list} bundleIds=${value.supports.apps.bundleIds} pids=${value.supports.apps.pids}`,
+ ` Windows: list=${value.supports.windows.list} targetById=${value.supports.windows.targetById} targetByIndex=${value.supports.windows.targetByIndex}`,
+ ` Observation: screenshot=${value.supports.observation.screenshot} elementFrames=${value.supports.observation.elementFrames} annotatedScreenshot=${value.supports.observation.annotatedScreenshot}`,
+ ` Actions: ${Object.entries(value.supports.actions)
+ .filter(([, enabled]) => enabled)
+ .map(([name]) => name)
+ .join(', ')}`
+ ].join('\n')
+}
+
+function getRequiredNonNegativeIntegerFlag(
+ flags: Map,
+ name: string
+): number {
+ const value = getOptionalNonNegativeIntegerFlag(flags, name)
+ if (value === undefined) {
+ throw new RuntimeClientError('invalid_argument', `Missing required --${name}`)
+ }
+ return value
+}
+
+function getComputerActionObserveFlags(flags: Map): {
+ noScreenshot?: boolean
+ restoreWindow?: boolean
+ windowId?: number
+ windowIndex?: number
+} {
+ const observeFlags: {
+ noScreenshot?: boolean
+ restoreWindow?: boolean
+ windowId?: number
+ windowIndex?: number
+ } = {
+ noScreenshot: flags.has('no-screenshot') ? true : undefined
+ }
+ if (flags.has('restore-window')) {
+ observeFlags.restoreWindow = true
+ }
+ const windowId = getOptionalNumberFlag(flags, 'window-id')
+ if (windowId !== undefined) {
+ observeFlags.windowId = windowId
+ }
+ const windowIndex = getOptionalNonNegativeIntegerFlag(flags, 'window-index')
+ if (windowIndex !== undefined) {
+ observeFlags.windowIndex = windowIndex
+ }
+ return observeFlags
+}
diff --git a/src/cli/help.ts b/src/cli/help.ts
index 3adb9cc9810..4b34a18a7f9 100644
--- a/src/cli/help.ts
+++ b/src/cli/help.ts
@@ -1,3 +1,4 @@
+/* eslint-disable max-lines -- Why: root and generated command help text live together so CLI discovery stays greppable. */
import type { CommandSpec } from './args'
import { findCommandSpec, isCommandGroup, supportsBrowserPageFlag } from './args'
@@ -56,6 +57,21 @@ Orchestration:
orchestration gate-list List decision gates
orchestration reset Reset orchestration state
+Computer Use:
+ computer permissions Open the macOS permission setup for computer-use
+ computer list-apps List running apps available to computer-use
+ computer list-windows List visible windows for a target app
+ computer get-app-state Capture a compact accessibility snapshot of an app
+ computer click Click an app element or window coordinate
+ computer perform-secondary-action Run an advertised accessibility action
+ computer scroll Scroll an app element
+ computer drag Drag between app elements or window coordinates
+ computer type-text Type literal text at the current app focus
+ computer press-key Press a key using xdotool-style syntax
+ computer hotkey Press a shortcut combination such as CmdOrCtrl+A
+ computer paste-text Paste text through the native clipboard path
+ computer set-value Set the value of a settable app element
+
Browser Automation:
tab create Create a new browser tab (navigates to --url)
tab list List open browser tabs
@@ -290,27 +306,48 @@ export function formatFlagHelp(flag: string): string {
command: '--command Command to run in the terminal on startup',
comment: '--comment Comment stored in Orca metadata',
cursor: '--cursor Line cursor from a previous read (returns only new output)',
- direction: '--direction Direction: horizontal|vertical (split) or up|down (scroll)',
+ action: '--action Secondary accessibility action name',
+ app: '--app App name, bundle ID, or pid:N',
+ direction:
+ '--direction Direction: up|down|left|right for scroll, horizontal|vertical for split',
'display-name': '--display-name Override the Orca display name',
+ 'element-index': '--element-index Element index from get-app-state',
title: '--title Custom title for the terminal tab (omit to reset)',
enter: '--enter Append Enter after sending text',
force: '--force Force worktree removal when supported',
for: '--for exit|tui-idle Wait condition to satisfy',
+ 'from-element-index': '--from-element-index Source element index from get-app-state',
+ 'from-x': '--from-x Source window-local x coordinate',
+ 'from-y': '--from-y Source window-local y coordinate',
help: '--help Show this help message',
interrupt: '--interrupt Send as an interrupt-style input when supported',
issue: '--issue Linked GitHub issue number',
json: '--json Emit machine-readable JSON',
+ key: '--key Key or combo to press, e.g. Escape or CmdOrCtrl+L',
limit: '--limit Maximum number of rows to return',
+ 'mouse-button': '--mouse-button Mouse button: left, right, or middle',
name: '--name Name for the new worktree',
+ 'no-screenshot': '--no-screenshot Skip screenshot capture after the operation',
+ pages: '--pages Number of scroll pages',
path: '--path Filesystem path to the repo',
query: '--query Search text for matching refs',
ref: '--ref [ Base ref to persist for the repo',
repo: '--repo Repo selector such as id:, name:, or path:',
+ 'restore-window':
+ '--restore-window Bring the target app/window forward before the operation',
+ session: '--session Snapshot namespace for a related computer-use workflow',
terminal: '--terminal Runtime-issued terminal handle',
- text: '--text Text to send to the terminal',
+ text: '--text Text payload to send or type',
+ 'text-stdin': '--text-stdin Read text payload from stdin',
'timeout-ms': '--timeout-ms Maximum wait time before timing out',
+ 'to-element-index': '--to-element-index Destination element index from get-app-state',
+ 'to-x': '--to-x Destination window-local x coordinate',
+ 'to-y': '--to-y Destination window-local y coordinate',
worktree:
'--worktree Worktree selector such as id:, branch:, issue:, path:, or active/current',
+ 'value-stdin': '--value-stdin Read set-value payload from stdin',
+ 'window-id': '--window-id Target a window id from list-windows',
+ 'window-index': '--window-index Target a window index from list-windows',
// Browser automation flags
element: '--element ][ Element ref from snapshot (e.g. e3)',
url: '--url URL to navigate to',
diff --git a/src/cli/selectors.ts b/src/cli/selectors.ts
index dfdfa7377a3..14420cebb69 100644
--- a/src/cli/selectors.ts
+++ b/src/cli/selectors.ts
@@ -1,5 +1,5 @@
import { isAbsolute, relative, resolve as resolvePath } from 'path'
-import type { RuntimeWorktreeListResult } from '../shared/runtime-types'
+import type { ComputerAppQuery, RuntimeWorktreeListResult } from '../shared/runtime-types'
import type { RuntimeClient } from './runtime-client'
import { RuntimeClientError } from './runtime-client'
import { getOptionalStringFlag, getRequiredStringFlag } from './flags'
@@ -9,6 +9,12 @@ export type BrowserCliTarget = {
page?: string
}
+export type ComputerCliTarget = {
+ session?: string
+ worktree?: string
+ app?: ComputerAppQuery
+}
+
export function buildCurrentWorktreeSelector(cwd: string): string {
return `path:${resolvePath(cwd)}`
}
@@ -150,3 +156,19 @@ export async function getBrowserCommandTarget(
worktree: normalizeWorktreeSelector(explicitWorktree, cwd)
}
}
+
+export async function getComputerCommandTarget(
+ flags: Map,
+ cwd: string,
+ client: RuntimeClient
+): Promise {
+ const app = getOptionalStringFlag(flags, 'app')
+ const session = getOptionalStringFlag(flags, 'session')
+ if (session) {
+ return { session, app }
+ }
+ return {
+ app,
+ worktree: await getBrowserWorktreeSelector(flags, cwd, client)
+ }
+}
diff --git a/src/cli/specs/computer.test.ts b/src/cli/specs/computer.test.ts
new file mode 100644
index 00000000000..11f4d4cee74
--- /dev/null
+++ b/src/cli/specs/computer.test.ts
@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest'
+
+import { COMPUTER_COMMAND_SPECS } from './computer'
+
+describe('computer command specs', () => {
+ it('allows explicit window targeting on action commands', () => {
+ const actionSpecs = COMPUTER_COMMAND_SPECS.filter((spec) =>
+ [
+ 'computer click',
+ 'computer drag',
+ 'computer hotkey',
+ 'computer paste-text',
+ 'computer perform-secondary-action',
+ 'computer press-key',
+ 'computer scroll',
+ 'computer set-value',
+ 'computer type-text'
+ ].includes(spec.path.join(' '))
+ )
+
+ expect(actionSpecs).not.toHaveLength(0)
+ for (const spec of actionSpecs) {
+ expect(spec.allowedFlags).toEqual(expect.arrayContaining(['window-id', 'window-index']))
+ }
+ })
+})
diff --git a/src/cli/specs/computer.ts b/src/cli/specs/computer.ts
new file mode 100644
index 00000000000..93366c976b9
--- /dev/null
+++ b/src/cli/specs/computer.ts
@@ -0,0 +1,124 @@
+import type { CommandSpec } from '../args'
+import { GLOBAL_FLAGS } from '../args'
+
+const COMPUTER_FLAGS = [...GLOBAL_FLAGS, 'worktree', 'session', 'app']
+const COMPUTER_WINDOW_TARGET_FLAGS = ['window-id', 'window-index']
+const COMPUTER_ACTION_FLAGS = [
+ ...COMPUTER_FLAGS,
+ ...COMPUTER_WINDOW_TARGET_FLAGS,
+ 'restore-window',
+ 'no-screenshot'
+]
+
+export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
+ {
+ path: ['computer', 'capabilities'],
+ summary: 'Show computer-use provider capabilities',
+ usage: 'orca computer capabilities [--json]',
+ allowedFlags: [...GLOBAL_FLAGS]
+ },
+ {
+ path: ['computer', 'list-apps'],
+ summary: 'List running apps available to computer-use',
+ usage: 'orca computer list-apps [--worktree ] [--json]',
+ allowedFlags: [...GLOBAL_FLAGS, 'worktree']
+ },
+ {
+ path: ['computer', 'permissions'],
+ summary: 'Open computer-use permission setup',
+ usage: 'orca computer permissions [--json]',
+ allowedFlags: [...GLOBAL_FLAGS]
+ },
+ {
+ path: ['computer', 'list-windows'],
+ summary: 'List windows for an app available to computer-use',
+ usage:
+ 'orca computer list-windows --app [--worktree | --session ] [--json]',
+ allowedFlags: COMPUTER_FLAGS
+ },
+ {
+ path: ['computer', 'get-app-state'],
+ summary: 'Capture a compact accessibility snapshot of an app',
+ usage:
+ 'orca computer get-app-state --app [--window-id | --window-index ] [--worktree | --session ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, ...COMPUTER_WINDOW_TARGET_FLAGS, 'restore-window']
+ },
+ {
+ path: ['computer', 'click'],
+ summary: 'Click an app element or window coordinate',
+ usage:
+ 'orca computer click --app (--element-index | --x --y ) [--window-id | --window-index ] [--click-count ] [--mouse-button ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [
+ ...COMPUTER_ACTION_FLAGS,
+ 'element-index',
+ 'x',
+ 'y',
+ 'click-count',
+ 'mouse-button'
+ ]
+ },
+ {
+ path: ['computer', 'perform-secondary-action'],
+ summary: 'Perform an advertised secondary accessibility action',
+ usage:
+ 'orca computer perform-secondary-action --app --element-index --action [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'action']
+ },
+ {
+ path: ['computer', 'scroll'],
+ summary: 'Scroll an app element or window coordinate',
+ usage:
+ 'orca computer scroll --app (--element-index | --x --y ) --direction [--window-id | --window-index ] [--pages ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'x', 'y', 'direction', 'pages']
+ },
+ {
+ path: ['computer', 'drag'],
+ summary: 'Drag between app elements or window coordinates',
+ usage:
+ 'orca computer drag --app (--from-element-index --to-element-index | --from-x --from-y --to-x --to-y ) [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [
+ ...COMPUTER_ACTION_FLAGS,
+ 'from-element-index',
+ 'to-element-index',
+ 'from-x',
+ 'from-y',
+ 'to-x',
+ 'to-y'
+ ]
+ },
+ {
+ path: ['computer', 'type-text'],
+ summary: 'Type literal text at the current app focus',
+ usage:
+ 'orca computer type-text --app (--text | --text-stdin) [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
+ },
+ {
+ path: ['computer', 'press-key'],
+ summary: 'Press a key using xdotool-style syntax',
+ usage:
+ 'orca computer press-key --app --key [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
+ },
+ {
+ path: ['computer', 'hotkey'],
+ summary: 'Press a platform-aware key combination',
+ usage:
+ 'orca computer hotkey --app --key [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
+ },
+ {
+ path: ['computer', 'paste-text'],
+ summary: 'Paste exact text at the current app focus',
+ usage:
+ 'orca computer paste-text --app (--text | --text-stdin) [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
+ },
+ {
+ path: ['computer', 'set-value'],
+ summary: 'Set the value of a settable app element',
+ usage:
+ 'orca computer set-value --app --element-index (--value | --value-stdin) [--window-id | --window-index ] [--restore-window] [--no-screenshot] [--json]',
+ allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'value', 'value-stdin']
+ }
+]
diff --git a/src/cli/specs/index.ts b/src/cli/specs/index.ts
index a29a48b6abf..dad04e7cc6f 100644
--- a/src/cli/specs/index.ts
+++ b/src/cli/specs/index.ts
@@ -3,10 +3,12 @@ import { BROWSER_ADVANCED_COMMAND_SPECS } from './browser-advanced'
import { BROWSER_BASIC_COMMAND_SPECS } from './browser-basic'
import { CORE_COMMAND_SPECS } from './core'
import { ORCHESTRATION_COMMAND_SPECS } from './orchestration'
+import { COMPUTER_COMMAND_SPECS } from './computer'
export const COMMAND_SPECS: CommandSpec[] = [
...CORE_COMMAND_SPECS,
...BROWSER_BASIC_COMMAND_SPECS,
...BROWSER_ADVANCED_COMMAND_SPECS,
- ...ORCHESTRATION_COMMAND_SPECS
+ ...ORCHESTRATION_COMMAND_SPECS,
+ ...COMPUTER_COMMAND_SPECS
]
diff --git a/src/main/computer/desktop-script-provider-client.test.ts b/src/main/computer/desktop-script-provider-client.test.ts
new file mode 100644
index 00000000000..cdc5436dc7e
--- /dev/null
+++ b/src/main/computer/desktop-script-provider-client.test.ts
@@ -0,0 +1,453 @@
+/* eslint-disable max-lines -- Why: desktop provider contract coverage shares one mocked bridge harness. */
+import { execFile } from 'child_process'
+import { readFileSync } from 'fs'
+import { describe, expect, it, vi } from 'vitest'
+import { DesktopScriptProviderClient } from './desktop-script-provider-client'
+
+vi.mock('child_process', () => ({
+ execFile: vi.fn()
+}))
+
+describe('DesktopScriptProviderClient', () => {
+ it('normalizes list-apps responses', async () => {
+ mockBridgeResponse({
+ ok: true,
+ apps: [{ name: 'Notepad', bundleIdentifier: 'notepad', pid: 42 }]
+ })
+
+ const client = new DesktopScriptProviderClient('windows', '/tmp/runtime.ps1')
+
+ await expect(client.listApps()).resolves.toEqual({
+ apps: [
+ {
+ name: 'Notepad',
+ bundleId: 'notepad',
+ pid: 42,
+ isRunning: true,
+ lastUsedAt: null,
+ useCount: null
+ }
+ ]
+ })
+ })
+
+ it('normalizes snapshots and remembers elements for follow-up actions', async () => {
+ mockBridgeResponse({
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+ })
+ mockBridgeResponse({
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+ })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ const initial = await client.snapshot({ app: 'Text Editor' })
+ expect(initial.snapshot.elementCount).toBe(1)
+ expect(initial.snapshot.window).toMatchObject({
+ title: 'Text Editor',
+ id: 99,
+ x: 10,
+ y: 20,
+ width: 300,
+ height: 200
+ })
+ expect(initial.screenshotStatus).toEqual({
+ state: 'captured',
+ metadata: { engine: 'unknown', windowId: 99 }
+ })
+ expect(initial.snapshot.treeText).toContain('initial')
+ expect(publicSnapshotKeys(initial.snapshot)).toEqual([
+ 'app',
+ 'coordinateSpace',
+ 'elementCount',
+ 'focusedElementId',
+ 'id',
+ 'treeText',
+ 'truncation',
+ 'window'
+ ])
+
+ await client.action('setValue', {
+ app: 'Text Editor',
+ elementIndex: 0,
+ value: 'changed',
+ noScreenshot: true
+ })
+
+ const secondCall = vi.mocked(execFile).mock.calls[1]
+ const operationPath = secondCall[1]?.at(-1)
+ expect(typeof operationPath).toBe('string')
+ })
+
+ it('targets cached elements by session and explicit window id', async () => {
+ mockBridgeResponse({
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+ })
+ mockBridgeResponse(
+ {
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+ },
+ (operation) => {
+ expect(operation).toMatchObject({
+ tool: 'drag',
+ app: 'Text Editor',
+ windowId: 99,
+ windowIndex: 0,
+ fromElement: expect.objectContaining({ index: 0 }),
+ toElement: expect.objectContaining({ index: 0 })
+ })
+ expect(operation.from_x).toBeUndefined()
+ expect(operation.to_x).toBeUndefined()
+ expect(operation.windowBounds).toBeNull()
+ }
+ )
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await client.snapshot({ app: 'Text Editor', session: 'agent-a', windowId: 99 })
+ await client.action('drag', {
+ app: 'Text Editor',
+ session: 'agent-a',
+ windowId: 99,
+ windowIndex: 0,
+ fromElementIndex: 0,
+ toElementIndex: 0
+ })
+ })
+
+ it('forwards restore-window to desktop providers', async () => {
+ mockBridgeResponse(
+ {
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+ },
+ (operation) => {
+ expect(operation).toMatchObject({
+ tool: 'get_app_state',
+ restoreWindow: true
+ })
+ }
+ )
+ mockBridgeResponse(
+ {
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+ },
+ (operation) => {
+ expect(operation).toMatchObject({
+ tool: 'set_value',
+ restoreWindow: true
+ })
+ }
+ )
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await client.snapshot({ app: 'Text Editor', restoreWindow: true })
+ const result = await client.action('setValue', {
+ app: 'Text Editor',
+ elementIndex: 0,
+ value: 'changed',
+ restoreWindow: true,
+ noScreenshot: true
+ })
+
+ expect(result.action).toMatchObject({
+ path: 'accessibility',
+ actionName: 'setValue'
+ })
+ })
+
+ it('keeps window-index snapshots scoped to the matching session', async () => {
+ mockBridgeResponse({
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+ })
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await client.snapshot({ app: 'Text Editor', session: 'agent-a', windowIndex: 0 })
+
+ await expect(
+ client.action('click', {
+ app: 'Text Editor',
+ session: 'agent-b',
+ windowIndex: 0,
+ elementIndex: 0
+ })
+ ).rejects.toMatchObject({ code: 'element_not_found' })
+ })
+
+ it('explains screenshot capture failures while keeping accessibility state usable', async () => {
+ mockBridgeResponse({
+ ok: true,
+ snapshot: {
+ ...sampleBridgeSnapshot('Text Editor', 'initial'),
+ screenshotPngBase64: null
+ }
+ })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await expect(client.snapshot({ app: 'Text Editor' })).resolves.toMatchObject({
+ snapshot: {
+ elementCount: 1
+ },
+ screenshotStatus: {
+ state: 'failed',
+ code: 'screenshot_failed',
+ message: expect.stringContaining('--no-screenshot')
+ }
+ })
+ })
+
+ it('normalizes list-windows responses after provider handshake', async () => {
+ mockBridgeResponse({
+ ok: true,
+ capabilities: {
+ platform: 'linux',
+ provider: 'orca-computer-use-linux',
+ providerVersion: '1.0.0',
+ protocolVersion: 1,
+ supports: {
+ apps: { list: true, bundleIds: false, pids: true },
+ windows: {
+ list: true,
+ targetById: true,
+ targetByIndex: true,
+ focus: false,
+ moveResize: false
+ },
+ observation: {
+ screenshot: true,
+ annotatedScreenshot: false,
+ elementFrames: true,
+ ocr: false
+ },
+ actions: {
+ click: true,
+ typeText: true,
+ pressKey: true,
+ hotkey: false,
+ pasteText: false,
+ scroll: true,
+ drag: true,
+ setValue: true,
+ performAction: true
+ },
+ surfaces: { menus: false, dialogs: false, dock: false, menubar: false }
+ }
+ }
+ })
+ mockBridgeResponse({
+ ok: true,
+ app: { name: 'Text Editor', bundleIdentifier: 'Text Editor', pid: 100 },
+ windows: [
+ {
+ index: 0,
+ app: { name: 'Text Editor', bundleIdentifier: 'Text Editor', pid: 100 },
+ id: 99,
+ title: 'Document',
+ x: 10,
+ y: 20,
+ width: 300,
+ height: 200,
+ isMinimized: false,
+ isOffscreen: false,
+ screenIndex: null,
+ platform: { backend: 'at-spi' }
+ }
+ ]
+ })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await expect(client.listWindows({ app: 'Text Editor' })).resolves.toMatchObject({
+ app: { name: 'Text Editor', bundleId: 'Text Editor', pid: 100 },
+ windows: [{ index: 0, id: 99, title: 'Document' }]
+ })
+ })
+
+ it('maps bridge app errors to RuntimeClientError codes', async () => {
+ mockBridgeResponse({ ok: false, error: 'appNotFound("Missing")' })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await expect(client.snapshot({ app: 'Missing' })).rejects.toMatchObject({
+ code: 'app_not_found'
+ })
+ })
+
+ it('maps blocked app bridge errors to a policy error', async () => {
+ mockBridgeResponse({ ok: false, error: 'appBlocked("1Password")' })
+
+ const client = new DesktopScriptProviderClient('windows', '/tmp/runtime.ps1')
+
+ await expect(client.snapshot({ app: '1Password' })).rejects.toMatchObject({
+ code: 'app_blocked'
+ })
+ })
+
+ it('maps action-specific bridge errors to actionable codes', async () => {
+ mockBridgeResponse({ ok: false, error: 'element value is not settable' })
+ mockBridgeResponse({ ok: false, error: 'Raise is not a valid secondary action' })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
+ code: 'value_not_settable'
+ })
+ await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
+ code: 'action_not_supported'
+ })
+ })
+
+ it('rejects actions that the provider does not advertise', async () => {
+ mockBridgeResponse({
+ ok: true,
+ capabilities: sampleCapabilities({
+ hotkey: false,
+ pasteText: false
+ })
+ })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await expect(
+ client.action('pasteText', { app: 'Text Editor', text: 'hello' })
+ ).rejects.toMatchObject({
+ code: 'unsupported_capability',
+ message: expect.stringContaining('actions.pasteText')
+ })
+ })
+
+ it('uses provider action metadata when bridge reports the actual path', async () => {
+ mockBridgeResponse({
+ ok: true,
+ snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+ })
+ mockBridgeResponse({
+ ok: true,
+ action: {
+ path: 'accessibility',
+ actionName: 'Press',
+ fallbackReason: null
+ },
+ snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+ })
+
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await client.snapshot({ app: 'Text Editor' })
+ await expect(
+ client.action('click', {
+ app: 'Text Editor',
+ elementIndex: 0,
+ noScreenshot: true
+ })
+ ).resolves.toMatchObject({
+ action: {
+ path: 'accessibility',
+ actionName: 'Press'
+ }
+ })
+ })
+
+ it('explains that missing element indexes require a fresh snapshot', async () => {
+ const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+ await expect(
+ client.action('click', { app: 'Text Editor', elementIndex: 4 })
+ ).rejects.toMatchObject({
+ code: 'element_not_found',
+ message: expect.stringContaining('run get-app-state again')
+ })
+ })
+})
+
+function mockBridgeResponse(
+ response: unknown,
+ inspectOperation?: (operation: Record) => void
+): void {
+ vi.mocked(execFile).mockImplementationOnce((_command, _args, _options, callback) => {
+ const operationPath = _args?.at(-1)
+ if (inspectOperation && typeof operationPath === 'string') {
+ inspectOperation(JSON.parse(readFileSync(operationPath, 'utf8')) as Record)
+ }
+ const done = callback as (error: Error | null, stdout: string, stderr: string) => void
+ done(null, JSON.stringify(response), '')
+ return null as never
+ })
+}
+
+function sampleBridgeSnapshot(name: string, value: string) {
+ return {
+ app: { name, bundleIdentifier: name, pid: 100 },
+ snapshotId: 'snap-test',
+ windowTitle: name,
+ windowId: 99,
+ windowBounds: { x: 10, y: 20, width: 300, height: 200 },
+ screenshotPngBase64: 'iVBORw0KGgo=',
+ coordinateSpace: 'window',
+ truncation: { truncated: false, maxNodes: 1200, maxDepth: 64, maxDepthReached: false },
+ treeLines: [`0 text entry area, Value: ${value}`],
+ focusedSummary: 'text entry area',
+ elements: [
+ {
+ index: 0,
+ runtimeId: [0, 0],
+ name: 'Body',
+ controlType: 'text',
+ localizedControlType: 'text entry area',
+ value,
+ frame: { x: 1, y: 2, width: 100, height: 20 },
+ actions: ['SetValue']
+ }
+ ]
+ }
+}
+
+function sampleCapabilities(actions: Partial> = {}) {
+ return {
+ platform: 'linux',
+ provider: 'orca-computer-use-linux',
+ providerVersion: '1.0.0',
+ protocolVersion: 1,
+ supports: {
+ apps: { list: true, bundleIds: false, pids: true },
+ windows: {
+ list: true,
+ targetById: true,
+ targetByIndex: true,
+ focus: false,
+ moveResize: false
+ },
+ observation: {
+ screenshot: true,
+ annotatedScreenshot: false,
+ elementFrames: true,
+ ocr: false
+ },
+ actions: {
+ click: true,
+ typeText: true,
+ pressKey: true,
+ hotkey: true,
+ pasteText: true,
+ scroll: true,
+ drag: true,
+ setValue: true,
+ performAction: true,
+ ...actions
+ },
+ surfaces: { menus: false, dialogs: false, dock: false, menubar: false }
+ }
+ }
+}
+
+function publicSnapshotKeys(snapshot: unknown): string[] {
+ return Object.keys(snapshot as Record).sort()
+}
diff --git a/src/main/computer/desktop-script-provider-client.ts b/src/main/computer/desktop-script-provider-client.ts
new file mode 100644
index 00000000000..7d0c7a4a8a0
--- /dev/null
+++ b/src/main/computer/desktop-script-provider-client.ts
@@ -0,0 +1,742 @@
+/* eslint-disable max-lines -- Why: bridge schema, request mapping, and result normalization stay together so platform scripts have one audited contract. */
+import { execFile } from 'child_process'
+import { mkdtemp, rm, writeFile } from 'fs/promises'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import type {
+ ComputerActionMetadata,
+ ComputerActionResult,
+ ComputerListAppsResult,
+ ComputerListWindowsResult,
+ ComputerProviderCapabilities,
+ ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import {
+ desktopScriptPlatform,
+ resolveDesktopScriptProviderPath,
+ type DesktopScriptPlatform
+} from './desktop-script-provider-paths'
+import { RuntimeClientError } from './runtime-client-error'
+
+type NativeMethod =
+ | 'handshake'
+ | 'listApps'
+ | 'listWindows'
+ | 'getAppState'
+ | 'click'
+ | 'performSecondaryAction'
+ | 'scroll'
+ | 'drag'
+ | 'typeText'
+ | 'pressKey'
+ | 'hotkey'
+ | 'pasteText'
+ | 'setValue'
+
+type NativeActionMethod = Exclude<
+ NativeMethod,
+ 'handshake' | 'listApps' | 'listWindows' | 'getAppState'
+>
+
+type BridgeFrame = {
+ x: number
+ y: number
+ width: number
+ height: number
+}
+
+type BridgeElement = {
+ index: number
+ runtimeId?: unknown
+ automationId?: string
+ name?: string
+ controlType?: string
+ localizedControlType?: string
+ className?: string
+ value?: string
+ nativeWindowHandle?: number
+ frame?: BridgeFrame | null
+ actions?: string[]
+}
+
+type BridgeSnapshot = {
+ snapshotId?: string
+ app: {
+ name: string
+ bundleIdentifier?: string
+ bundleId?: string
+ pid: number
+ }
+ windowTitle?: string
+ windowId?: number | null
+ windowBounds?: BridgeFrame | null
+ screenshotPngBase64?: string | null
+ coordinateSpace?: 'window'
+ truncation?: {
+ truncated?: boolean
+ maxNodes?: number
+ maxDepth?: number
+ maxDepthReached?: boolean
+ }
+ treeLines?: string[]
+ focusedSummary?: string | null
+ focusedElementId?: number | null
+ selectedText?: string | null
+ elements?: BridgeElement[]
+}
+
+type BridgeWindow = {
+ index: number
+ app: {
+ name: string
+ bundleIdentifier?: string | null
+ bundleId?: string | null
+ pid: number
+ }
+ id?: number | null
+ title: string
+ x?: number | null
+ y?: number | null
+ width: number
+ height: number
+ isMinimized?: boolean | null
+ isOffscreen?: boolean | null
+ screenIndex?: number | null
+ platform?: Record
+}
+
+type BridgeResponse = {
+ ok: boolean
+ error?: string
+ capabilities?: ComputerProviderCapabilities
+ apps?: {
+ name: string
+ bundleIdentifier?: string | null
+ bundleId?: string | null
+ pid: number
+ }[]
+ app?: {
+ name: string
+ bundleIdentifier?: string | null
+ bundleId?: string | null
+ pid: number
+ }
+ windows?: BridgeWindow[]
+ snapshot?: BridgeSnapshot
+ action?: ComputerActionMetadata
+}
+
+type BridgeRequest = {
+ tool: string
+ app?: string
+ element?: BridgeElement
+ fromElement?: BridgeElement
+ toElement?: BridgeElement
+ x?: number
+ y?: number
+ from_x?: number
+ from_y?: number
+ to_x?: number
+ to_y?: number
+ click_count?: number
+ mouse_button?: string
+ action?: string
+ direction?: string
+ pages?: number
+ text?: string
+ key?: string
+ value?: string
+ windowBounds?: BridgeFrame | null
+ windowId?: number
+ windowIndex?: number
+ noScreenshot?: boolean
+ restoreWindow?: boolean
+}
+
+const REQUEST_TIMEOUT_MS = 30_000
+
+export function shouldUseDesktopScriptProvider(): boolean {
+ return desktopScriptPlatform() !== null && resolveDesktopScriptProviderPath() !== null
+}
+
+export class DesktopScriptProviderClient {
+ private readonly snapshots = new Map()
+ private providerCapabilities: ComputerProviderCapabilities | null = null
+
+ constructor(
+ private readonly platform: DesktopScriptPlatform = requiredPlatform(),
+ private readonly scriptPath: string = requiredScriptPath()
+ ) {}
+
+ async listApps(): Promise {
+ const response = await this.callBridge({ tool: 'list_apps' })
+ return {
+ apps: (response.apps ?? []).map((app) => ({
+ name: app.name,
+ bundleId: app.bundleId ?? app.bundleIdentifier ?? null,
+ pid: app.pid,
+ isRunning: true,
+ lastUsedAt: null,
+ useCount: null
+ }))
+ }
+ }
+
+ async capabilities(): Promise {
+ return await this.readCapabilities()
+ }
+
+ async listWindows(params: Record): Promise {
+ const capabilities = await this.readCapabilities()
+ if (!capabilities.supports.windows.list) {
+ throw new RuntimeClientError(
+ 'unsupported_capability',
+ `${capabilities.provider} does not support windows.list`
+ )
+ }
+ const response = await this.callBridge({
+ tool: 'list_windows',
+ app: stringParam(params, 'app')
+ })
+ return {
+ app: normalizeBridgeApp(response.app),
+ windows: (response.windows ?? []).map((window) => ({
+ index: window.index,
+ app: normalizeBridgeApp(window.app),
+ id: window.id ?? null,
+ title: window.title,
+ x: window.x ?? null,
+ y: window.y ?? null,
+ width: window.width,
+ height: window.height,
+ isMinimized: window.isMinimized ?? null,
+ isOffscreen: window.isOffscreen ?? null,
+ screenIndex: window.screenIndex ?? null,
+ platform: window.platform
+ }))
+ }
+ }
+
+ async snapshot(params: Record): Promise {
+ const app = stringParam(params, 'app')
+ const response = await this.callBridge({
+ tool: 'get_app_state',
+ app,
+ windowId: optionalNumberParam(params, 'windowId'),
+ windowIndex: optionalNumberParam(params, 'windowIndex'),
+ noScreenshot: params.noScreenshot === true,
+ restoreWindow: params.restoreWindow === true
+ })
+ return this.rememberAndRender(app, response, params.noScreenshot === true, params)
+ }
+
+ async action(
+ method: NativeActionMethod,
+ params: Record
+ ): Promise {
+ const app = stringParam(params, 'app')
+ await this.ensureActionSupported(method)
+ const explicitWindowId = optionalNumberParam(params, 'windowId')
+ const explicitWindowIndex = optionalNumberParam(params, 'windowIndex')
+ const current = this.currentSnapshot(app, explicitWindowId, params)
+ const element = elementParam(current, optionalNumberParam(params, 'elementIndex'))
+ const fromElement = elementParam(current, optionalNumberParam(params, 'fromElementIndex'))
+ const toElement = elementParam(current, optionalNumberParam(params, 'toElementIndex'))
+ const response = await this.callBridge({
+ tool: bridgeTool(method),
+ app,
+ element,
+ fromElement,
+ toElement,
+ x: optionalNumberParam(params, 'x'),
+ y: optionalNumberParam(params, 'y'),
+ from_x: optionalNumberParam(params, 'fromX'),
+ from_y: optionalNumberParam(params, 'fromY'),
+ to_x: optionalNumberParam(params, 'toX'),
+ to_y: optionalNumberParam(params, 'toY'),
+ click_count: optionalNumberParam(params, 'clickCount'),
+ mouse_button: optionalStringParam(params, 'mouseButton'),
+ action: optionalStringParam(params, 'action'),
+ direction: optionalStringParam(params, 'direction'),
+ pages: optionalNumberParam(params, 'pages'),
+ text: optionalStringParam(params, 'text'),
+ key: optionalStringParam(params, 'key'),
+ value: optionalStringParam(params, 'value'),
+ windowBounds: null,
+ windowId: explicitWindowId ?? current?.windowId ?? undefined,
+ windowIndex: explicitWindowIndex,
+ noScreenshot: params.noScreenshot === true,
+ restoreWindow: params.restoreWindow === true
+ })
+ return {
+ ...this.rememberAndRender(app, response, params.noScreenshot === true, params),
+ action:
+ response.action ??
+ desktopActionMetadata(method, response.snapshot?.windowId ?? current?.windowId ?? null)
+ }
+ }
+
+ private async ensureActionSupported(method: NativeActionMethod): Promise {
+ if (method !== 'hotkey' && method !== 'pasteText') {
+ return
+ }
+ const capabilities = await this.readCapabilities()
+ const actionKey = actionCapabilityKey(method)
+ if (!capabilities.supports.actions[actionKey]) {
+ throw new RuntimeClientError(
+ 'unsupported_capability',
+ `${capabilities.provider} does not support actions.${actionKey}`
+ )
+ }
+ }
+
+ private async callBridge(request: BridgeRequest): Promise {
+ const operationDirectory = await mkdtemp(join(tmpdir(), 'orca-computer-use-'))
+ const operationPath = join(operationDirectory, 'operation.json')
+ try {
+ await writeFile(operationPath, JSON.stringify(request), { encoding: 'utf8', mode: 0o600 })
+ const { stdout, stderr } = await execBridge(this.platform, this.scriptPath, operationPath)
+ let response: BridgeResponse
+ try {
+ response = JSON.parse(stdout) as BridgeResponse
+ } catch (error) {
+ throw new RuntimeClientError(
+ 'accessibility_error',
+ `desktop provider returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`
+ )
+ }
+ if (!response.ok) {
+ throw mapBridgeError(response.error ?? stderr)
+ }
+ return response
+ } finally {
+ await rm(operationDirectory, { force: true, recursive: true })
+ }
+ }
+
+ private async readCapabilities(): Promise {
+ if (this.providerCapabilities) {
+ return this.providerCapabilities
+ }
+ const response = await this.callBridge({ tool: 'handshake' })
+ if (!response.capabilities) {
+ throw new RuntimeClientError(
+ 'accessibility_error',
+ 'desktop provider returned no capabilities'
+ )
+ }
+ this.providerCapabilities = response.capabilities
+ return response.capabilities
+ }
+
+ private rememberAndRender(
+ app: string,
+ response: BridgeResponse,
+ noScreenshot: boolean,
+ params: Record
+ ): ComputerSnapshotResult {
+ if (!response.snapshot) {
+ throw new RuntimeClientError('accessibility_error', 'desktop provider returned no snapshot')
+ }
+ this.rememberSnapshot(app, response.snapshot, params)
+ return renderSnapshot(response.snapshot, noScreenshot)
+ }
+
+ private rememberSnapshot(
+ query: string,
+ snapshot: BridgeSnapshot,
+ params: Record
+ ): void {
+ const namespace = snapshotNamespace(params)
+ for (const key of [
+ query,
+ snapshot.app.name,
+ snapshot.app.bundleId,
+ snapshot.app.bundleIdentifier,
+ String(snapshot.app.pid),
+ ...snapshotKeysForWindow(query, snapshot),
+ ...snapshotKeysForWindow(snapshot.app.name, snapshot),
+ ...(snapshot.app.bundleId ? snapshotKeysForWindow(snapshot.app.bundleId, snapshot) : []),
+ ...(snapshot.app.bundleIdentifier
+ ? snapshotKeysForWindow(snapshot.app.bundleIdentifier, snapshot)
+ : []),
+ ...snapshotKeysForWindowIndex(query, params),
+ ...snapshotKeysForWindowIndex(snapshot.app.name, params),
+ ...(snapshot.app.bundleId ? snapshotKeysForWindowIndex(snapshot.app.bundleId, params) : []),
+ ...(snapshot.app.bundleIdentifier
+ ? snapshotKeysForWindowIndex(snapshot.app.bundleIdentifier, params)
+ : [])
+ ]) {
+ if (key) {
+ if (!isExplicitSnapshotNamespace(namespace)) {
+ this.snapshots.set(key.toLowerCase(), snapshot)
+ }
+ this.snapshots.set(namespacedSnapshotKey(namespace, key), snapshot)
+ }
+ }
+ }
+
+ private currentSnapshot(
+ app: string,
+ windowId: number | undefined,
+ params: Record
+ ): BridgeSnapshot | null {
+ const namespace = snapshotNamespace(params)
+ if (windowId !== undefined) {
+ const windowKey = snapshotWindowKey(app, windowId)
+ return (
+ this.snapshots.get(namespacedSnapshotKey(namespace, windowKey)) ??
+ (isExplicitSnapshotNamespace(namespace) ? undefined : this.snapshots.get(windowKey)) ??
+ null
+ )
+ }
+ const windowIndex = optionalNumberParam(params, 'windowIndex')
+ if (windowIndex !== undefined) {
+ const windowIndexKey = snapshotWindowIndexKey(app, windowIndex)
+ return (
+ this.snapshots.get(namespacedSnapshotKey(namespace, windowIndexKey)) ??
+ (isExplicitSnapshotNamespace(namespace) ? undefined : this.snapshots.get(windowIndexKey)) ??
+ null
+ )
+ }
+ return (
+ this.snapshots.get(namespacedSnapshotKey(namespace, app)) ??
+ (isExplicitSnapshotNamespace(namespace)
+ ? undefined
+ : this.snapshots.get(app.toLowerCase())) ??
+ null
+ )
+ }
+}
+
+function execBridge(
+ platform: DesktopScriptPlatform,
+ scriptPath: string,
+ operationPath: string
+): Promise<{ stdout: string; stderr: string }> {
+ const command = platform === 'windows' ? 'powershell.exe' : 'python3'
+ const args =
+ platform === 'windows'
+ ? [
+ '-NoProfile',
+ '-NonInteractive',
+ '-ExecutionPolicy',
+ 'Bypass',
+ '-File',
+ scriptPath,
+ operationPath
+ ]
+ : [scriptPath, operationPath]
+ return new Promise((resolve, reject) => {
+ execFile(
+ command,
+ args,
+ {
+ env: process.env,
+ maxBuffer: 20 * 1024 * 1024,
+ timeout: REQUEST_TIMEOUT_MS,
+ windowsHide: true
+ },
+ (error, stdout, stderr) => {
+ if (error) {
+ const message = stderr.trim() || stdout.trim() || error.message
+ reject(
+ error.killed
+ ? new RuntimeClientError('action_timeout', message)
+ : mapBridgeError(message)
+ )
+ return
+ }
+ resolve({ stdout, stderr })
+ }
+ )
+ })
+}
+
+function renderSnapshot(snapshot: BridgeSnapshot, noScreenshot: boolean): ComputerSnapshotResult {
+ const bounds = snapshot.windowBounds
+ const treeText = renderTreeText(snapshot)
+ const screenshot = snapshot.screenshotPngBase64
+ ? {
+ data: snapshot.screenshotPngBase64,
+ format: 'png' as const,
+ width: Math.max(1, Math.round(bounds?.width ?? 1)),
+ height: Math.max(1, Math.round(bounds?.height ?? 1)),
+ scale: 1
+ }
+ : null
+ return {
+ snapshot: {
+ // Why: bridge elements can be large; keep them cached internally for actions
+ // instead of returning duplicated metadata in every agent-facing snapshot.
+ id: snapshot.snapshotId ?? fallbackSnapshotId(snapshot),
+ app: {
+ name: snapshot.app.name,
+ bundleId: snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? null,
+ pid: snapshot.app.pid
+ },
+ window: {
+ title: snapshot.windowTitle ?? snapshot.app.name,
+ id: snapshot.windowId ?? null,
+ x: bounds ? Math.round(bounds.x) : null,
+ y: bounds ? Math.round(bounds.y) : null,
+ width: Math.max(0, Math.round(bounds?.width ?? 0)),
+ height: Math.max(0, Math.round(bounds?.height ?? 0)),
+ isMinimized: null,
+ isOffscreen: null,
+ screenIndex: null
+ },
+ coordinateSpace: snapshot.coordinateSpace ?? 'window',
+ treeText,
+ elementCount: snapshot.elements?.length ?? 0,
+ focusedElementId: snapshot.focusedElementId ?? null,
+ truncation: {
+ truncated: snapshot.truncation?.truncated === true,
+ maxNodes: snapshot.truncation?.maxNodes,
+ maxDepth: snapshot.truncation?.maxDepth,
+ maxDepthReached: snapshot.truncation?.maxDepthReached === true
+ }
+ },
+ screenshot,
+ screenshotStatus: screenshot
+ ? {
+ state: 'captured',
+ metadata: { engine: 'unknown', windowId: snapshot.windowId ?? null }
+ }
+ : noScreenshot
+ ? { state: 'skipped', reason: 'no_screenshot_flag' }
+ : {
+ state: 'failed',
+ code: 'screenshot_failed',
+ message:
+ 'desktop provider returned no image; grant screen capture permission or pass --no-screenshot to inspect accessibility state only.'
+ }
+ }
+}
+
+function fallbackSnapshotId(snapshot: BridgeSnapshot): string {
+ const appRef = snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? snapshot.app.name
+ return `${appRef}:${snapshot.app.pid}:${snapshot.windowId ?? 'window'}`
+}
+
+function renderTreeText(snapshot: BridgeSnapshot): string {
+ const appRef = snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? snapshot.app.name
+ const lines = [
+ `App=${appRef} (pid ${snapshot.app.pid})`,
+ `Window: "${sanitize(snapshot.windowTitle ?? snapshot.app.name)}", App: ${sanitize(snapshot.app.name)}.`,
+ '',
+ ...(snapshot.treeLines ?? [])
+ ]
+ if (snapshot.selectedText) {
+ lines.push('', `Selected text: [${sanitize(snapshot.selectedText)}]`)
+ } else if (snapshot.focusedSummary) {
+ lines.push('', `The focused UI element is ${sanitize(snapshot.focusedSummary)}.`)
+ }
+ return lines.join('\n')
+}
+
+function bridgeTool(method: NativeActionMethod): string {
+ return {
+ click: 'click',
+ performSecondaryAction: 'perform_secondary_action',
+ scroll: 'scroll',
+ drag: 'drag',
+ typeText: 'type_text',
+ pressKey: 'press_key',
+ hotkey: 'hotkey',
+ pasteText: 'paste_text',
+ setValue: 'set_value'
+ }[method]
+}
+
+function actionCapabilityKey(
+ method: NativeActionMethod
+): keyof ComputerProviderCapabilities['supports']['actions'] {
+ const keys = {
+ click: 'click',
+ performSecondaryAction: 'performAction',
+ scroll: 'scroll',
+ drag: 'drag',
+ typeText: 'typeText',
+ pressKey: 'pressKey',
+ hotkey: 'hotkey',
+ pasteText: 'pasteText',
+ setValue: 'setValue'
+ } satisfies Record
+ return keys[method]
+}
+
+function desktopActionMetadata(method: NativeActionMethod, targetWindowId: number | null) {
+ const path =
+ method === 'pasteText'
+ ? ('clipboard' as const)
+ : method === 'setValue' || method === 'performSecondaryAction'
+ ? ('accessibility' as const)
+ : ('synthetic' as const)
+ return {
+ path,
+ actionName:
+ method === 'hotkey'
+ ? 'hotkey'
+ : method === 'pasteText'
+ ? 'paste'
+ : method === 'setValue'
+ ? 'setValue'
+ : method === 'performSecondaryAction'
+ ? 'performSecondaryAction'
+ : null,
+ fallbackReason: null,
+ targetWindowId,
+ verification:
+ method === 'hotkey' || method === 'pasteText'
+ ? {
+ state: 'unverified' as const,
+ reason:
+ method === 'pasteText' ? ('clipboard_paste' as const) : ('synthetic_input' as const)
+ }
+ : undefined
+ }
+}
+
+function normalizeBridgeApp(app: BridgeResponse['app'] | BridgeWindow['app'] | undefined) {
+ if (!app) {
+ throw new RuntimeClientError('accessibility_error', 'desktop provider returned no app')
+ }
+ return {
+ name: app.name,
+ bundleId: app.bundleId ?? app.bundleIdentifier ?? null,
+ pid: app.pid
+ }
+}
+
+function elementParam(
+ snapshot: BridgeSnapshot | null,
+ index: number | undefined
+): BridgeElement | undefined {
+ if (index === undefined) {
+ return undefined
+ }
+ const element = snapshot?.elements?.find((candidate) => candidate.index === index)
+ if (!element) {
+ throw new RuntimeClientError(
+ 'element_not_found',
+ `element ${index} is not in the current cached snapshot; run get-app-state again and use a fresh element index`
+ )
+ }
+ return element
+}
+
+function snapshotKeysForWindow(query: string, snapshot: BridgeSnapshot): string[] {
+ return snapshot.windowId === null || snapshot.windowId === undefined
+ ? []
+ : [snapshotWindowKey(query, snapshot.windowId)]
+}
+
+function snapshotWindowKey(query: string, windowId: number): string {
+ return `${query.toLowerCase()}#window:${windowId}`
+}
+
+function snapshotKeysForWindowIndex(query: string, params: Record): string[] {
+ const windowIndex = optionalNumberParam(params, 'windowIndex')
+ return windowIndex === undefined ? [] : [snapshotWindowIndexKey(query, windowIndex)]
+}
+
+function snapshotWindowIndexKey(query: string, windowIndex: number): string {
+ return `${query.toLowerCase()}#window-index:${windowIndex}`
+}
+
+function snapshotNamespace(params: Record): string {
+ const session = optionalStringParam(params, 'session')
+ const worktree = optionalStringParam(params, 'worktree')
+ return session ? `session:${session}` : worktree ? `worktree:${worktree}` : 'default'
+}
+
+function namespacedSnapshotKey(namespace: string, key: string): string {
+ return `${namespace}:${key.toLowerCase()}`
+}
+
+function isExplicitSnapshotNamespace(namespace: string): boolean {
+ return namespace !== 'default'
+}
+
+function mapBridgeError(message: string): RuntimeClientError {
+ const text = message.trim() || 'desktop provider failed'
+ if (/appNotFound|app not found/i.test(text)) {
+ return new RuntimeClientError('app_not_found', text)
+ }
+ if (/appBlocked|app blocked/i.test(text)) {
+ return new RuntimeClientError('app_blocked', text)
+ }
+ if (/unsupported capability|hotkey.*require|paste_text requires/i.test(text)) {
+ return new RuntimeClientError('unsupported_capability', text)
+ }
+ if (/ModuleNotFoundError: No module named 'gi'|PyGObject|python3-gi/i.test(text)) {
+ return new RuntimeClientError(
+ 'unsupported_capability',
+ 'Linux Computer Use requires python3-gi and AT-SPI packages. Install python3-gi gir1.2-atspi-2.0 at-spi2-core, then retry.'
+ )
+ }
+ if (/not a valid secondary action|action.*not supported/i.test(text)) {
+ return new RuntimeClientError('action_not_supported', text)
+ }
+ if (/value is not settable|not settable/i.test(text)) {
+ return new RuntimeClientError('value_not_settable', text)
+ }
+ if (/stale element|fresh element index/i.test(text)) {
+ return new RuntimeClientError('element_not_found', text)
+ }
+ if (/windowStale|window stale/i.test(text)) {
+ return new RuntimeClientError('window_stale', text)
+ }
+ if (/No top-level|No .*window|window/i.test(text)) {
+ return new RuntimeClientError('window_not_found', text)
+ }
+ if (/permission|desktop session|DBUS|XDG_RUNTIME_DIR|AT-SPI/i.test(text)) {
+ return new RuntimeClientError('permission_denied', text)
+ }
+ if (/element|element_index/i.test(text)) {
+ return new RuntimeClientError('element_not_found', text)
+ }
+ return new RuntimeClientError('accessibility_error', text)
+}
+
+function requiredPlatform(): DesktopScriptPlatform {
+ const platform = desktopScriptPlatform()
+ if (!platform) {
+ throw new RuntimeClientError('accessibility_error', 'desktop script provider is not available')
+ }
+ return platform
+}
+
+function requiredScriptPath(): string {
+ const scriptPath = resolveDesktopScriptProviderPath()
+ if (!scriptPath) {
+ throw new RuntimeClientError(
+ 'accessibility_error',
+ 'desktop script provider script was not found'
+ )
+ }
+ return scriptPath
+}
+
+function stringParam(params: Record, key: string): string {
+ const value = params[key]
+ if (typeof value !== 'string' || value.length === 0) {
+ throw new RuntimeClientError('invalid_argument', `missing ${key}`)
+ }
+ return value
+}
+
+function optionalStringParam(params: Record, key: string): string | undefined {
+ const value = params[key]
+ return typeof value === 'string' ? value : undefined
+}
+
+function optionalNumberParam(params: Record, key: string): number | undefined {
+ const value = params[key]
+ return typeof value === 'number' && Number.isFinite(value) ? value : undefined
+}
+
+function sanitize(value: string): string {
+ return value.replaceAll('\n', ' ').replaceAll('\r', ' ')
+}
diff --git a/src/main/computer/desktop-script-provider-paths.ts b/src/main/computer/desktop-script-provider-paths.ts
new file mode 100644
index 00000000000..59931cf549b
--- /dev/null
+++ b/src/main/computer/desktop-script-provider-paths.ts
@@ -0,0 +1,39 @@
+import { existsSync } from 'fs'
+import { join, resolve } from 'path'
+
+export type DesktopScriptPlatform = 'linux' | 'windows'
+
+export function desktopScriptPlatform(): DesktopScriptPlatform | null {
+ if (process.platform === 'linux') {
+ return 'linux'
+ }
+ if (process.platform === 'win32') {
+ return 'windows'
+ }
+ return null
+}
+
+export function resolveDesktopScriptProviderPath(
+ platform = desktopScriptPlatform()
+): string | null {
+ const override = process.env.ORCA_COMPUTER_DESKTOP_SCRIPT_PROVIDER_PATH
+ if (override && existsSync(override)) {
+ return override
+ }
+ if (!platform) {
+ return null
+ }
+
+ const filename = platform === 'windows' ? 'runtime.ps1' : 'runtime.py'
+ const directory = platform === 'windows' ? 'computer-use-windows' : 'computer-use-linux'
+ const sourceDirectory =
+ platform === 'windows' ? 'native/computer-use-windows' : 'native/computer-use-linux'
+ const packaged = [join(process.resourcesPath ?? '', directory, filename)]
+ const dev = [
+ join(process.cwd(), sourceDirectory, filename),
+ resolve(__dirname, '../../', sourceDirectory, filename)
+ ]
+ const candidates = process.resourcesPath ? [...packaged, ...dev] : dev
+
+ return candidates.find((candidate) => candidate && existsSync(candidate)) ?? null
+}
diff --git a/src/main/computer/key-mapping.test.ts b/src/main/computer/key-mapping.test.ts
new file mode 100644
index 00000000000..2604df8ab93
--- /dev/null
+++ b/src/main/computer/key-mapping.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest'
+import { parseKey } from './key-mapping'
+
+describe('parseKey', () => {
+ it.each([
+ ['a', { key: 'a', modifiers: [] }],
+ ['7', { key: '7', modifiers: [] }],
+ [';', { key: ';', modifiers: [] }],
+ ['Return', { key: 'Enter', modifiers: [] }],
+ ['Escape', { key: 'Escape', modifiers: [] }],
+ ['BackSpace', { key: 'Backspace', modifiers: [] }],
+ ['Tab', { key: 'Tab', modifiers: [] }],
+ ['space', { key: 'Space', modifiers: [] }],
+ ['Page_Up', { key: 'PageUp', modifiers: [] }],
+ ['Page_Down', { key: 'PageDown', modifiers: [] }],
+ ['Home', { key: 'Home', modifiers: [] }],
+ ['End', { key: 'End', modifiers: [] }],
+ ['Up', { key: 'ArrowUp', modifiers: [] }],
+ ['Down', { key: 'ArrowDown', modifiers: [] }],
+ ['Left', { key: 'ArrowLeft', modifiers: [] }],
+ ['Right', { key: 'ArrowRight', modifiers: [] }],
+ ['Delete', { key: 'Delete', modifiers: [] }],
+ ['Insert', { key: 'Insert', modifiers: [] }],
+ ['F1', { key: 'F1', modifiers: [] }],
+ ['F24', { key: 'F24', modifiers: [] }],
+ ['KP_0', { key: 'Numpad0', modifiers: [] }],
+ ['KP_9', { key: 'Numpad9', modifiers: [] }],
+ ['KP_Add', { key: 'NumpadAdd', modifiers: [] }],
+ ['KP_Subtract', { key: 'NumpadSubtract', modifiers: [] }],
+ ['KP_Multiply', { key: 'NumpadMultiply', modifiers: [] }],
+ ['KP_Divide', { key: 'NumpadDivide', modifiers: [] }],
+ ['KP_Enter', { key: 'NumpadEnter', modifiers: [] }]
+ ])('maps %s', (input, expected) => {
+ expect(parseKey(input)).toEqual(expected)
+ })
+
+ it.each([
+ ['ctrl+a', { key: 'a', modifiers: ['Ctrl'] }],
+ ['control+a', { key: 'a', modifiers: ['Ctrl'] }],
+ ['ctrl+shift+t', { key: 't', modifiers: ['Ctrl', 'Shift'] }],
+ ['alt+F4', { key: 'F4', modifiers: ['Alt'] }],
+ ['cmd+a', { key: 'a', modifiers: ['Meta'] }],
+ ['command+a', { key: 'a', modifiers: ['Meta'] }],
+ ['CmdOrCtrl+a', { key: 'a', modifiers: [process.platform === 'darwin' ? 'Meta' : 'Ctrl'] }],
+ ['super+Left', { key: 'ArrowLeft', modifiers: ['Meta'] }],
+ ['win+Right', { key: 'ArrowRight', modifiers: ['Meta'] }]
+ ])('maps chord %s', (input, expected) => {
+ expect(parseKey(input)).toEqual(expected)
+ })
+
+ it('rejects unknown modifiers and key names', () => {
+ expect(() => parseKey('hyper+a')).toThrow(expect.objectContaining({ code: 'invalid_argument' }))
+ expect(() => parseKey('NotAKey')).toThrow(expect.objectContaining({ code: 'invalid_argument' }))
+ expect(() => parseKey('F25')).toThrow(expect.objectContaining({ code: 'invalid_argument' }))
+ })
+})
diff --git a/src/main/computer/key-mapping.ts b/src/main/computer/key-mapping.ts
new file mode 100644
index 00000000000..cc6cb414ac1
--- /dev/null
+++ b/src/main/computer/key-mapping.ts
@@ -0,0 +1,109 @@
+import { RuntimeClientError } from './runtime-client-error'
+
+export type KeyChord = {
+ key: string
+ modifiers: string[]
+}
+
+const MODIFIER_NAMES: Record = {
+ ctrl: 'Ctrl',
+ control: 'Ctrl',
+ shift: 'Shift',
+ alt: 'Alt',
+ meta: 'Meta',
+ super: 'Meta',
+ cmd: 'Meta',
+ command: 'Meta',
+ win: 'Meta'
+}
+
+const PLATFORM_MODIFIER_NAMES: Record string> = {
+ cmdorctrl: () => (process.platform === 'darwin' ? 'Meta' : 'Ctrl'),
+ commandorcontrol: () => (process.platform === 'darwin' ? 'Meta' : 'Ctrl')
+}
+
+const KEY_NAMES: Record = {
+ Return: 'Enter',
+ Enter: 'Enter',
+ Escape: 'Escape',
+ Esc: 'Escape',
+ BackSpace: 'Backspace',
+ Backspace: 'Backspace',
+ Tab: 'Tab',
+ space: 'Space',
+ Space: 'Space',
+ Page_Up: 'PageUp',
+ Page_Down: 'PageDown',
+ Home: 'Home',
+ End: 'End',
+ Up: 'ArrowUp',
+ Down: 'ArrowDown',
+ Left: 'ArrowLeft',
+ Right: 'ArrowRight',
+ Delete: 'Delete',
+ Insert: 'Insert',
+ KP_Add: 'NumpadAdd',
+ KP_Subtract: 'NumpadSubtract',
+ KP_Multiply: 'NumpadMultiply',
+ KP_Divide: 'NumpadDivide',
+ KP_Enter: 'NumpadEnter'
+}
+
+export function parseKey(input: string): KeyChord {
+ const parts = input
+ .split('+')
+ .map((part) => part.trim())
+ .filter(Boolean)
+ if (parts.length === 0) {
+ throw new RuntimeClientError('invalid_argument', 'key must not be empty')
+ }
+
+ const keyPart = parts.at(-1)!
+ const modifiers = parts.slice(0, -1).map(parseModifier)
+ return {
+ key: parseBaseKey(keyPart),
+ modifiers: dedupeModifiers(modifiers)
+ }
+}
+
+function parseModifier(input: string): string {
+ const platformModifier = PLATFORM_MODIFIER_NAMES[input.toLowerCase()]
+ if (platformModifier) {
+ return platformModifier()
+ }
+ const modifier = MODIFIER_NAMES[input.toLowerCase()]
+ if (!modifier) {
+ throw new RuntimeClientError('invalid_argument', `unknown modifier '${input}'`)
+ }
+ return modifier
+}
+
+function parseBaseKey(input: string): string {
+ const mapped = KEY_NAMES[input]
+ if (mapped) {
+ return mapped
+ }
+ if (/^F([1-9]|1[0-9]|2[0-4])$/.test(input)) {
+ return input
+ }
+ const keypadDigit = input.match(/^KP_([0-9])$/)
+ if (keypadDigit) {
+ return `Numpad${keypadDigit[1]}`
+ }
+ if (isPrintableAscii(input)) {
+ return input
+ }
+ throw new RuntimeClientError('invalid_argument', `unknown key '${input}'`)
+}
+
+function isPrintableAscii(input: string): boolean {
+ if (input.length !== 1) {
+ return false
+ }
+ const code = input.charCodeAt(0)
+ return code >= 0x20 && code <= 0x7e
+}
+
+function dedupeModifiers(modifiers: string[]): string[] {
+ return [...new Set(modifiers)]
+}
diff --git a/src/main/computer/macos-app-catalog.ts b/src/main/computer/macos-app-catalog.ts
new file mode 100644
index 00000000000..273f5199fe4
--- /dev/null
+++ b/src/main/computer/macos-app-catalog.ts
@@ -0,0 +1,121 @@
+import { execFile } from 'child_process'
+import type { ComputerAppInfo } from '../../shared/runtime-types'
+
+type MacOSRunningApp = ComputerAppInfo & {
+ activationPolicy: number
+ isFrontmost: boolean
+}
+
+const JXA_LIST_RUNNING_APPS = `
+ObjC.import('AppKit');
+const apps = $.NSWorkspace.sharedWorkspace.runningApplications;
+const out = [];
+for (let i = 0; i < apps.count; i += 1) {
+ const app = apps.objectAtIndex(i);
+ if (app.isTerminated) continue;
+ const name = ObjC.unwrap(app.localizedName) || '';
+ const bundleId = ObjC.unwrap(app.bundleIdentifier) || null;
+ const pid = Number(app.processIdentifier);
+ const activationPolicy = Number(app.activationPolicy);
+ if (!name || !Number.isInteger(pid) || pid <= 0) continue;
+ out.push({
+ name,
+ bundleId,
+ pid,
+ activationPolicy,
+ isFrontmost: Boolean(app.active)
+ });
+}
+JSON.stringify(out);
+`
+
+export async function listMacOSApps(): Promise {
+ const { stdout } = await execFilePromise(
+ 'osascript',
+ ['-l', 'JavaScript', '-e', JXA_LIST_RUNNING_APPS],
+ {
+ encoding: 'utf8',
+ timeout: 5000,
+ maxBuffer: 1024 * 1024
+ }
+ )
+ const apps = parseMacOSApps(stdout).filter(hasLivePid)
+ return dedupeApps(apps.filter(isUserFacingApp)).map(({ name, bundleId, pid }) => ({
+ name,
+ bundleId,
+ pid
+ }))
+}
+
+function parseMacOSApps(stdout: string): MacOSRunningApp[] {
+ const parsed: unknown = JSON.parse(stdout)
+ if (!Array.isArray(parsed)) {
+ return []
+ }
+ return parsed.flatMap((value): MacOSRunningApp[] => {
+ if (!value || typeof value !== 'object') {
+ return []
+ }
+ const record = value as Record
+ const name = typeof record.name === 'string' ? record.name : ''
+ const bundleId = typeof record.bundleId === 'string' ? record.bundleId : null
+ const pid = typeof record.pid === 'number' && Number.isInteger(record.pid) ? record.pid : 0
+ const activationPolicy =
+ typeof record.activationPolicy === 'number' && Number.isInteger(record.activationPolicy)
+ ? record.activationPolicy
+ : -1
+ const isFrontmost = record.isFrontmost === true
+ return name && pid > 0 ? [{ name, bundleId, pid, activationPolicy, isFrontmost }] : []
+ })
+}
+
+function isUserFacingApp(app: MacOSRunningApp): boolean {
+ return app.activationPolicy === 0
+}
+
+function hasLivePid(app: MacOSRunningApp): boolean {
+ try {
+ process.kill(app.pid, 0)
+ return true
+ } catch {
+ // NSWorkspace can briefly retain terminated apps; filter them before provider calls.
+ return false
+ }
+}
+
+function dedupeApps(apps: MacOSRunningApp[]): MacOSRunningApp[] {
+ const seen = new Set()
+ const sorted = [...apps].sort((a, b) => {
+ if (a.isFrontmost !== b.isFrontmost) {
+ return a.isFrontmost ? -1 : 1
+ }
+ return a.name.localeCompare(b.name)
+ })
+ const result: MacOSRunningApp[] = []
+ for (const app of sorted) {
+ const key = app.bundleId ? `bundle:${app.bundleId.toLowerCase()}` : `pid:${app.pid}`
+ if (seen.has(key)) {
+ continue
+ }
+ seen.add(key)
+ result.push(app)
+ }
+ return result
+}
+
+function execFilePromise(
+ file: string,
+ args: string[],
+ options: { encoding: 'utf8'; timeout: number; maxBuffer: number }
+): Promise<{ stdout: string }> {
+ return new Promise((resolve, reject) => {
+ execFile(file, args, options, (error, stdout, stderr) => {
+ if (error) {
+ const detail = typeof stderr === 'string' && stderr.trim() ? `: ${stderr.trim()}` : ''
+ reject(new Error(`${error.message}${detail}`))
+ return
+ }
+ resolve({ stdout })
+ })
+ })
+}
diff --git a/src/main/computer/macos-computer-use-permissions.test.ts b/src/main/computer/macos-computer-use-permissions.test.ts
new file mode 100644
index 00000000000..f2f469cc215
--- /dev/null
+++ b/src/main/computer/macos-computer-use-permissions.test.ts
@@ -0,0 +1,180 @@
+import { spawn, spawnSync } from 'child_process'
+import { existsSync, readFileSync, rmSync } from 'fs'
+import type * as Fs from 'fs'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { openComputerUsePermissions } from './macos-computer-use-permissions'
+
+const resolveHelperAppPathMock = vi.hoisted(() => vi.fn())
+
+vi.mock('child_process', () => ({
+ spawn: vi.fn(() => ({ unref: vi.fn() })),
+ spawnSync: vi.fn()
+}))
+
+vi.mock('fs', async (importOriginal) => {
+ const actual = (await importOriginal()) as typeof Fs
+ return {
+ ...actual,
+ existsSync: vi.fn(),
+ readFileSync: vi.fn(),
+ rmSync: vi.fn()
+ }
+})
+
+vi.mock('./macos-native-provider-paths', () => ({
+ resolveMacOSComputerUseAppPath: resolveHelperAppPathMock
+}))
+
+describe('openComputerUsePermissions', () => {
+ const originalPlatform = process.platform
+
+ beforeEach(() => {
+ vi.mocked(spawn).mockClear()
+ vi.mocked(spawnSync).mockClear()
+ vi.mocked(existsSync).mockReset()
+ vi.mocked(readFileSync).mockReset()
+ vi.mocked(rmSync).mockReset()
+ resolveHelperAppPathMock.mockReset()
+ mockPermissionStatus('{"accessibility":"granted","screenshots":"granted"}')
+ setPlatform('darwin')
+ })
+
+ afterEach(() => {
+ setPlatform(originalPlatform)
+ })
+
+ it('does not launch the setup helper when all permissions are granted', () => {
+ resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+
+ expect(openComputerUsePermissions()).toEqual({
+ platform: 'darwin',
+ helperAppPath: '/Applications/Orca Computer Use.app',
+ permissionId: undefined,
+ openedSettings: false,
+ launchedHelper: false,
+ permissions: [
+ { id: 'accessibility', status: 'granted' },
+ { id: 'screenshots', status: 'granted' }
+ ],
+ nextStep: null
+ })
+ expect(spawn).not.toHaveBeenCalled()
+ })
+
+ it('launches the helper app in permissions mode', () => {
+ resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+ mockPermissionStatus('{"accessibility":"granted","screenshots":"not-granted"}')
+
+ expect(openComputerUsePermissions()).toEqual({
+ platform: 'darwin',
+ helperAppPath: '/Applications/Orca Computer Use.app',
+ permissionId: undefined,
+ openedSettings: false,
+ launchedHelper: true,
+ permissions: [
+ { id: 'accessibility', status: 'granted' },
+ { id: 'screenshots', status: 'not-granted' }
+ ],
+ nextStep: 'Grant Screen Recording to Orca Computer Use, then retry get-app-state.'
+ })
+ expect(spawn).toHaveBeenCalledTimes(1)
+ expect(spawnSync).toHaveBeenCalledWith(
+ '/usr/bin/pkill',
+ ['-f', 'orca-computer-use-macos --permission'],
+ { stdio: 'ignore' }
+ )
+ expect(spawnSync).toHaveBeenCalledWith(
+ '/usr/bin/pkill',
+ ['-f', 'orca-computer-use-macos --permissions'],
+ { stdio: 'ignore' }
+ )
+ expect(spawn).toHaveBeenCalledWith(
+ '/usr/bin/open',
+ ['-n', '/Applications/Orca Computer Use.app', '--args', '--permissions'],
+ { detached: true, stdio: 'ignore' }
+ )
+ })
+
+ it('launches a targeted permission helper flow', () => {
+ resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+ mockPermissionStatus('{"accessibility":"not-granted","screenshots":"not-granted"}')
+
+ expect(openComputerUsePermissions('accessibility')).toEqual({
+ platform: 'darwin',
+ helperAppPath: '/Applications/Orca Computer Use.app',
+ permissionId: 'accessibility',
+ openedSettings: true,
+ launchedHelper: true,
+ permissions: [
+ { id: 'accessibility', status: 'not-granted' },
+ { id: 'screenshots', status: 'not-granted' }
+ ],
+ nextStep: 'Grant Accessibility to Orca Computer Use, then retry get-app-state.'
+ })
+ expect(spawn).toHaveBeenCalledWith(
+ '/usr/bin/open',
+ ['-n', '/Applications/Orca Computer Use.app', '--args', '--permission', 'accessibility'],
+ { detached: true, stdio: 'ignore' }
+ )
+ })
+
+ it('returns a no-op result on non-macOS platforms', () => {
+ setPlatform('linux')
+
+ expect(openComputerUsePermissions()).toEqual({
+ platform: 'linux',
+ helperAppPath: null,
+ permissionId: undefined,
+ openedSettings: false,
+ launchedHelper: false,
+ permissions: [
+ { id: 'accessibility', status: 'unsupported' },
+ { id: 'screenshots', status: 'unsupported' }
+ ],
+ nextStep: null
+ })
+ expect(spawn).not.toHaveBeenCalled()
+ })
+
+ it('throws when the helper app is missing on macOS', () => {
+ resolveHelperAppPathMock.mockReturnValue(null)
+
+ expect(() => openComputerUsePermissions()).toThrow('Orca Computer Use.app was not found')
+ })
+
+ it('reads permission status through the helper app bundle', async () => {
+ const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions')
+ resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+ mockPermissionStatus('{"accessibility":"granted","screenshots":"not-granted"}')
+
+ expect(getComputerUsePermissionStatus()).toEqual({
+ platform: 'darwin',
+ permissions: [
+ { id: 'accessibility', status: 'granted' },
+ { id: 'screenshots', status: 'not-granted' }
+ ]
+ })
+ expect(spawnSync).toHaveBeenCalledWith(
+ '/usr/bin/open',
+ [
+ '-n',
+ '/Applications/Orca Computer Use.app',
+ '--args',
+ '--permission-status-file',
+ expect.stringContaining('/status.json')
+ ],
+ { stdio: 'ignore' }
+ )
+ expect(rmSync).toHaveBeenCalled()
+ })
+})
+
+function mockPermissionStatus(json: string): void {
+ vi.mocked(spawnSync).mockReturnValue({} as ReturnType)
+ vi.mocked(existsSync).mockReturnValue(true)
+ vi.mocked(readFileSync).mockReturnValue(json)
+}
+
+function setPlatform(platform: NodeJS.Platform): void {
+ Object.defineProperty(process, 'platform', { configurable: true, value: platform })
+}
diff --git a/src/main/computer/macos-computer-use-permissions.ts b/src/main/computer/macos-computer-use-permissions.ts
new file mode 100644
index 00000000000..244683f5400
--- /dev/null
+++ b/src/main/computer/macos-computer-use-permissions.ts
@@ -0,0 +1,153 @@
+import { spawn, spawnSync } from 'child_process'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import { RuntimeClientError } from './runtime-client-error'
+import { resolveMacOSComputerUseAppPath } from './macos-native-provider-paths'
+import type {
+ ComputerUsePermissionId,
+ ComputerUsePermissionSetupResult,
+ ComputerUsePermissionStatus,
+ ComputerUsePermissionStatusResult
+} from '../../shared/computer-use-permissions-types'
+
+export function openComputerUsePermissions(
+ permissionId?: ComputerUsePermissionId
+): ComputerUsePermissionSetupResult {
+ if (process.platform !== 'darwin') {
+ return {
+ platform: process.platform,
+ helperAppPath: null,
+ permissionId,
+ openedSettings: false,
+ launchedHelper: false,
+ permissions: [
+ { id: 'accessibility', status: 'unsupported' },
+ { id: 'screenshots', status: 'unsupported' }
+ ],
+ nextStep: null
+ }
+ }
+
+ const helperAppPath = resolveMacOSComputerUseAppPath()
+ if (!helperAppPath) {
+ throw new RuntimeClientError('accessibility_error', 'Orca Computer Use.app was not found')
+ }
+ const status = getComputerUsePermissionStatus()
+ const nextStep = nextPermissionStep(status.permissions)
+
+ if (!permissionId && !nextStep) {
+ return {
+ platform: process.platform,
+ helperAppPath,
+ permissionId,
+ openedSettings: false,
+ launchedHelper: false,
+ permissions: status.permissions,
+ nextStep
+ }
+ }
+
+ closeExistingPermissionHelpers()
+ const helperArgs = permissionId ? ['--permission', permissionId] : ['--permissions']
+ const helper = spawn('/usr/bin/open', ['-n', helperAppPath, '--args', ...helperArgs], {
+ detached: true,
+ stdio: 'ignore'
+ })
+ helper.unref()
+
+ return {
+ platform: process.platform,
+ helperAppPath,
+ permissionId,
+ openedSettings: permissionId !== undefined,
+ launchedHelper: true,
+ permissions: status.permissions,
+ nextStep
+ }
+}
+
+function closeExistingPermissionHelpers(): void {
+ spawnSync('/usr/bin/pkill', ['-f', 'orca-computer-use-macos --permission'], {
+ stdio: 'ignore'
+ })
+ spawnSync('/usr/bin/pkill', ['-f', 'orca-computer-use-macos --permissions'], {
+ stdio: 'ignore'
+ })
+}
+
+export function getComputerUsePermissionStatus(): ComputerUsePermissionStatusResult {
+ if (process.platform !== 'darwin') {
+ return {
+ platform: process.platform,
+ permissions: [
+ { id: 'accessibility', status: 'unsupported' },
+ { id: 'screenshots', status: 'unsupported' }
+ ]
+ }
+ }
+
+ const helperAppPath = resolveMacOSComputerUseAppPath()
+ if (!helperAppPath) {
+ throw new RuntimeClientError('accessibility_error', 'Orca Computer Use.app was not found')
+ }
+
+ const raw = readPermissionStatusFromHelperApp(helperAppPath)
+
+ return {
+ platform: process.platform,
+ permissions: [
+ { id: 'accessibility', status: raw.accessibility ?? 'not-granted' },
+ { id: 'screenshots', status: raw.screenshots ?? 'not-granted' }
+ ]
+ }
+}
+
+function readPermissionStatusFromHelperApp(
+ helperAppPath: string
+): Partial> {
+ const directory = mkdtempSync(join(tmpdir(), 'orca-computer-permissions-'))
+ const statusPath = join(directory, 'status.json')
+ try {
+ // Why: TCC can attribute direct executable probes to the parent shell;
+ // launch the app bundle so status uses the same identity as real actions.
+ const result = spawnSync(
+ '/usr/bin/open',
+ ['-n', helperAppPath, '--args', '--permission-status-file', statusPath],
+ { stdio: 'ignore' }
+ )
+ if (result.error) {
+ throw result.error
+ }
+ waitForStatusFile(statusPath)
+ return JSON.parse(readFileSync(statusPath, 'utf8')) as Partial<
+ Record
+ >
+ } finally {
+ rmSync(directory, { force: true, recursive: true })
+ }
+}
+
+function waitForStatusFile(statusPath: string): void {
+ const deadline = Date.now() + 3_000
+ while (Date.now() < deadline) {
+ if (existsSync(statusPath)) {
+ return
+ }
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50)
+ }
+ throw new RuntimeClientError(
+ 'action_timeout',
+ 'Orca Computer Use.app did not report permission status'
+ )
+}
+
+function nextPermissionStep(
+ permissions: ComputerUsePermissionStatusResult['permissions']
+): string | null {
+ const missing = permissions.find((permission) => permission.status !== 'granted')
+ if (!missing) {
+ return null
+ }
+ return `Grant ${missing.id === 'accessibility' ? 'Accessibility' : 'Screen Recording'} to Orca Computer Use, then retry get-app-state.`
+}
diff --git a/src/main/computer/macos-native-provider-client.ts b/src/main/computer/macos-native-provider-client.ts
new file mode 100644
index 00000000000..6f4bf751b78
--- /dev/null
+++ b/src/main/computer/macos-native-provider-client.ts
@@ -0,0 +1,282 @@
+/* eslint-disable max-lines -- Why: the macOS provider transport owns one lifecycle across stdio fallback and helper-app socket mode. */
+import { spawn } from 'child_process'
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'fs'
+import type net from 'net'
+import { release, tmpdir } from 'os'
+import { join } from 'path'
+import { randomUUID } from 'crypto'
+import type {
+ ComputerActionResult,
+ ComputerListAppsResult,
+ ComputerListWindowsResult,
+ ComputerProviderCapabilities,
+ ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import {
+ assertMacOSProviderCapability,
+ REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION,
+ type NativeActionMethod,
+ type NativeMethod,
+ type NativeResponse,
+ type PendingNativeRequest,
+ writeNativeProviderLine
+} from './macos-native-provider-contract'
+import { resolveMacOSComputerUseAppPath } from './macos-native-provider-paths'
+import { connectMacOSProviderSocket } from './macos-native-provider-socket'
+import { RuntimeClientError } from './runtime-client-error'
+
+const REQUEST_TIMEOUT_MS = 60_000
+const HELPER_CONNECT_TIMEOUT_MS = 10_000
+
+export function shouldUseMacOSNativeProvider(): boolean {
+ return (
+ process.platform === 'darwin' && isMacOS14OrNewer() && resolveMacOSComputerUseAppPath() !== null
+ )
+}
+
+export class MacOSNativeProviderClient {
+ private socket: net.Socket | null = null
+ private socketStartPromise: Promise | null = null
+ private socketPath: string | null = null
+ private socketDirectory: string | null = null
+ private socketTokenPath: string | null = null
+ private socketToken: string | null = null
+ private nextId = 1
+ private pending = new Map()
+ private socketBuffer = ''
+ private providerCapabilities: ComputerProviderCapabilities | null = null
+ async listApps(): Promise {
+ return (await this.call('listApps', {})) as ComputerListAppsResult
+ }
+ async capabilities(): Promise {
+ await this.ensureCompatible()
+ return this.providerCapabilities!
+ }
+ async listWindows(params: unknown): Promise {
+ await this.ensureCapability('windows', 'list')
+ return (await this.call('listWindows', params)) as ComputerListWindowsResult
+ }
+ async snapshot(params: unknown): Promise {
+ return (await this.call('getAppState', params)) as ComputerSnapshotResult
+ }
+ async action(method: NativeActionMethod, params: unknown): Promise {
+ return (await this.call(method, params)) as ComputerActionResult
+ }
+ shutdown(): void {
+ const socket = this.socket
+ const token = this.socketToken
+ this.socket = null
+ this.socketStartPromise = null
+ this.providerCapabilities = null
+ if (socket && !socket.destroyed) {
+ const id = this.nextId++
+ socket.write(`${JSON.stringify({ id, method: 'terminate', params: {}, token })}\n`)
+ socket.end()
+ }
+ for (const [id, pending] of this.pending) {
+ clearTimeout(pending.timer)
+ pending.reject(
+ new RuntimeClientError('accessibility_error', 'native macOS provider shut down')
+ )
+ this.pending.delete(id)
+ }
+ this.cleanupSocketDirectory()
+ }
+ private async call(method: NativeMethod, params: unknown): Promise {
+ if (method !== 'handshake') {
+ await this.ensureCompatible()
+ }
+ return await this.send(method, params)
+ }
+ private async send(method: NativeMethod, params: unknown): Promise {
+ const id = this.nextId++
+ const helperAppPath = resolveMacOSComputerUseAppPath()
+ if (!helperAppPath) {
+ throw new RuntimeClientError('accessibility_error', 'Orca Computer Use.app was not found')
+ }
+ const transport = await this.ensureSocketStarted(helperAppPath)
+ const token = this.socketToken
+ const line = `${JSON.stringify({ id, method, params, token })}\n`
+ const result = new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.pending.delete(id)
+ this.shutdown()
+ reject(
+ new RuntimeClientError('action_timeout', `native macOS provider ${method} timed out`)
+ )
+ }, REQUEST_TIMEOUT_MS)
+
+ this.pending.set(id, { resolve, reject, timer })
+ })
+ try {
+ await writeNativeProviderLine(transport, line)
+ } catch (error) {
+ const pending = this.pending.get(id)
+ if (pending) {
+ clearTimeout(pending.timer)
+ this.pending.delete(id)
+ }
+ throw new RuntimeClientError(
+ 'accessibility_error',
+ error instanceof Error ? error.message : String(error)
+ )
+ }
+ return await result
+ }
+ private async ensureCompatible(): Promise {
+ if (this.providerCapabilities) {
+ return
+ }
+ const capabilities = await this.readCapabilities()
+ if (capabilities.protocolVersion === REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION) {
+ this.providerCapabilities = capabilities
+ return
+ }
+ this.shutdown()
+ const restarted = await this.readCapabilities()
+ if (restarted.protocolVersion !== REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION) {
+ throw new RuntimeClientError(
+ 'provider_incompatible',
+ `native macOS provider protocol ${restarted.protocolVersion} is incompatible with required protocol ${REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION}`
+ )
+ }
+ this.providerCapabilities = restarted
+ }
+ private async readCapabilities(): Promise {
+ return (await this.send('handshake', {})) as ComputerProviderCapabilities
+ }
+ private async ensureCapability(
+ group: keyof ComputerProviderCapabilities['supports'],
+ capability: string
+ ): Promise {
+ await this.ensureCompatible()
+ if (assertMacOSProviderCapability(this.providerCapabilities, group, capability)) {
+ return
+ }
+ throw new RuntimeClientError(
+ 'unsupported_capability',
+ `native macOS provider does not support ${String(group)}.${capability}`
+ )
+ }
+ private async ensureSocketStarted(helperAppPath: string): Promise {
+ if (this.socket && !this.socket.destroyed) {
+ return this.socket
+ }
+ if (this.socketStartPromise) {
+ return await this.socketStartPromise
+ }
+ this.socketStartPromise = this.startSocket(helperAppPath)
+ try {
+ return await this.socketStartPromise
+ } finally {
+ this.socketStartPromise = null
+ }
+ }
+ private async startSocket(helperAppPath: string): Promise {
+ this.socketDirectory = mkdtempSync(join(tmpdir(), 'orca-computer-use-'))
+ chmodSync(this.socketDirectory, 0o700)
+ this.socketPath = join(this.socketDirectory, 'provider.sock')
+ this.socketToken = randomUUID()
+ this.socketTokenPath = join(this.socketDirectory, 'provider.token')
+ writeFileSync(this.socketTokenPath, this.socketToken, { encoding: 'utf8', mode: 0o600 })
+ // Why: macOS TCC attaches Accessibility/Screen Recording to the helper
+ // app bundle; direct child-process stdio cannot reliably read AX windows.
+ const opener = spawn(
+ '/usr/bin/open',
+ [
+ '-n',
+ helperAppPath,
+ '--args',
+ '--agent',
+ this.socketPath,
+ '--token-file',
+ this.socketTokenPath
+ ],
+ { detached: true, stdio: 'ignore' }
+ )
+ opener.unref()
+ try {
+ const socket = await connectMacOSProviderSocket(this.socketPath, HELPER_CONNECT_TIMEOUT_MS)
+ socket.setEncoding('utf8')
+ socket.on('data', (chunk: string) => this.handleSocketData(chunk))
+ socket.on('close', () => this.handleSocketClose())
+ socket.on('error', (error) => this.handleTransportError(error))
+ this.socket = socket
+ return socket
+ } catch (error) {
+ this.cleanupSocketDirectory()
+ this.socketPath = null
+ this.socketTokenPath = null
+ this.socketToken = null
+ throw error
+ }
+ }
+ private handleSocketData(chunk: string): void {
+ this.socketBuffer += chunk
+ this.socketBuffer = this.consumeLines(this.socketBuffer)
+ }
+ private consumeLines(buffer: string): string {
+ let remaining = buffer
+ while (true) {
+ const newline = remaining.indexOf('\n')
+ if (newline < 0) {
+ return remaining
+ }
+ const line = remaining.slice(0, newline)
+ remaining = remaining.slice(newline + 1)
+ if (line.trim()) {
+ this.handleLine(line)
+ }
+ }
+ }
+ private handleLine(line: string): void {
+ let response: NativeResponse
+ try {
+ response = JSON.parse(line) as NativeResponse
+ } catch {
+ return
+ }
+ const pending = this.pending.get(response.id)
+ if (!pending) {
+ return
+ }
+ clearTimeout(pending.timer)
+ this.pending.delete(response.id)
+ if (response.ok) {
+ pending.resolve(response.result)
+ return
+ }
+ pending.reject(new RuntimeClientError(response.error.code, response.error.message))
+ }
+ private handleSocketClose(): void {
+ this.socket = null
+ this.cleanupSocketDirectory()
+ this.rejectPending(
+ new RuntimeClientError('accessibility_error', 'native macOS helper app connection closed')
+ )
+ }
+ private handleTransportError(error: Error): void {
+ this.rejectPending(new RuntimeClientError('accessibility_error', error.message))
+ }
+ private cleanupSocketDirectory(): void {
+ if (!this.socketDirectory) {
+ return
+ }
+ rmSync(this.socketDirectory, { recursive: true, force: true })
+ this.socketDirectory = null
+ this.socketPath = null
+ this.socketTokenPath = null
+ }
+ private rejectPending(error: Error): void {
+ for (const [id, pending] of this.pending) {
+ clearTimeout(pending.timer)
+ pending.reject(error)
+ this.pending.delete(id)
+ }
+ }
+}
+
+function isMacOS14OrNewer(): boolean {
+ const darwinMajor = Number.parseInt(release().split('.')[0] ?? '', 10)
+ return Number.isFinite(darwinMajor) && darwinMajor >= 23
+}
diff --git a/src/main/computer/macos-native-provider-contract.ts b/src/main/computer/macos-native-provider-contract.ts
new file mode 100644
index 00000000000..73e8341cf7f
--- /dev/null
+++ b/src/main/computer/macos-native-provider-contract.ts
@@ -0,0 +1,56 @@
+import type net from 'net'
+import type { ComputerProviderCapabilities } from '../../shared/runtime-types'
+
+export type NativeMethod =
+ | 'handshake'
+ | 'listApps'
+ | 'listWindows'
+ | 'getAppState'
+ | 'click'
+ | 'performSecondaryAction'
+ | 'scroll'
+ | 'drag'
+ | 'typeText'
+ | 'pressKey'
+ | 'hotkey'
+ | 'pasteText'
+ | 'setValue'
+ | 'terminate'
+
+export type NativeActionMethod = Exclude<
+ NativeMethod,
+ 'handshake' | 'listApps' | 'listWindows' | 'getAppState' | 'terminate'
+>
+
+export type NativeResponse =
+ | { id: number; ok: true; result: unknown }
+ | { id: number; ok: false; error: { code: string; message: string } }
+
+export type PendingNativeRequest = {
+ resolve: (value: unknown) => void
+ reject: (error: Error) => void
+ timer: NodeJS.Timeout
+}
+
+export const REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION = 1
+
+export function assertMacOSProviderCapability(
+ capabilities: ComputerProviderCapabilities | null,
+ group: keyof ComputerProviderCapabilities['supports'],
+ capability: string
+): boolean {
+ const groupCapabilities = capabilities?.supports[group] as Record | undefined
+ return groupCapabilities?.[capability] === true
+}
+
+export function writeNativeProviderLine(transport: net.Socket, line: string): Promise {
+ return new Promise((resolve, reject) => {
+ transport.write(line, (error) => {
+ if (error) {
+ reject(error)
+ return
+ }
+ resolve()
+ })
+ })
+}
diff --git a/src/main/computer/macos-native-provider-paths.ts b/src/main/computer/macos-native-provider-paths.ts
new file mode 100644
index 00000000000..54179ec413f
--- /dev/null
+++ b/src/main/computer/macos-native-provider-paths.ts
@@ -0,0 +1,44 @@
+import { existsSync } from 'fs'
+import { join, resolve } from 'path'
+
+export function resolveMacOSComputerUseAppPath(): string | null {
+ const override = process.env.ORCA_COMPUTER_MACOS_HELPER_APP_PATH
+ if (override && existsSync(override)) {
+ return override
+ }
+
+ const packaged = [join(process.resourcesPath ?? '', 'Orca Computer Use.app')]
+ const dev = [
+ join(process.cwd(), 'native/computer-use-macos/.build/release/Orca Computer Use.app'),
+ resolve(__dirname, '../../native/computer-use-macos/.build/release/Orca Computer Use.app')
+ ]
+ const candidates = process.resourcesPath ? [...packaged, ...dev] : dev
+
+ return candidates.find((candidate) => candidate && existsSync(candidate)) ?? null
+}
+
+export function resolveMacOSComputerUseExecutablePath(): string | null {
+ const appPath = resolveMacOSComputerUseAppPath()
+ if (!appPath) {
+ return null
+ }
+ const executablePath = join(appPath, 'Contents', 'MacOS', 'orca-computer-use-macos')
+ return existsSync(executablePath) ? executablePath : null
+}
+
+export function resolveMacOSNativeProviderPath(): string | null {
+ const override = process.env.ORCA_COMPUTER_MACOS_PROVIDER_PATH
+ if (override && existsSync(override)) {
+ return override
+ }
+
+ const packaged = [join(process.resourcesPath ?? '', 'computer-use-macos/orca-computer-use-macos')]
+ const dev = [
+ join(process.cwd(), 'native/computer-use-macos/.build/debug/orca-computer-use-macos'),
+ join(process.cwd(), 'native/computer-use-macos/.build/release/orca-computer-use-macos'),
+ resolve(__dirname, '../../native/computer-use-macos/.build/debug/orca-computer-use-macos')
+ ]
+ const candidates = process.resourcesPath ? [...packaged, ...dev] : dev
+
+ return candidates.find((candidate) => candidate && existsSync(candidate)) ?? null
+}
diff --git a/src/main/computer/macos-native-provider-socket.ts b/src/main/computer/macos-native-provider-socket.ts
new file mode 100644
index 00000000000..021ab111794
--- /dev/null
+++ b/src/main/computer/macos-native-provider-socket.ts
@@ -0,0 +1,37 @@
+import net from 'net'
+import { RuntimeClientError } from './runtime-client-error'
+
+export async function connectMacOSProviderSocket(
+ socketPath: string,
+ timeoutMs: number
+): Promise {
+ const deadline = Date.now() + timeoutMs
+ let lastError: Error | null = null
+ while (Date.now() < deadline) {
+ try {
+ return await connectSocket(socketPath)
+ } catch (error) {
+ lastError = error instanceof Error ? error : new Error(String(error))
+ await new Promise((resolve) => setTimeout(resolve, 100))
+ }
+ }
+ throw new RuntimeClientError(
+ 'action_timeout',
+ `native macOS helper app did not open its socket: ${lastError?.message ?? 'timed out'}`
+ )
+}
+
+function connectSocket(socketPath: string): Promise {
+ return new Promise((resolve, reject) => {
+ const socket = net.createConnection(socketPath)
+ const onError = (error: Error) => {
+ socket.destroy()
+ reject(error)
+ }
+ socket.once('error', onError)
+ socket.once('connect', () => {
+ socket.off('error', onError)
+ resolve(socket)
+ })
+ })
+}
diff --git a/src/main/computer/permissions.ts b/src/main/computer/permissions.ts
new file mode 100644
index 00000000000..cb7a7dd91ca
--- /dev/null
+++ b/src/main/computer/permissions.ts
@@ -0,0 +1,51 @@
+import { Notification, shell, systemPreferences } from 'electron'
+
+const ACCESSIBILITY_SETTINGS_URL =
+ 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility'
+const DEFAULT_ACCESSIBILITY_INSTRUCTIONS =
+ 'System Settings -> Privacy & Security -> Accessibility -> enable Orca'
+
+const activePermissionNotifications = new Set()
+
+/** Probe accessibility permissions; lazy -- invoked only on first failure path. */
+export async function checkAccessibilityPermission(): Promise<{
+ ok: boolean
+ instructions?: string
+}> {
+ if (process.platform !== 'darwin') {
+ return { ok: true }
+ }
+
+ try {
+ const ok = systemPreferences.isTrustedAccessibilityClient(false)
+ return ok ? { ok: true } : { ok: false, instructions: DEFAULT_ACCESSIBILITY_INSTRUCTIONS }
+ } catch {
+ return { ok: false, instructions: DEFAULT_ACCESSIBILITY_INSTRUCTIONS }
+ }
+}
+
+/** Surface a notification through Orca's existing notification system (do not duplicate UI). */
+export function notifyPermissionRequired(instructions: string): void {
+ if (!Notification.isSupported()) {
+ return
+ }
+
+ const notification = new Notification({
+ title: 'Accessibility permission required',
+ body: instructions
+ })
+ activePermissionNotifications.add(notification)
+
+ const release = (): void => {
+ activePermissionNotifications.delete(notification)
+ }
+ notification.on('close', release)
+ notification.on('click', () => {
+ release()
+ if (process.platform === 'darwin') {
+ void shell.openExternal(ACCESSIBILITY_SETTINGS_URL)
+ }
+ })
+ setTimeout(release, 5 * 60 * 1000)
+ notification.show()
+}
diff --git a/src/main/computer/runtime-client-error.ts b/src/main/computer/runtime-client-error.ts
new file mode 100644
index 00000000000..f0773c6cd8c
--- /dev/null
+++ b/src/main/computer/runtime-client-error.ts
@@ -0,0 +1,9 @@
+export class RuntimeClientError extends Error {
+ readonly code: string
+
+ constructor(code: string, message: string) {
+ super(message)
+ this.name = 'RuntimeClientError'
+ this.code = code
+ }
+}
diff --git a/src/main/computer/sidecar-client.ts b/src/main/computer/sidecar-client.ts
new file mode 100644
index 00000000000..8981a0c6568
--- /dev/null
+++ b/src/main/computer/sidecar-client.ts
@@ -0,0 +1,227 @@
+import { fork, type ChildProcess } from 'child_process'
+import { join } from 'path'
+import type {
+ ComputerActionResult,
+ ComputerListAppsResult,
+ ComputerListWindowsResult,
+ ComputerProviderCapabilities,
+ ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import { RuntimeClientError } from './runtime-client-error'
+
+type ComputerSidecarMethod =
+ | 'capabilities'
+ | 'listApps'
+ | 'listWindows'
+ | 'getAppState'
+ | 'click'
+ | 'performSecondaryAction'
+ | 'scroll'
+ | 'drag'
+ | 'typeText'
+ | 'pressKey'
+ | 'hotkey'
+ | 'pasteText'
+ | 'setValue'
+
+type ComputerSidecarRequest = {
+ id: number
+ method: ComputerSidecarMethod
+ params: unknown
+}
+
+type ComputerSidecarResponse =
+ | { id: number; ok: true; result: unknown }
+ | { id: number; ok: false; error: { code: string; message: string } }
+
+type PendingRequest = {
+ resolve: (value: unknown) => void
+ reject: (error: Error) => void
+ timer: NodeJS.Timeout
+}
+
+const REQUEST_TIMEOUT_MS = 60_000
+let sidecar: ComputerSidecarProcess | null = null
+
+export function shouldUseComputerSidecar(): boolean {
+ return (
+ (process.platform === 'darwin' ||
+ process.platform === 'linux' ||
+ process.platform === 'win32') &&
+ typeof process.versions.electron === 'string' &&
+ process.env.ORCA_COMPUTER_SIDECAR !== '1'
+ )
+}
+
+export async function callComputerSidecarListApps(): Promise {
+ return (await getComputerSidecar().call('listApps', {})) as ComputerListAppsResult
+}
+
+export async function callComputerSidecarCapabilities(): Promise {
+ return (await getComputerSidecar().call('capabilities', {})) as ComputerProviderCapabilities
+}
+
+export async function callComputerSidecarListWindows(
+ params: unknown
+): Promise {
+ return (await getComputerSidecar().call('listWindows', params)) as ComputerListWindowsResult
+}
+
+export async function callComputerSidecarSnapshot(
+ params: unknown
+): Promise {
+ return (await getComputerSidecar().call('getAppState', params)) as ComputerSnapshotResult
+}
+
+export async function callComputerSidecarAction(
+ method: Exclude<
+ ComputerSidecarMethod,
+ 'capabilities' | 'listApps' | 'listWindows' | 'getAppState'
+ >,
+ params: unknown
+): Promise {
+ return (await getComputerSidecar().call(method, params)) as ComputerActionResult
+}
+
+export function resetComputerSidecarForTest(): void {
+ sidecar?.shutdown()
+ sidecar = null
+}
+
+function getComputerSidecar(): ComputerSidecarProcess {
+ if (!sidecar) {
+ sidecar = new ComputerSidecarProcess(getComputerSidecarEntryPath())
+ }
+ return sidecar
+}
+
+function getComputerSidecarEntryPath(): string {
+ const app = loadElectronApp()
+ const appPath = app?.getAppPath() ?? process.cwd()
+ const isPackaged = app?.isPackaged ?? false
+ // Why: packaged sidecars must be forked from app.asar.unpacked because
+ // ELECTRON_RUN_AS_NODE bypasses Electron's asar require integration.
+ const basePath = isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath
+ return join(basePath, 'out', 'main', 'computer-sidecar.js')
+}
+
+function loadElectronApp(): { getAppPath(): string; isPackaged: boolean } | null {
+ try {
+ return require('electron').app
+ } catch {
+ return null
+ }
+}
+
+class ComputerSidecarProcess {
+ private child: ChildProcess | null = null
+ private nextId = 1
+ private pending = new Map()
+
+ constructor(private readonly entryPath: string) {}
+
+ call(method: ComputerSidecarMethod, params: unknown): Promise {
+ const child = this.ensureStarted()
+ const id = this.nextId++
+ const request: ComputerSidecarRequest = { id, method, params }
+
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.pending.delete(id)
+ this.shutdown()
+ reject(new RuntimeClientError('action_timeout', `computer sidecar ${method} timed out`))
+ }, REQUEST_TIMEOUT_MS)
+
+ this.pending.set(id, { resolve, reject, timer })
+ child.send?.(request, (error) => {
+ if (!error) {
+ return
+ }
+ clearTimeout(timer)
+ this.pending.delete(id)
+ reject(new RuntimeClientError('accessibility_error', error.message))
+ })
+ })
+ }
+
+ shutdown(): void {
+ const child = this.child
+ this.child = null
+ for (const [id, pending] of this.pending) {
+ clearTimeout(pending.timer)
+ pending.reject(new RuntimeClientError('accessibility_error', 'computer sidecar shut down'))
+ this.pending.delete(id)
+ }
+ child?.kill('SIGTERM')
+ }
+
+ private ensureStarted(): ChildProcess {
+ if (this.child && !this.child.killed) {
+ return this.child
+ }
+
+ const child = fork(this.entryPath, [], {
+ stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+ env: {
+ ...process.env,
+ ELECTRON_RUN_AS_NODE: '1',
+ ORCA_COMPUTER_SIDECAR: '1'
+ },
+ ...(process.platform === 'win32' ? { windowsHide: true } : {})
+ })
+
+ child.on('message', (message) => this.handleMessage(message))
+ child.on('exit', (code, signal) => this.handleExit(code, signal))
+ child.on('error', (error) => this.handleError(error))
+ this.child = child
+ return child
+ }
+
+ private handleMessage(message: unknown): void {
+ if (!isSidecarResponse(message)) {
+ return
+ }
+ const pending = this.pending.get(message.id)
+ if (!pending) {
+ return
+ }
+ clearTimeout(pending.timer)
+ this.pending.delete(message.id)
+ if (message.ok) {
+ pending.resolve(message.result)
+ return
+ }
+ pending.reject(new RuntimeClientError(message.error.code, message.error.message))
+ }
+
+ private handleExit(code: number | null, signal: NodeJS.Signals | null): void {
+ this.child = null
+ const detail = signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`
+ const error = new RuntimeClientError(
+ 'accessibility_error',
+ `computer sidecar exited with ${detail}`
+ )
+ for (const [id, pending] of this.pending) {
+ clearTimeout(pending.timer)
+ pending.reject(error)
+ this.pending.delete(id)
+ }
+ }
+
+ private handleError(error: Error): void {
+ const wrapped = new RuntimeClientError('accessibility_error', error.message)
+ for (const [id, pending] of this.pending) {
+ clearTimeout(pending.timer)
+ pending.reject(wrapped)
+ this.pending.delete(id)
+ }
+ }
+}
+
+function isSidecarResponse(message: unknown): message is ComputerSidecarResponse {
+ if (!message || typeof message !== 'object') {
+ return false
+ }
+ const record = message as Record
+ return typeof record.id === 'number' && typeof record.ok === 'boolean'
+}
diff --git a/src/main/computer/sidecar-entry.ts b/src/main/computer/sidecar-entry.ts
new file mode 100644
index 00000000000..8b6ccb1e9ac
--- /dev/null
+++ b/src/main/computer/sidecar-entry.ts
@@ -0,0 +1,137 @@
+/* eslint-disable max-lines -- Why: sidecar dispatch keeps one method table shared across native providers. */
+import {
+ MacOSNativeProviderClient,
+ shouldUseMacOSNativeProvider
+} from './macos-native-provider-client'
+import {
+ DesktopScriptProviderClient,
+ shouldUseDesktopScriptProvider
+} from './desktop-script-provider-client'
+import { RuntimeClientError } from './runtime-client-error'
+
+type SidecarRequest = {
+ id: number
+ method: string
+ params?: Record
+}
+
+const nativeMacOSProvider = shouldUseMacOSNativeProvider() ? new MacOSNativeProviderClient() : null
+const desktopScriptProvider = shouldUseDesktopScriptProvider()
+ ? new DesktopScriptProviderClient()
+ : null
+
+process.once('disconnect', shutdownProviders)
+process.once('SIGTERM', () => {
+ shutdownProviders()
+ process.exit(0)
+})
+process.once('SIGINT', () => {
+ shutdownProviders()
+ process.exit(130)
+})
+process.once('beforeExit', shutdownProviders)
+
+process.on('message', (message: unknown) => {
+ void handleMessage(message)
+})
+
+async function handleMessage(message: unknown): Promise {
+ if (!isRequest(message)) {
+ return
+ }
+
+ try {
+ const result = await dispatch(message.method, message.params ?? {})
+ process.send?.({ id: message.id, ok: true, result })
+ } catch (error) {
+ const mapped = errorToResponse(error)
+ process.send?.({ id: message.id, ok: false, error: mapped })
+ }
+}
+
+async function dispatch(method: string, params: Record): Promise {
+ const provider = nativeMacOSProvider ?? desktopScriptProvider
+ if (!provider) {
+ throw new RuntimeClientError(
+ 'unsupported_capability',
+ `computer-use has no native provider for ${process.platform}`
+ )
+ }
+
+ switch (method) {
+ case 'capabilities': {
+ return await provider.capabilities()
+ }
+ case 'listApps': {
+ return await provider.listApps()
+ }
+ case 'listWindows': {
+ return await provider.listWindows(params)
+ }
+ case 'getAppState': {
+ return await provider.snapshot(params)
+ }
+ case 'click': {
+ return await provider.action('click', params)
+ }
+ case 'performSecondaryAction': {
+ return await provider.action('performSecondaryAction', params)
+ }
+ case 'scroll': {
+ return await provider.action('scroll', params)
+ }
+ case 'drag': {
+ return await provider.action('drag', params)
+ }
+ case 'typeText': {
+ return await provider.action('typeText', params)
+ }
+ case 'pressKey': {
+ return await provider.action('pressKey', params)
+ }
+ case 'hotkey': {
+ return await provider.action('hotkey', params)
+ }
+ case 'pasteText': {
+ return await provider.action('pasteText', params)
+ }
+ case 'setValue': {
+ return await provider.action('setValue', params)
+ }
+ default:
+ throw new RuntimeClientError(
+ 'invalid_argument',
+ `unknown computer sidecar method '${method}'`
+ )
+ }
+}
+
+function isRequest(message: unknown): message is SidecarRequest {
+ if (!message || typeof message !== 'object') {
+ return false
+ }
+ const record = message as Record
+ return (
+ typeof record.id === 'number' &&
+ typeof record.method === 'string' &&
+ (record.params === undefined || (typeof record.params === 'object' && record.params !== null))
+ )
+}
+
+function errorToResponse(error: unknown): { code: string; message: string } {
+ if (
+ error instanceof Error &&
+ 'code' in error &&
+ typeof (error as { code: unknown }).code === 'string'
+ ) {
+ return { code: (error as { code: string }).code, message: error.message }
+ }
+ return {
+ code: 'accessibility_error',
+ message: error instanceof Error ? error.message : String(error)
+ }
+}
+
+function shutdownProviders(): void {
+ nativeMacOSProvider?.shutdown()
+}
diff --git a/src/main/ipc/computer-use-permissions.test.ts b/src/main/ipc/computer-use-permissions.test.ts
new file mode 100644
index 00000000000..fea36c778d8
--- /dev/null
+++ b/src/main/ipc/computer-use-permissions.test.ts
@@ -0,0 +1,71 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { handleMock, openComputerUsePermissionsMock, getComputerUsePermissionStatusMock } =
+ vi.hoisted(() => ({
+ handleMock: vi.fn(),
+ openComputerUsePermissionsMock: vi.fn(),
+ getComputerUsePermissionStatusMock: vi.fn()
+ }))
+
+vi.mock('electron', () => ({
+ ipcMain: {
+ handle: handleMock
+ }
+}))
+
+vi.mock('../computer/macos-computer-use-permissions', () => ({
+ getComputerUsePermissionStatus: getComputerUsePermissionStatusMock,
+ openComputerUsePermissions: openComputerUsePermissionsMock
+}))
+
+import { registerComputerUsePermissionHandlers } from './computer-use-permissions'
+
+describe('registerComputerUsePermissionHandlers', () => {
+ beforeEach(() => {
+ handleMock.mockReset()
+ getComputerUsePermissionStatusMock.mockReset()
+ openComputerUsePermissionsMock.mockReset()
+ })
+
+ it('launches the computer-use helper setup', async () => {
+ const result = {
+ platform: 'darwin',
+ helperAppPath: '/Applications/Orca Computer Use.app',
+ permissionId: 'accessibility',
+ openedSettings: false,
+ launchedHelper: true
+ }
+ openComputerUsePermissionsMock.mockReturnValue(result)
+
+ registerComputerUsePermissionHandlers()
+
+ const registration = handleMock.mock.calls.find(
+ ([channel]) => channel === 'computerUsePermissions:openSetup'
+ )
+ expect(registration).toBeTruthy()
+
+ await expect(registration).resolves.toBe(result)
+ expect(openComputerUsePermissionsMock).toHaveBeenCalledWith('accessibility')
+ })
+
+ it('returns computer-use permission status', async () => {
+ const result = {
+ platform: 'darwin',
+ permissions: [
+ { id: 'accessibility', status: 'granted' },
+ { id: 'screenshots', status: 'not-granted' }
+ ]
+ }
+ getComputerUsePermissionStatusMock.mockReturnValue(result)
+
+ registerComputerUsePermissionHandlers()
+
+ const registration = handleMock.mock.calls.find(
+ ([channel]) => channel === 'computerUsePermissions:getStatus'
+ )
+ expect(registration).toBeTruthy()
+
+ await expect(registration![1]()).resolves.toBe(result)
+ expect(getComputerUsePermissionStatusMock).toHaveBeenCalledWith()
+ })
+})
diff --git a/src/main/ipc/computer-use-permissions.ts b/src/main/ipc/computer-use-permissions.ts
new file mode 100644
index 00000000000..8665fa9a096
--- /dev/null
+++ b/src/main/ipc/computer-use-permissions.ts
@@ -0,0 +1,28 @@
+import { ipcMain } from 'electron'
+import type {
+ ComputerUsePermissionId,
+ ComputerUsePermissionSetupResult,
+ ComputerUsePermissionStatusResult
+} from '../../shared/computer-use-permissions-types'
+
+export function registerComputerUsePermissionHandlers(): void {
+ ipcMain.handle(
+ 'computerUsePermissions:openSetup',
+ async (
+ _event,
+ args?: { id?: ComputerUsePermissionId }
+ ): Promise => {
+ const { openComputerUsePermissions } =
+ await import('../computer/macos-computer-use-permissions')
+ return openComputerUsePermissions(args?.id)
+ }
+ )
+ ipcMain.handle(
+ 'computerUsePermissions:getStatus',
+ async (): Promise => {
+ const { getComputerUsePermissionStatus } =
+ await import('../computer/macos-computer-use-permissions')
+ return getComputerUsePermissionStatus()
+ }
+ )
+}
diff --git a/src/main/ipc/register-core-handlers.test.ts b/src/main/ipc/register-core-handlers.test.ts
index 40840246e0e..76d422abfeb 100644
--- a/src/main/ipc/register-core-handlers.test.ts
+++ b/src/main/ipc/register-core-handlers.test.ts
@@ -11,6 +11,7 @@ const {
registerMemoryHandlersMock,
registerNotificationHandlersMock,
registerDeveloperPermissionHandlersMock,
+ registerComputerUsePermissionHandlersMock,
registerSettingsHandlersMock,
registerTelemetryHandlersMock,
registerShellHandlersMock,
@@ -45,6 +46,7 @@ const {
registerMemoryHandlersMock: vi.fn(),
registerNotificationHandlersMock: vi.fn(),
registerDeveloperPermissionHandlersMock: vi.fn(),
+ registerComputerUsePermissionHandlersMock: vi.fn(),
registerSettingsHandlersMock: vi.fn(),
registerTelemetryHandlersMock: vi.fn(),
registerShellHandlersMock: vi.fn(),
@@ -118,6 +120,10 @@ vi.mock('./developer-permissions', () => ({
registerDeveloperPermissionHandlers: registerDeveloperPermissionHandlersMock
}))
+vi.mock('./computer-use-permissions', () => ({
+ registerComputerUsePermissionHandlers: registerComputerUsePermissionHandlersMock
+}))
+
vi.mock('./settings', () => ({
registerSettingsHandlers: registerSettingsHandlersMock
}))
@@ -207,6 +213,7 @@ describe('registerCoreHandlers', () => {
registerMemoryHandlersMock.mockReset()
registerNotificationHandlersMock.mockReset()
registerDeveloperPermissionHandlersMock.mockReset()
+ registerComputerUsePermissionHandlersMock.mockReset()
registerSettingsHandlersMock.mockReset()
registerTelemetryHandlersMock.mockReset()
registerShellHandlersMock.mockReset()
@@ -266,6 +273,7 @@ describe('registerCoreHandlers', () => {
expect(registerMemoryHandlersMock).toHaveBeenCalledWith(store)
expect(registerNotificationHandlersMock).toHaveBeenCalledWith(store, runtime)
expect(registerDeveloperPermissionHandlersMock).toHaveBeenCalled()
+ expect(registerComputerUsePermissionHandlersMock).toHaveBeenCalled()
expect(registerSettingsHandlersMock).toHaveBeenCalledWith(store)
expect(registerTelemetryHandlersMock).toHaveBeenCalledWith(store)
expect(registerSessionHandlersMock).toHaveBeenCalledWith(store)
diff --git a/src/main/ipc/register-core-handlers.ts b/src/main/ipc/register-core-handlers.ts
index b69896f3b52..cf0a7467d11 100644
--- a/src/main/ipc/register-core-handlers.ts
+++ b/src/main/ipc/register-core-handlers.ts
@@ -19,6 +19,7 @@ import { registerRuntimeHandlers } from './runtime'
import { registerNotificationHandlers } from './notifications'
import { registerOnboardingHandlers } from './onboarding'
import { registerDeveloperPermissionHandlers } from './developer-permissions'
+import { registerComputerUsePermissionHandlers } from './computer-use-permissions'
import { setTrustedBrowserRendererWebContentsId, setAgentBrowserBridgeRef } from './browser'
import { registerSessionHandlers } from './session'
import { registerSettingsHandlers } from './settings'
@@ -86,6 +87,7 @@ export function registerCoreHandlers(
registerNotificationHandlers(store, runtime)
registerOnboardingHandlers(store)
registerDeveloperPermissionHandlers()
+ registerComputerUsePermissionHandlers()
registerSettingsHandlers(store)
registerTelemetryHandlers(store)
registerBrowserHandlers()
diff --git a/src/main/runtime/rpc/errors.ts b/src/main/runtime/rpc/errors.ts
index e53ab52c276..da19f1cba1a 100644
--- a/src/main/runtime/rpc/errors.ts
+++ b/src/main/runtime/rpc/errors.ts
@@ -3,6 +3,7 @@
// format human-facing messages. Centralizing this mapping keeps the allowlist
// auditable in one place instead of spread across per-method branches.
import type { RpcEnvelopeMeta, RpcFailure, RpcSuccess } from './core'
+import { COMPUTER_ERROR_CODES } from '../../../shared/runtime-types'
export function successResponse(id: string, meta: RpcEnvelopeMeta, result: unknown): RpcSuccess {
return {
@@ -46,8 +47,18 @@ const RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([
'invalid_limit'
])
+const COMPUTER_PASSTHROUGH_CODES: ReadonlySet = new Set(Object.values(COMPUTER_ERROR_CODES))
+
export function mapRuntimeError(id: string, meta: RpcEnvelopeMeta, error: unknown): RpcFailure {
const message = error instanceof Error ? error.message : String(error)
+ if (
+ error instanceof Error &&
+ 'code' in error &&
+ typeof (error as { code: unknown }).code === 'string' &&
+ COMPUTER_PASSTHROUGH_CODES.has((error as { code: string }).code)
+ ) {
+ return errorResponse(id, meta, (error as { code: string }).code, message)
+ }
if (RUNTIME_PASSTHROUGH_CODES.has(message)) {
return errorResponse(id, meta, message, message)
}
diff --git a/src/main/runtime/rpc/methods/computer.test.ts b/src/main/runtime/rpc/methods/computer.test.ts
new file mode 100644
index 00000000000..4791e518240
--- /dev/null
+++ b/src/main/runtime/rpc/methods/computer.test.ts
@@ -0,0 +1,341 @@
+/* eslint-disable max-lines -- Why: computer RPC coverage shares one mocked registry setup across all method contracts. */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { buildRegistry } from '../core'
+
+const computerMocks = vi.hoisted(() => ({
+ callComputerSidecarAction: vi.fn(),
+ callComputerSidecarCapabilities: vi.fn(),
+ callComputerSidecarListApps: vi.fn(),
+ callComputerSidecarListWindows: vi.fn(),
+ callComputerSidecarSnapshot: vi.fn(),
+ resetComputerSidecarForTest: vi.fn(),
+ openComputerUsePermissions: vi.fn()
+}))
+
+vi.mock('../../../computer/sidecar-client', () => ({
+ callComputerSidecarAction: computerMocks.callComputerSidecarAction,
+ callComputerSidecarCapabilities: computerMocks.callComputerSidecarCapabilities,
+ callComputerSidecarListApps: computerMocks.callComputerSidecarListApps,
+ callComputerSidecarListWindows: computerMocks.callComputerSidecarListWindows,
+ callComputerSidecarSnapshot: computerMocks.callComputerSidecarSnapshot,
+ resetComputerSidecarForTest: computerMocks.resetComputerSidecarForTest
+}))
+
+vi.mock('../../../computer/macos-computer-use-permissions', () => ({
+ openComputerUsePermissions: computerMocks.openComputerUsePermissions
+}))
+
+import { COMPUTER_METHODS, resetComputerSessionsForTest } from './computer'
+
+describe('computer RPC methods', () => {
+ beforeEach(() => {
+ computerMocks.callComputerSidecarAction.mockReset()
+ computerMocks.callComputerSidecarCapabilities.mockReset()
+ computerMocks.callComputerSidecarListApps.mockReset()
+ computerMocks.callComputerSidecarListWindows.mockReset()
+ computerMocks.callComputerSidecarSnapshot.mockReset()
+ computerMocks.resetComputerSidecarForTest.mockReset()
+ computerMocks.openComputerUsePermissions.mockReset()
+ resetComputerSessionsForTest()
+ computerMocks.resetComputerSidecarForTest.mockClear()
+ })
+
+ it('registers all computer methods', () => {
+ const registry = buildRegistry(COMPUTER_METHODS)
+
+ expect([...registry.keys()].sort()).toEqual([
+ 'computer.capabilities',
+ 'computer.click',
+ 'computer.drag',
+ 'computer.getAppState',
+ 'computer.hotkey',
+ 'computer.listApps',
+ 'computer.listWindows',
+ 'computer.pasteText',
+ 'computer.performSecondaryAction',
+ 'computer.permissions',
+ 'computer.pressKey',
+ 'computer.scroll',
+ 'computer.setValue',
+ 'computer.typeText'
+ ])
+ })
+
+ it('resets the sidecar test process', () => {
+ resetComputerSessionsForTest()
+
+ expect(computerMocks.resetComputerSidecarForTest).toHaveBeenCalledTimes(1)
+ })
+
+ it('lists running apps through the sidecar', async () => {
+ const result = {
+ apps: [{ name: 'Finder', bundleId: 'com.apple.finder', pid: 100 }]
+ }
+ computerMocks.callComputerSidecarListApps.mockResolvedValue(result)
+
+ await expect(call('computer.listApps', {})).resolves.toBe(result)
+ expect(computerMocks.callComputerSidecarListApps).toHaveBeenCalledWith()
+ })
+
+ it('returns provider capabilities through the sidecar', async () => {
+ const result = { platform: 'darwin', provider: 'orca-computer-use-macos', protocolVersion: 1 }
+ computerMocks.callComputerSidecarCapabilities.mockResolvedValue(result)
+
+ await expect(call('computer.capabilities', {})).resolves.toBe(result)
+ expect(computerMocks.callComputerSidecarCapabilities).toHaveBeenCalledWith()
+ })
+
+ it('opens computer-use permission setup', async () => {
+ const result = {
+ platform: 'darwin',
+ helperAppPath: '/Applications/Orca Computer Use.app',
+ openedSettings: false,
+ launchedHelper: true
+ }
+ computerMocks.openComputerUsePermissions.mockReturnValue(result)
+
+ await expect(call('computer.permissions', {})).resolves.toBe(result)
+ expect(computerMocks.openComputerUsePermissions).toHaveBeenCalledWith()
+ })
+
+ it('lists windows through the sidecar', async () => {
+ const result = {
+ app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+ windows: []
+ }
+ const params = { app: 'Finder', worktree: 'path:/tmp/repo' }
+ computerMocks.callComputerSidecarListWindows.mockResolvedValue(result)
+
+ await expect(call('computer.listWindows', params)).resolves.toBe(result)
+ expect(computerMocks.callComputerSidecarListWindows).toHaveBeenCalledWith(params)
+ })
+
+ it('gets app state through the sidecar', async () => {
+ const snapshot = {
+ snapshot: {
+ id: 'snap-test',
+ app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+ window: { title: 'Finder', width: 100, height: 100 },
+ treeText: 'tree',
+ elementCount: 1,
+ focusedElementId: null
+ },
+ screenshot: null,
+ screenshotStatus: { state: 'skipped', reason: 'no_screenshot_flag' }
+ }
+ const params = {
+ app: 'Finder',
+ worktree: 'path:/tmp/repo',
+ noScreenshot: true,
+ restoreWindow: true,
+ windowId: 123
+ }
+ computerMocks.callComputerSidecarSnapshot.mockResolvedValue(snapshot)
+
+ await expect(call('computer.getAppState', params)).resolves.toBe(snapshot)
+ expect(computerMocks.callComputerSidecarSnapshot).toHaveBeenCalledWith(params)
+ })
+
+ it('rejects missing app in getAppState schema', () => {
+ const method = findMethod('computer.getAppState')
+ expect(() => method.params!.parse({})).toThrow()
+ })
+
+ it('rejects ambiguous window targeting', () => {
+ expect(() =>
+ findMethod('computer.getAppState').params!.parse({
+ app: 'Finder',
+ windowId: 1,
+ windowIndex: 0
+ })
+ ).toThrow(/either --window-id or --window-index/)
+ expect(() =>
+ findMethod('computer.click').params!.parse({
+ app: 'Finder',
+ elementIndex: 0,
+ windowId: 1,
+ windowIndex: 0
+ })
+ ).toThrow(/either --window-id or --window-index/)
+ })
+
+ it('rejects incomplete pointer action coordinates', () => {
+ expect(() => findMethod('computer.click').params!.parse({ app: 'Finder' })).toThrow(
+ /Click requires/
+ )
+ expect(() => findMethod('computer.click').params!.parse({ app: 'Finder', x: 1 })).toThrow(
+ /both --x and --y/
+ )
+ expect(() =>
+ findMethod('computer.click').params!.parse({ app: 'Finder', elementIndex: 0, x: 1, y: 2 })
+ ).toThrow(/either --element-index or coordinate flags/)
+ expect(() =>
+ findMethod('computer.scroll').params!.parse({ app: 'Finder', direction: 'down' })
+ ).toThrow(/Scroll requires/)
+ expect(() =>
+ findMethod('computer.scroll').params!.parse({
+ app: 'Finder',
+ elementIndex: 0,
+ x: 1,
+ y: 2,
+ direction: 'down'
+ })
+ ).toThrow(/either --element-index or coordinate flags/)
+ expect(() =>
+ findMethod('computer.drag').params!.parse({ app: 'Finder', fromX: 1, fromY: 2 })
+ ).toThrow(/Drag coordinates/)
+ expect(() =>
+ findMethod('computer.drag').params!.parse({ app: 'Finder', fromElementIndex: 1 })
+ ).toThrow(/both --from-element-index and --to-element-index/)
+ expect(() =>
+ findMethod('computer.drag').params!.parse({
+ app: 'Finder',
+ fromElementIndex: 0,
+ toElementIndex: 1,
+ fromX: 1,
+ fromY: 2,
+ toX: 3,
+ toY: 4
+ })
+ ).toThrow(/either element indexes or coordinate flags/)
+ })
+
+ it('rejects unsupported scroll directions', () => {
+ expect(() =>
+ findMethod('computer.scroll').params!.parse({
+ app: 'Finder',
+ elementIndex: 0,
+ direction: 'diagonal'
+ })
+ ).toThrow()
+ expect(() =>
+ findMethod('computer.scroll').params!.parse({
+ app: 'Finder',
+ elementIndex: 0,
+ direction: 'down',
+ pages: 0
+ })
+ ).toThrow()
+ })
+
+ it('dispatches pointer and element actions through the sidecar', async () => {
+ computerMocks.callComputerSidecarAction.mockResolvedValue({ ok: true })
+
+ await call('computer.click', {
+ app: 'Finder',
+ worktree: 'path:/tmp/repo',
+ elementIndex: 0,
+ clickCount: 2,
+ mouseButton: 'left',
+ noScreenshot: true
+ })
+ await call('computer.performSecondaryAction', {
+ app: 'Finder',
+ elementIndex: 0,
+ action: 'Raise'
+ })
+ await call('computer.drag', {
+ app: 'Finder',
+ fromX: 1,
+ fromY: 2,
+ toX: 3,
+ toY: 4
+ })
+
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(1, 'click', {
+ app: 'Finder',
+ worktree: 'path:/tmp/repo',
+ elementIndex: 0,
+ clickCount: 2,
+ mouseButton: 'left',
+ noScreenshot: true
+ })
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(
+ 2,
+ 'performSecondaryAction',
+ {
+ app: 'Finder',
+ elementIndex: 0,
+ action: 'Raise'
+ }
+ )
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(3, 'drag', {
+ app: 'Finder',
+ fromX: 1,
+ fromY: 2,
+ toX: 3,
+ toY: 4
+ })
+ })
+
+ it('dispatches keyboard and text actions through the sidecar', async () => {
+ computerMocks.callComputerSidecarAction.mockResolvedValue({ ok: true })
+
+ await call('computer.typeText', { app: 'Finder', text: 'hello', noScreenshot: true })
+ await call('computer.pressKey', { app: 'Finder', key: 'Return' })
+ await call('computer.hotkey', { app: 'Finder', key: 'CmdOrCtrl+L' })
+ await call('computer.pasteText', { app: 'Finder', text: 'long text' })
+
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(1, 'typeText', {
+ app: 'Finder',
+ text: 'hello',
+ noScreenshot: true
+ })
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(2, 'pressKey', {
+ app: 'Finder',
+ key: 'Return'
+ })
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(3, 'hotkey', {
+ app: 'Finder',
+ key: 'CmdOrCtrl+L'
+ })
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(4, 'pasteText', {
+ app: 'Finder',
+ text: 'long text'
+ })
+ })
+
+ it('dispatches scroll and setValue actions through the sidecar', async () => {
+ computerMocks.callComputerSidecarAction.mockResolvedValue({ ok: true })
+
+ await call('computer.scroll', {
+ app: 'Finder',
+ elementIndex: 0,
+ direction: 'down',
+ pages: 2
+ })
+ await call('computer.setValue', {
+ app: 'Finder',
+ elementIndex: 1,
+ value: ''
+ })
+
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(1, 'scroll', {
+ app: 'Finder',
+ elementIndex: 0,
+ direction: 'down',
+ pages: 2
+ })
+ expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(2, 'setValue', {
+ app: 'Finder',
+ elementIndex: 1,
+ value: ''
+ })
+ })
+})
+
+function findMethod(name: string) {
+ const method = COMPUTER_METHODS.find((candidate) => candidate.name === name)
+ if (!method) {
+ throw new Error(`missing method ${name}`)
+ }
+ return method
+}
+
+async function call(name: string, params: Record]