Add native computer-use automation (#1683)

Co-authored-by: Orca <help@stably.ai>

.github/workflows/computer-e2e.yml

@@ -0,0 +1,104 @@
+name: Computer-use e2e
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/computer-e2e.yml'
+      - 'config/electron-builder.config.cjs'
+      - 'config/scripts/build-computer-macos.mjs'
+      - 'config/scripts/verify-computer-native.mjs'
+      - 'native/computer-use-macos/**'
+      - 'native/computer-use-linux/**'
+      - 'native/computer-use-windows/**'
+      - 'src/cli/**'
+      - 'src/main/computer/**'
+      - 'src/main/runtime/rpc/methods/computer.ts'
+      - 'src/shared/runtime-types.ts'
+      - 'tests/e2e/computer-linux.e2e.ts'
+      - 'tests/e2e/computer-mac.e2e.ts'
+      - 'tests/e2e/computer-windows.e2e.ts'
+      - 'tests/e2e/helpers/computer-driver.ts'
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * *'
+
+jobs:
+  native-smoke:
+    if: github.event_name == 'pull_request'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: package.json
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: false
+      - if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y python3 python3-gi gir1.2-atspi-2.0 at-spi2-core xclip xdotool
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm verify:computer-native
+      - run: pnpm build:cli
+
+  mac:
+    # macOS Accessibility and Screen Recording require user-granted TCC entries.
+    # Keep this on manual/scheduled permission-bearing runners instead of PR CI.
+    if: github.event_name != 'pull_request'
+    runs-on: macos-14
+    env:
+      ORCA_COMPUTER_E2E: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: package.json
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: false
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm build:computer-macos
+      - run: pnpm verify:computer-native
+      - run: pnpm build:cli
+      - run: pnpm test:e2e:computer -- --reporter=verbose tests/e2e/computer-mac.e2e.ts
+
+  linux:
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-22.04
+    env:
+      ORCA_COMPUTER_E2E: '1'
+      ACCESSIBILITY_ENABLED: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: package.json
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: false
+      - run: sudo apt-get update && sudo apt-get install -y build-essential python3 python3-gi gir1.2-atspi-2.0 gedit at-spi2-core xvfb xclip xdotool
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm verify:computer-native
+      - run: pnpm build:cli
+      - run: xvfb-run --auto-servernum dbus-run-session -- pnpm test:e2e:computer -- --reporter=verbose tests/e2e/computer-linux.e2e.ts
+
+  windows:
+    if: github.event_name != 'pull_request'
+    runs-on: windows-latest
+    env:
+      ORCA_COMPUTER_E2E: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: package.json
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: false
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm verify:computer-native
+      - run: pnpm build:cli
+      - run: pnpm test:e2e:computer -- --reporter=verbose tests/e2e/computer-windows.e2e.ts

.gitignore

@@ -24,6 +24,7 @@ dist-electron/
 out/
 /build/
 release/
+native/**/.build/
 
 # pnpm
 .pnpm-store/

config/electron-builder.config.cjs

@@ -1,5 +1,6 @@
 const { chmodSync, existsSync, readdirSync } = require('node:fs')
-const { join } = require('node:path')
+const { execFileSync } = require('node:child_process')
+const { join, resolve } = require('node:path')
 
 const isMacRelease = process.env.ORCA_MAC_RELEASE === '1'
 
@@ -35,6 +36,7 @@ module.exports = {
     'out/cli/**',
     'out/shared/**',
     'out/main/daemon-entry.js',
+    'out/main/computer-sidecar.js',
     'out/main/chunks/**',
     'resources/**',
     'node_modules/zod/**'
@@ -42,7 +44,12 @@ module.exports = {
   afterPack: async (context) => {
     const resourcesDir =
       context.electronPlatformName === 'darwin'
-        ? join(context.appOutDir, `${context.packager.appInfo.productFilename}.app`, 'Contents', 'Resources')
+        ? join(
+            context.appOutDir,
+            `${context.packager.appInfo.productFilename}.app`,
+            'Contents',
+            'Resources'
+          )
         : join(context.appOutDir, 'resources')
     if (!existsSync(resourcesDir)) {
       return
@@ -56,6 +63,9 @@ module.exports = {
       // the copied binary to be executable in packaged apps.
       chmodSync(join(resourcesDir, filename), 0o755)
     }
+    if (context.electronPlatformName === 'darwin') {
+      await signMacComputerUseHelper(join(resourcesDir, 'Orca Computer Use.app'), context.packager)
+    }
   },
   win: {
     executableName: 'Orca',
@@ -67,6 +77,10 @@ module.exports = {
       {
         from: 'node_modules/agent-browser/bin/agent-browser-win32-x64.exe',
         to: 'agent-browser-win32-x64.exe'
+      },
+      {
+        from: 'native/computer-use-windows/runtime.ps1',
+        to: 'computer-use-windows/runtime.ps1'
       }
     ]
   },
@@ -115,6 +129,10 @@ module.exports = {
       {
         from: 'node_modules/agent-browser/bin/agent-browser-darwin-${arch}',
         to: 'agent-browser-darwin-${arch}'
+      },
+      {
+        from: 'native/computer-use-macos/.build/release/Orca Computer Use.app',
+        to: 'Orca Computer Use.app'
       }
     ],
     target: [
@@ -134,6 +152,9 @@ module.exports = {
   dmg: {
     artifactName: 'orca-macos-${arch}.${ext}'
   },
+  deb: {
+    depends: ['python3', 'python3-gi', 'gir1.2-atspi-2.0', 'at-spi2-core', 'xdotool', 'xclip']
+  },
   linux: {
     extraResources: [
       {
@@ -143,6 +164,10 @@ module.exports = {
       {
         from: 'node_modules/agent-browser/bin/agent-browser-linux-${arch}',
         to: 'agent-browser-linux-${arch}'
+      },
+      {
+        from: 'native/computer-use-linux/runtime.py',
+        to: 'computer-use-linux/runtime.py'
       }
     ],
     target: ['AppImage', 'deb'],
@@ -165,3 +190,67 @@ module.exports = {
     releaseType: 'release'
   }
 }
+
+async function signMacComputerUseHelper(helperAppPath, packager) {
+  if (!existsSync(helperAppPath)) {
+    if (isMacRelease) {
+      throw new Error(`Missing Orca Computer Use helper app at ${helperAppPath}`)
+    }
+    return
+  }
+  const codeSigningInfo =
+    isMacRelease && process.env.CSC_LINK && packager?.codeSigningInfo?.value
+      ? await packager.codeSigningInfo.value
+      : null
+  const identity =
+    process.env.ORCA_COMPUTER_MACOS_SIGN_IDENTITY ??
+    process.env.CSC_NAME ??
+    findInstalledMacSigningIdentity(codeSigningInfo?.keychainFile) ??
+    (isMacRelease ? null : '-')
+  if (!identity) {
+    throw new Error('Missing signing identity for Orca Computer Use helper app')
+  }
+  // Why: TCC grants attach to this nested app's code identity. Sign it before
+  // the outer Orca.app is sealed so production builds preserve that identity.
+  execFileSync('codesign', codesignArgs(identity, helperAppPath), { stdio: 'inherit' })
+  execFileSync('codesign', ['--verify', '--deep', '--strict', helperAppPath], {
+    stdio: 'inherit'
+  })
+}
+
+function codesignArgs(identity, targetPath) {
+  const args = ['--force', '--deep', '--sign', identity]
+  if (isMacRelease) {
+    args.push(
+      '--options',
+      'runtime',
+      '--timestamp',
+      '--entitlements',
+      resolve(__dirname, '../resources/build/entitlements.computer-use.mac.plist')
+    )
+  }
+  args.push(targetPath)
+  return args
+}
+
+function findInstalledMacSigningIdentity(keychainFile) {
+  try {
+    const output = execFileSync(
+      'security',
+      ['find-identity', '-v', '-p', 'codesigning', ...(keychainFile ? [keychainFile] : [])],
+      {
+        encoding: 'utf8'
+      }
+    )
+    const releaseMatch =
+      output.match(/"([^"]*Developer ID Application:[^"]+)"/) ??
+      output.match(/"([^"]*Apple Distribution:[^"]+)"/)
+    if (releaseMatch?.[1]) {
+      return releaseMatch[1]
+    }
+    if (!isMacRelease) {
+      return output.match(/"([^"]*Apple Development:[^"]+)"/)?.[1] ?? null
+    }
+  } catch {}
+  return null
+}

config/scripts/build-computer-macos.mjs

@@ -0,0 +1,142 @@
+import { spawnSync } from 'node:child_process'
+import { chmodSync, copyFileSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
+const packagePath = path.join(repoRoot, 'native', 'computer-use-macos')
+const binaryPath = path.join(packagePath, '.build', 'release', 'orca-computer-use-macos')
+const appPath = path.join(packagePath, '.build', 'release', 'Orca Computer Use.app')
+const appExecutablePath = path.join(appPath, 'Contents', 'MacOS', 'orca-computer-use-macos')
+const appIconPath = path.join(appPath, 'Contents', 'Resources', 'AppIcon.icns')
+const entitlementsPath = path.join(
+  repoRoot,
+  'resources',
+  'build',
+  'entitlements.computer-use.mac.plist'
+)
+const bundleId = process.env.ORCA_COMPUTER_MACOS_BUNDLE_ID ?? 'com.stablyai.orca.computer-use'
+const displayName = 'Orca Computer Use'
+const signingIdentity = resolveSigningIdentity()
+const universalTriples = ['arm64-apple-macosx', 'x86_64-apple-macosx']
+
+if (process.platform !== 'darwin') {
+  process.exit(0)
+}
+
+buildUniversalBinary()
+chmodSync(binaryPath, 0o755)
+createHelperApp()
+
+function buildUniversalBinary() {
+  const builtBinaries = universalTriples.map((triple) => {
+    run('swift', ['build', '-c', 'release', '--package-path', packagePath, '--triple', triple])
+    return path.join(packagePath, '.build', triple, 'release', 'orca-computer-use-macos')
+  })
+  mkdirSync(path.dirname(binaryPath), { recursive: true })
+  run('lipo', ['-create', ...builtBinaries, '-output', binaryPath])
+}
+
+function createHelperApp() {
+  rmSync(appPath, { recursive: true, force: true })
+  mkdirSync(path.dirname(appExecutablePath), { recursive: true })
+  mkdirSync(path.join(appPath, 'Contents', 'Resources'), { recursive: true })
+  copyFileSync(binaryPath, appExecutablePath)
+  copyFileSync(path.join(repoRoot, 'resources', 'build', 'icon.icns'), appIconPath)
+  chmodSync(appExecutablePath, 0o755)
+  writeFileSync(path.join(appPath, 'Contents', 'Info.plist'), infoPlist(), 'utf8')
+  const signer = spawnSync('codesign', codesignArgs(signingIdentity, appPath), { stdio: 'inherit' })
+  if (signer.signal) {
+    process.kill(process.pid, signer.signal)
+  }
+  if (signer.status !== 0) {
+    process.exit(signer.status ?? 1)
+  }
+}
+
+function codesignArgs(identity, targetPath) {
+  const args = ['--force', '--deep', '--sign', identity]
+  if (process.env.ORCA_MAC_RELEASE === '1' && identity !== '-') {
+    args.push('--options', 'runtime', '--timestamp', '--entitlements', entitlementsPath)
+  }
+  args.push(targetPath)
+  return args
+}
+
+function resolveSigningIdentity() {
+  const explicitIdentity = process.env.ORCA_COMPUTER_MACOS_SIGN_IDENTITY ?? process.env.CSC_NAME
+  if (explicitIdentity) {
+    return explicitIdentity
+  }
+  const identities = spawnSync('security', ['find-identity', '-v', '-p', 'codesigning'], {
+    encoding: 'utf8'
+  })
+  if (identities.status !== 0 || !identities.stdout) {
+    return '-'
+  }
+  const developmentMatch = identities.stdout.match(/"([^"]*Apple Development:[^"]+)"/)
+  if (process.env.ORCA_MAC_RELEASE !== '1' && developmentMatch) {
+    return developmentMatch[1]
+  }
+  const releaseMatch =
+    identities.stdout.match(/"([^"]*Developer ID Application:[^"]+)"/) ??
+    identities.stdout.match(/"([^"]*Apple Distribution:[^"]+)"/)
+  return releaseMatch?.[1] ?? developmentMatch?.[1] ?? '-'
+}
+
+function run(command, args) {
+  const result = spawnSync(command, args, { stdio: 'inherit' })
+  if (result.signal) {
+    process.kill(process.pid, result.signal)
+  }
+  if (result.status !== 0) {
+    process.exit(result.status ?? 1)
+  }
+}
+
+function infoPlist() {
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>orca-computer-use-macos</string>
+  <key>CFBundleIdentifier</key>
+  <string>${escapePlist(bundleId)}</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleIconFile</key>
+  <string>AppIcon</string>
+  <key>CFBundleName</key>
+  <string>${escapePlist(displayName)}</string>
+  <key>CFBundleDisplayName</key>
+  <string>${escapePlist(displayName)}</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSMinimumSystemVersion</key>
+  <string>14.0</string>
+  <key>LSUIElement</key>
+  <true/>
+  <key>NSAccessibilityUsageDescription</key>
+  <string>Orca Computer Use needs Accessibility permission to read and interact with app interfaces when you ask Orca to use apps.</string>
+  <key>NSScreenCaptureUsageDescription</key>
+  <string>Orca Computer Use needs Screen Recording permission to capture app windows when you ask Orca to inspect your screen.</string>
+</dict>
+</plist>
+`
+}
+
+function escapePlist(value) {
+  return value
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+    .replaceAll("'", '&apos;')
+}

config/scripts/computer-use-smoke.mjs

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { resolve } from 'node:path'
+import { spawnSync } from 'node:child_process'
+
+const repoRoot = resolve(import.meta.dirname, '..', '..')
+const cliPath = resolve(repoRoot, 'out', 'cli', 'index.js')
+const args = new Set(process.argv.slice(2))
+const requestedApps = valueFlag('--apps')
+const includeScreenshot = args.has('--screenshot')
+const session = valueFlag('--session') ?? `computer-smoke-${process.pid}`
+const preferredApps = (requestedApps ?? process.env.ORCA_COMPUTER_SMOKE_APPS ?? 'Finder,TextEdit,Spotify,Slack,Microsoft Edge')
+  .split(',')
+  .map((app) => app.trim())
+  .filter(Boolean)
+
+if (!existsSync(cliPath)) {
+  fail(`Missing built CLI at ${cliPath}. Run pnpm build:cli first.`)
+}
+
+const list = unwrapResult(runCli(['computer', 'list-apps', '--json']))
+const apps = Array.isArray(list.apps) ? list.apps : []
+const availableNames = new Set(apps.map((app) => String(app.name ?? '').toLowerCase()))
+const availableBundles = new Set(apps.map((app) => String(app.bundleId ?? '').toLowerCase()).filter(Boolean))
+const targets = preferredApps.filter((app) => availableNames.has(app.toLowerCase()) || availableBundles.has(app.toLowerCase()))
+
+console.log(`computer-use smoke: ${apps.length} apps listed`)
+if (targets.length === 0) {
+  console.log(`computer-use smoke: no preferred apps are running (${preferredApps.join(', ')})`)
+  process.exit(0)
+}
+
+let failures = 0
+for (const app of targets) {
+  const result = runCli([
+    'computer',
+    'get-app-state',
+    '--session',
+    session,
+    '--app',
+    app,
+    ...(includeScreenshot ? [] : ['--no-screenshot']),
+    '--json'
+  ], { allowFailure: true })
+
+  if (!result.ok) {
+    failures += 1
+    console.log(`computer-use smoke: ${app}: failed: ${result.error}`)
+    continue
+  }
+
+  const state = unwrapResult(result.value)
+  const snapshot = state.snapshot
+  const treeText = String(snapshot.treeText ?? '')
+  const lineCount = treeText.split('\n').filter(Boolean).length
+  const secondaryActions = (treeText.match(/Secondary Actions:/g) ?? []).length
+  const settable = (treeText.match(/\bsettable\b/g) ?? []).length
+  const screenshotState = state.screenshot ? `${state.screenshot.width}x${state.screenshot.height}` : 'missing'
+  console.log([
+    `computer-use smoke: ${snapshot.app.name}`,
+    `${snapshot.elementCount} elements`,
+    `${lineCount} lines`,
+    `${secondaryActions} secondary-action lines`,
+    `${settable} settable elements`,
+    `screenshot=${screenshotState}`
+  ].join(' | '))
+}
+
+if (failures > 0) {
+  fail(`${failures} app snapshot smoke check(s) failed`)
+}
+
+function valueFlag(name) {
+  const index = process.argv.indexOf(name)
+  if (index === -1) {
+    return null
+  }
+  return process.argv[index + 1] ?? null
+}
+
+function runCli(cliArgs, options = {}) {
+  const child = spawnSync(process.execPath, [cliPath, ...cliArgs], {
+    cwd: repoRoot,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      ORCA_USER_DATA_PATH: process.env.ORCA_COMPUTER_SMOKE_USER_DATA_PATH ?? defaultDevUserDataPath()
+    }
+  })
+  if (child.status !== 0) {
+    const error = (child.stderr || child.stdout || `exit ${child.status}`).trim()
+    if (options.allowFailure) {
+      return { ok: false, error }
+    }
+    fail(error)
+  }
+  try {
+    return options.allowFailure
+      ? { ok: true, value: JSON.parse(child.stdout) }
+      : JSON.parse(child.stdout)
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error)
+    fail(`Could not parse CLI JSON for ${cliArgs.join(' ')}: ${detail}\n${child.stdout}`)
+  }
+}
+
+function defaultDevUserDataPath() {
+  if (process.platform === 'darwin') {
+    return resolve(homedir(), 'Library', 'Application Support', 'orca-dev')
+  }
+  if (process.platform === 'win32') {
+    return resolve(process.env.APPDATA ?? resolve(homedir(), 'AppData', 'Roaming'), 'orca-dev')
+  }
+  return resolve(process.env.XDG_CONFIG_HOME ?? resolve(homedir(), '.config'), 'orca-dev')
+}
+
+function unwrapResult(value) {
+  if (value && typeof value === 'object' && 'result' in value) {
+    return value.result
+  }
+  return value
+}
+
+function fail(message) {
+  console.error(`computer-use smoke: ${message}`)
+  process.exit(1)
+}

config/scripts/verify-computer-native.mjs

@@ -0,0 +1,180 @@
+#!/usr/bin/env node
+
+import { spawnSync } from 'node:child_process'
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+const repoRoot = resolve(import.meta.dirname, '..', '..')
+const checks = [
+  {
+    name: 'macOS Swift renderer/provider tests',
+    command: 'swift',
+    args: ['test', '--package-path', 'native/computer-use-macos'],
+    enabled: process.platform === 'darwin'
+  },
+  {
+    name: 'Linux provider Python syntax',
+    command: 'python3',
+    args: ['-m', 'py_compile', 'native/computer-use-linux/runtime.py'],
+    enabled: true
+  },
+  {
+    name: 'Linux provider imports',
+    command: 'python3',
+    args: [
+      '-c',
+      [
+        'import importlib.util',
+        'spec=importlib.util.spec_from_file_location("orca_linux","native/computer-use-linux/runtime.py")',
+        'module=importlib.util.module_from_spec(spec)',
+        'spec.loader.exec_module(module)',
+        'print("import-ok")'
+      ].join(';')
+    ],
+    enabled: process.platform === 'linux'
+  },
+  {
+    name: 'Windows provider PowerShell parse',
+    command: process.platform === 'win32' ? 'powershell.exe' : 'pwsh',
+    args: [
+      '-NoLogo',
+      '-NoProfile',
+      '-Command',
+      [
+        '$errors=$null',
+        '$tokens=$null',
+        '[System.Management.Automation.Language.Parser]::ParseFile("native/computer-use-windows/runtime.ps1",[ref]$tokens,[ref]$errors) > $null',
+        'if ($errors.Count) { $errors | Format-List *; exit 1 }',
+        '"parse-ok"'
+      ].join('; ')
+    ],
+    enabled: true
+  },
+  {
+    name: 'Windows provider handshake',
+    run: verifyWindowsProviderHandshake,
+    enabled: process.platform === 'win32'
+  },
+  {
+    name: 'macOS helper app bundle and signature',
+    run: verifyMacOSHelperApp,
+    enabled: process.platform === 'darwin'
+  }
+]
+
+let failed = false
+for (const check of checks) {
+  if (!check.enabled) {
+    console.log(`[computer-native] skip ${check.name}`)
+    continue
+  }
+  if (check.run) {
+    console.log(`[computer-native] ${check.name}`)
+    if (!check.run()) {
+      failed = true
+    }
+    continue
+  }
+  if (!hasCommand(check.command)) {
+    console.log(`[computer-native] skip ${check.name}: ${check.command} not found`)
+    continue
+  }
+  console.log(`[computer-native] ${check.name}`)
+  const result = spawnSync(check.command, check.args, {
+    cwd: repoRoot,
+    stdio: 'inherit'
+  })
+  if (result.status !== 0 || result.error) {
+    failed = true
+  }
+}
+
+if (failed) {
+  process.exit(1)
+}
+
+function hasCommand(command) {
+  if (process.platform === 'win32') {
+    const result = spawnSync('where.exe', [command], { stdio: 'ignore' })
+    return result.status === 0
+  }
+  if (existsSync(command)) {
+    return true
+  }
+  const result = spawnSync('/bin/sh', ['-lc', `command -v ${quoteShell(command)}`], {
+    stdio: 'ignore'
+  })
+  return result.status === 0
+}
+
+function verifyMacOSHelperApp() {
+  const appPath = join(
+    repoRoot,
+    'native',
+    'computer-use-macos',
+    '.build',
+    'release',
+    'Orca Computer Use.app'
+  )
+  if (!existsSync(appPath)) {
+    console.error(
+      `[computer-native] missing helper app at ${appPath}; run pnpm build:computer-macos`
+    )
+    return false
+  }
+  return run('codesign', ['--verify', '--deep', '--strict', appPath])
+}
+
+function verifyWindowsProviderHandshake() {
+  const dir = mkdtempSync(join(tmpdir(), 'orca-computer-use-verify-'))
+  const operationPath = join(dir, 'operation.json')
+  try {
+    writeFileSync(operationPath, JSON.stringify({ tool: 'handshake' }), { mode: 0o600 })
+    const result = spawnSync(
+      'powershell.exe',
+      [
+        '-NoLogo',
+        '-NoProfile',
+        '-ExecutionPolicy',
+        'Bypass',
+        '-File',
+        'native/computer-use-windows/runtime.ps1',
+        operationPath
+      ],
+      { cwd: repoRoot, encoding: 'utf8' }
+    )
+    if (result.status !== 0 || result.error) {
+      process.stderr.write(result.stderr ?? '')
+      return false
+    }
+    const response = JSON.parse(result.stdout.trim())
+    if (response.ok === true && response.capabilities?.protocolVersion === 1) {
+      console.log('[computer-native] windows-handshake-ok')
+      return true
+    }
+    console.error(`[computer-native] invalid Windows handshake response: ${result.stdout}`)
+    return false
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error))
+    return false
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+function run(command, args) {
+  if (!hasCommand(command)) {
+    console.log(`[computer-native] skip ${command}: command not found`)
+    return true
+  }
+  const result = spawnSync(command, args, {
+    cwd: repoRoot,
+    stdio: 'inherit'
+  })
+  return result.status === 0 && !result.error
+}
+
+function quoteShell(value) {
+  return `'${value.replaceAll("'", "'\\''")}'`
+}

electron.vite.config.ts

@@ -41,7 +41,8 @@ export default defineConfig({
       rollupOptions: {
         input: {
           index: resolve('src/main/index.ts'),
-          'daemon-entry': resolve('src/main/daemon/daemon-entry.ts')
+          'daemon-entry': resolve('src/main/daemon/daemon-entry.ts'),
+          'computer-sidecar': resolve('src/main/computer/sidecar-entry.ts')
         }
       }
     },

native/computer-use-linux/runtime.py

@@ -0,0 +1,819 @@
+#!/usr/bin/env python3
+"""Orca Linux computer-use bridge.
+
+The Node sidecar owns Orca's public API. This process is intentionally a small
+AT-SPI adapter: read one JSON operation file, execute it in the user's desktop
+session, and print one JSON response.
+"""
+
+import base64
+import json
+import math
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+
+import gi
+
+gi.require_version("Atspi", "2.0")
+try:
+    gi.require_version("Gdk", "3.0")
+    from gi.repository import Gdk
+except (ImportError, ValueError):
+    Gdk = None
+from gi.repository import Atspi
+
+MAX_NODES = 1200
+MAX_DEPTH = 64
+TEXT_LIMIT = 500
+BLOCKED_APP_FRAGMENTS = (
+    "1password",
+    "bitwarden",
+    "dashlane",
+    "lastpass",
+    "nordpass",
+    "proton pass",
+)
+
+
+@dataclass
+class Rect:
+    x: float
+    y: float
+    width: float
+    height: float
+
+    def to_json(self):
+        return {"x": self.x, "y": self.y, "width": self.width, "height": self.height}
+
+
+def attempt(fn, fallback=None):
+    try:
+        value = fn()
+        return fallback if value is None else value
+    except Exception:
+        return fallback
+
+
+def ensure_desktop_bus():
+    missing = [name for name in ("XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS") if not os.environ.get(name)]
+    if missing:
+        raise RuntimeError("Linux computer use requires an active desktop session; missing " + ", ".join(missing))
+
+
+def desktop_root():
+    return Atspi.get_desktop(0)
+
+
+def children(node):
+    count = int(attempt(node.get_child_count, 0) or 0)
+    for index in range(count):
+        child = attempt(lambda i=index: node.get_child_at_index(i))
+        if child is not None:
+            yield index, child
+
+
+def text_attr(node, getter):
+    return str(attempt(getter, "") or "")
+
+
+def name_of(node):
+    return text_attr(node, node.get_name)
+
+
+def role_of(node):
+    return text_attr(node, node.get_role_name)
+
+
+def pid_of(node):
+    return int(attempt(node.get_process_id, 0) or 0)
+
+
+def has_state(node, state):
+    state_set = attempt(node.get_state_set)
+    return bool(state_set is not None and attempt(lambda: state_set.contains(state), False))
+
+
+def screen_rect(node):
+    component = attempt(node.get_component_iface)
+    if component is None:
+        return None
+    rect = attempt(lambda: Atspi.Component.get_extents(component, Atspi.CoordType.SCREEN))
+    if rect is None or rect.width <= 0 or rect.height <= 0:
+        return None
+    return Rect(float(rect.x), float(rect.y), float(rect.width), float(rect.height))
+
+
+def relative_rect(node, window_rect):
+    rect = screen_rect(node)
+    if rect is None or window_rect is None:
+        return rect
+    return Rect(rect.x - window_rect.x, rect.y - window_rect.y, rect.width, rect.height)
+
+
+def desktop_apps():
+    for _, app in children(desktop_root()):
+        if name_of(app):
+            yield app
+
+
+def windows_for(app):
+    result = []
+    for index, child in children(app):
+        role = role_of(child).lower()
+        rect = screen_rect(child)
+        if rect is not None or role in {"frame", "window", "dialog", "alert"}:
+            result.append((index, child))
+    return result
+
+
+def choose_window(app, window_id=None, window_index=None):
+    windows = windows_for(app)
+    if not windows:
+        raise RuntimeError("No top-level AT-SPI window is available for " + name_of(app))
+    target = window_id if window_id is not None else window_index
+    if target is not None:
+        for item in windows:
+            if item[0] == int(target):
+                return item
+        raise RuntimeError(f'windowNotFound("{target}")')
+    for item in windows:
+        if has_state(item[1], Atspi.StateType.ACTIVE):
+            return item
+    for item in windows:
+        if has_state(item[1], Atspi.StateType.SHOWING):
+            return item
+    return windows[0]
+
+
+def restore_window(app):
+    pid = pid_of(app)
+    if not pid or not shutil.which("xdotool"):
+        return
+    subprocess.run(
+        ["xdotool", "search", "--pid", str(pid), "windowactivate", "--sync"],
+        check=False,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+
+
+def require_keyboard_focus(window, operation):
+    if operation.get("restoreWindow") or has_state(window, Atspi.StateType.ACTIVE):
+        return
+    raise RuntimeError("window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window")
+
+
+def app_matches(app, query):
+    needle = str(query or "").strip().lower()
+    if not needle:
+        return False
+    if needle.startswith("pid:"):
+        return str(pid_of(app)) == needle[4:]
+    if needle.isdigit() and pid_of(app) == int(needle):
+        return True
+    haystacks = [name_of(app).lower()] + [name_of(window).lower() for _, window in windows_for(app)]
+    return any(value == needle or needle in value for value in haystacks)
+
+
+def find_app(query):
+    for app in desktop_apps():
+        if app_matches(app, query):
+            reject_blocked_app(app)
+            return app
+    raise RuntimeError(f'appNotFound("{query}")')
+
+
+def reject_blocked_app(app):
+    haystacks = [name_of(app).lower()] + [name_of(window).lower() for _, window in windows_for(app)]
+    if any(fragment in value for fragment in BLOCKED_APP_FRAGMENTS for value in haystacks):
+        raise RuntimeError(f'appBlocked("{name_of(app)}")')
+
+
+def action_labels(node):
+    labels = []
+    count = int(attempt(node.get_n_actions, 0) or 0)
+    for index in range(count):
+        label = str(attempt(lambda i=index: node.get_action_name(i), "") or "")
+        description = str(attempt(lambda i=index: node.get_action_description(i), "") or "")
+        value = label or description
+        if value and value not in labels:
+            labels.append(value)
+    return labels
+
+
+def meaningful_actions(actions):
+    noisy = {
+        "click",
+        "press",
+        "show default ui",
+        "show alternate ui",
+        "show menu",
+        "scroll to visible",
+        "raise",
+    }
+    return [action for action in actions if action.strip().lower() not in noisy]
+
+
+def display_action(action):
+    value = str(action or "").strip()
+    if not value:
+        return value
+    return " ".join(value.replace("_", " ").split())
+
+
+def sanitize_text(value):
+    return " ".join(str(value or "").replace("\r", " ").replace("\n", " ").split())
+
+
+def formatted_value(role_key, title, value):
+    clean = sanitize_text(value)
+    if not clean or clean == title:
+        return ""
+    if role_key in {"label", "static", "static text", "text", "entry", "text entry"}:
+        return " " + clean
+    return ", Value: " + clean
+
+
+def suppress_children(role_key, title, value, summary):
+    has_compact_label = bool(title or sanitize_text(value) or sanitize_text(summary))
+    return has_compact_label and role_key in {
+        "button",
+        "check box",
+        "checkbox",
+        "combo box",
+        "heading",
+        "link",
+        "menu item",
+        "page tab",
+        "push button",
+        "radio button",
+    }
+
+
+def string_value(node):
+    if is_secure_node(node):
+        return "[redacted]"
+    if bool(attempt(node.is_text, False)):
+        iface = attempt(node.get_text_iface)
+        count = int(attempt(lambda: Atspi.Text.get_character_count(iface), 0) or 0)
+        if iface is not None and count > 0:
+            value = str(attempt(lambda: Atspi.Text.get_text(iface, 0, min(count, TEXT_LIMIT)), "") or "")
+            return value + ("..." if count > TEXT_LIMIT else "")
+    value_iface = attempt(node.get_value_iface)
+    if value_iface is not None:
+        current = attempt(lambda: Atspi.Value.get_current_value(value_iface))
+        if current is not None:
+            return str(current)
+    return ""
+
+
+def is_secure_node(node):
+    role = role_of(node).lower()
+    label = " ".join([role, name_of(node).lower(), accessible_id(node).lower()])
+    if any(term in label for term in ("password", "passcode", "pin", "secret", "one-time code")):
+        return True
+    state_set = attempt(node.get_state_set)
+    protected_state = getattr(Atspi.StateType, "PROTECTED", None)
+    if state_set is not None and protected_state is not None and attempt(lambda: state_set.contains(protected_state), False):
+        return True
+    return False
+
+
+def accessible_id(node):
+    return str(attempt(node.get_accessible_id, "") or "")
+
+
+def record(node, index, path, window_rect):
+    role = role_of(node)
+    rect = relative_rect(node, window_rect)
+    return {
+        "index": index,
+        "runtimeId": path,
+        "automationId": accessible_id(node),
+        "name": name_of(node),
+        "controlType": role,
+        "localizedControlType": role,
+        "className": str(attempt(node.get_toolkit_name, "") or ""),
+        "value": string_value(node),
+        "nativeWindowHandle": 0,
+        "frame": rect.to_json() if rect else None,
+        "actions": action_labels(node),
+    }
+
+
+def render_accessibility_tree(root, window_rect, root_path):
+    records = []
+    lines = []
+    truncation = {"truncated": False, "maxNodes": MAX_NODES, "maxDepth": MAX_DEPTH, "maxDepthReached": False}
+
+    def text_snippets(node, limit=6, max_depth=3):
+        values = []
+        seen = set()
+
+        def collect(candidate, depth):
+            if len(values) >= limit or depth > max_depth:
+                return
+            role = role_of(candidate).lower()
+            if role in {"label", "static", "static text", "text", "link"}:
+                for raw in (name_of(candidate), string_value(candidate)):
+                    value = " ".join(str(raw or "").split())
+                    if value and value not in seen:
+                        seen.add(value)
+                        values.append(value[:80])
+                        if len(values) >= limit:
+                            return
+            for _, child in children(candidate):
+                collect(child, depth + 1)
+                if len(values) >= limit:
+                    return
+
+        collect(node, 0)
+        return values
+
+    def is_plain_text_subtree(node, max_depth=4):
+        saw_text = False
+        allowed = {"panel", "filler", "unknown", "section", "label", "static", "static text", "text", "link", "image"}
+
+        def visit(candidate, depth):
+            nonlocal saw_text
+            if depth > max_depth:
+                return False
+            role = role_of(candidate).lower()
+            if role not in allowed:
+                return False
+            if role in {"label", "static", "static text", "text", "link"}:
+                saw_text = True
+            if meaningful_actions(action_labels(candidate)):
+                return False
+            return all(visit(child, depth + 1) for _, child in children(candidate))
+
+        return visit(node, 0) and saw_text
+
+    def should_elide(item, child_count, summary):
+        role = (item["controlType"] or "").lower()
+        has_text = bool(item["name"] or item["automationId"] or item["value"])
+        return role in {"panel", "filler", "unknown", "section"} and not has_text and not meaningful_actions(item["actions"]) and summary is None and child_count <= 1
+
+    def walk(node, depth, path):
+        if len(records) >= MAX_NODES or depth > MAX_DEPTH:
+            truncation["truncated"] = True
+            if depth > MAX_DEPTH:
+                truncation["maxDepthReached"] = True
+            return
+        item = record(node, len(records), path, window_rect)
+        child_items = list(children(node))
+        role_key = (item["controlType"] or "").lower()
+        summary_values = text_snippets(node, limit=8, max_depth=4)
+        generic_summary = " ".join(summary_values) if role_key in {"panel", "filler", "unknown", "section"} and not item["name"] and not item["value"] and len(summary_values) >= 2 and is_plain_text_subtree(node) else None
+        if should_elide(item, len(child_items), generic_summary):
+            for child_index, child in child_items:
+                walk(child, depth, path + [child_index])
+            return
+        records.append(item)
+        title = item["name"] or item["automationId"] or ""
+        role_label = item["localizedControlType"] or item["controlType"]
+        line = f'{item["index"]} {role_label} {sanitize_text(title)}'.rstrip()
+        line += formatted_value(role_key, title, item["value"])
+        if generic_summary and generic_summary != title:
+            line += ", Text: " + sanitize_text(generic_summary)
+        elif role_key in {"row", "table row", "list item"}:
+            row_summary = " ".join(text_snippets(node))
+            if row_summary and row_summary != title:
+                line += ", Text: " + sanitize_text(row_summary)
+        filtered_actions = meaningful_actions(item["actions"])
+        if filtered_actions:
+            line += ", Secondary Actions: " + ", ".join(display_action(action) for action in filtered_actions)
+        lines.append(("\t" * depth) + line)
+        if generic_summary or suppress_children(role_key, title, item["value"], generic_summary):
+            return
+        for child_index, child in child_items:
+            walk(child, depth + 1, path + [child_index])
+
+    walk(root, 0, root_path)
+    return records, lines, truncation
+
+
+def capture_png(rect):
+    if Gdk is None or rect is None or os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland":
+        return None
+    screen = Gdk.Screen.get_default()
+    root = screen.get_root_window() if screen else None
+    if root is None:
+        return None
+    pixbuf = Gdk.pixbuf_get_from_window(root, round(rect.x), round(rect.y), max(1, round(rect.width)), max(1, round(rect.height)))
+    if pixbuf is None:
+        return None
+    ok, data = pixbuf.save_to_bufferv("png", [], [])
+    return base64.b64encode(bytes(data)).decode("ascii") if ok else None
+
+
+def first_descendant(root, predicate):
+    if predicate(root):
+        return root
+    for _, child in children(root):
+        found = first_descendant(child, predicate)
+        if found is not None:
+            return found
+    return None
+
+
+def focused_summary(window):
+    node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED))
+    if node is None:
+        return None
+    return (role_of(node) + " " + name_of(node)).strip()
+
+
+def selected_text(window):
+    node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED) and bool(attempt(candidate.is_text, False)))
+    iface = attempt(node.get_text_iface) if node is not None else None
+    selections = attempt(lambda: Atspi.Text.get_text_selections(iface), [])
+    if not selections:
+        return None
+    selection = selections[0]
+    return Atspi.Text.get_text(iface, selection.start_offset, selection.end_offset)
+
+
+def app_json(app):
+    return {"name": name_of(app), "bundleIdentifier": name_of(app), "pid": pid_of(app)}
+
+
+def window_json(app, index, window):
+    bounds = screen_rect(window)
+    showing = has_state(window, Atspi.StateType.SHOWING)
+    return {
+        "index": index,
+        "app": app_json(app),
+        "id": index,
+        "title": name_of(window),
+        "x": round(bounds.x) if bounds else None,
+        "y": round(bounds.y) if bounds else None,
+        "width": max(0, round(bounds.width if bounds else 0)),
+        "height": max(0, round(bounds.height if bounds else 0)),
+        "isMinimized": not showing,
+        "isOffscreen": not showing,
+        "screenIndex": None,
+        "platform": {"backend": "at-spi", "runtimeId": [index], "role": role_of(window)},
+    }
+
+
+def make_snapshot(query, include_screenshot, window_id=None, window_index=None, restore=False):
+    app = find_app(query)
+    if restore:
+        restore_window(app)
+    window_index, window = choose_window(app, window_id, window_index)
+    bounds = screen_rect(window)
+    records, lines, truncation = render_accessibility_tree(window, bounds, [window_index])
+    return {
+        "snapshotId": str(uuid.uuid4()),
+        "app": app_json(app),
+        "windowTitle": name_of(window),
+        "windowId": window_index,
+        "windowBounds": bounds.to_json() if bounds else None,
+        "screenshotPngBase64": capture_png(bounds) if include_screenshot else None,
+        "coordinateSpace": "window",
+        "truncation": truncation,
+        "treeLines": lines,
+        "focusedSummary": focused_summary(window),
+        "selectedText": selected_text(window),
+        "elements": records,
+    }
+
+
+def list_apps_response():
+    apps = []
+    for app in sorted(desktop_apps(), key=lambda value: (name_of(value).lower(), pid_of(value))):
+        if windows_for(app):
+            apps.append({"name": name_of(app), "bundleIdentifier": name_of(app), "pid": pid_of(app)})
+    return apps
+
+
+def list_windows_response(query):
+    app = find_app(query)
+    return {"app": app_json(app), "windows": [window_json(app, index, window) for index, window in windows_for(app)]}
+
+
+def handshake_response():
+    is_wayland = os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland"
+    has_hotkey = shutil.which("xdotool") is not None and not is_wayland
+    has_clipboard = any(shutil.which(command) for command in ("wl-copy", "xclip", "xsel"))
+    has_screenshot = Gdk is not None and not is_wayland
+    return {
+        "platform": "linux",
+        "provider": "orca-computer-use-linux",
+        "providerVersion": "1.0.0",
+        "protocolVersion": 1,
+        "supports": {
+            "apps": {"list": True, "bundleIds": False, "pids": True},
+            "windows": {"list": True, "targetById": True, "targetByIndex": True, "focus": False, "moveResize": False},
+            "observation": {"screenshot": has_screenshot, "annotatedScreenshot": False, "elementFrames": True, "ocr": False},
+            "actions": {
+                "click": True,
+                "typeText": True,
+                "pressKey": True,
+                "hotkey": has_hotkey,
+                "pasteText": has_clipboard,
+                "scroll": True,
+                "drag": True,
+                "setValue": True,
+                "performAction": True,
+            },
+            "surfaces": {"menus": False, "dialogs": False, "dock": False, "menubar": False},
+        },
+    }
+
+
+def find_by_path(app, path):
+    node = app
+    for index in path or []:
+        node = dict(children(node)).get(int(index))
+        if node is None:
+            return None
+    return node
+
+
+def all_nodes(root):
+    result = []
+
+    def walk(node):
+        if len(result) >= MAX_NODES:
+            return
+        result.append(node)
+        for _, child in children(node):
+            walk(child)
+
+    walk(root)
+    return result
+
+
+def find_element(app, saved):
+    if not saved:
+        return None
+    by_path = find_by_path(app, saved.get("runtimeId"))
+    if by_path is not None and same_element_signature(by_path, saved):
+        return by_path
+    return None
+
+
+def same_element_signature(node, saved):
+    if role_of(node) != str(saved.get("controlType") or ""):
+        return False
+    if name_of(node) != str(saved.get("name") or ""):
+        return False
+    if accessible_id(node) != str(saved.get("automationId") or ""):
+        return False
+    saved_actions = [str(action) for action in saved.get("actions") or []]
+    return action_labels(node) == saved_actions
+
+
+def preferred_action(node):
+    if node is None:
+        return None
+    priority = {"click", "press", "activate", "invoke", "select", "toggle", "open"}
+    fallback = None
+    for index in range(int(attempt(node.get_n_actions, 0) or 0)):
+        label = str(attempt(lambda i=index: node.get_action_name(i), "") or "").lower()
+        if label in priority:
+            return index
+        if fallback is None and any(term in label for term in ("click", "press", "activate")):
+            fallback = index
+    return fallback
+
+
+def perform_action(node, index):
+    return bool(index is not None and attempt(lambda: node.do_action(int(index)), False))
+
+
+def screen_point(window_rect, saved_element=None, x=None, y=None, node=None):
+    rect = screen_rect(node) if node is not None else None
+    if rect is not None:
+        return rect.x + rect.width / 2, rect.y + rect.height / 2
+    if saved_element is not None:
+        raise RuntimeError("stale element frame; run get-app-state again and use a fresh element index")
+    if window_rect is None or x is None or y is None:
+        raise RuntimeError("coordinate action requires a visible window and coordinates")
+    return window_rect.x + float(x), window_rect.y + float(y)
+
+
+def click_at(x, y, button, count):
+    button = (button or "left").lower()
+    down, up = {"right": ("b3p", "b3r"), "middle": ("b2p", "b2r")}.get(button, ("b1p", "b1r"))
+    for _ in range(max(1, int(count or 1))):
+        Atspi.generate_mouse_event(round(x), round(y), "abs")
+        Atspi.generate_mouse_event(round(x), round(y), down)
+        time.sleep(0.03)
+        Atspi.generate_mouse_event(round(x), round(y), up)
+
+
+def scroll_at(x, y, direction, pages):
+    down, up = {
+        "up": ("b4p", "b4r"),
+        "down": ("b5p", "b5r"),
+        "left": ("b6p", "b6r"),
+        "right": ("b7p", "b7r"),
+    }.get(str(direction or "down").lower(), ("b5p", "b5r"))
+    for _ in range(max(1, math.ceil(float(pages or 1)))):
+        Atspi.generate_mouse_event(round(x), round(y), "abs")
+        Atspi.generate_mouse_event(round(x), round(y), down)
+        time.sleep(0.03)
+        Atspi.generate_mouse_event(round(x), round(y), up)
+
+
+def drag_between(start, end):
+    Atspi.generate_mouse_event(round(start[0]), round(start[1]), "abs")
+    Atspi.generate_mouse_event(round(start[0]), round(start[1]), "b1p")
+    for step in range(1, 13):
+        x = start[0] + (end[0] - start[0]) * step / 12
+        y = start[1] + (end[1] - start[1]) * step / 12
+        Atspi.generate_mouse_event(round(x), round(y), "abs")
+        time.sleep(0.02)
+    Atspi.generate_mouse_event(round(end[0]), round(end[1]), "b1r")
+
+
+def key_name(raw):
+    aliases = {
+        "return": "Return", "enter": "Return", "tab": "Tab", "escape": "Escape", "esc": "Escape",
+        "backspace": "BackSpace", "delete": "Delete", "space": "space", "left": "Left", "right": "Right",
+        "up": "Up", "down": "Down", "page_up": "Page_Up", "page_down": "Page_Down",
+    }
+    return aliases.get(str(raw).lower(), str(raw))
+
+
+def press_key(raw):
+    name = key_name(raw)
+    if len(name) == 1:
+        Atspi.generate_keyboard_event(0, name, Atspi.KeySynthType.STRING)
+        return
+    if Gdk is None:
+        raise RuntimeError("GDK is required for non-character key synthesis")
+    Atspi.generate_keyboard_event(Gdk.keyval_from_name(name), None, Atspi.KeySynthType.PRESSRELEASE)
+
+
+def hotkey(raw):
+    key_spec = re.sub(r"(?i)commandorcontrol|cmdorctrl", "ctrl", str(raw))
+    xdotool = shutil.which("xdotool")
+    if xdotool:
+        subprocess.run([xdotool, "key", key_spec], check=True)
+        return
+    if "+" in key_spec:
+        raise RuntimeError("hotkey combinations require xdotool")
+    press_key(key_spec)
+
+
+def type_text(value):
+    Atspi.generate_keyboard_event(0, str(value), Atspi.KeySynthType.STRING)
+
+
+def paste_text(value):
+    text = str(value)
+    previous = read_clipboard()
+    try:
+        write_clipboard(text)
+        hotkey("ctrl+v")
+    finally:
+        if previous is not None:
+            write_clipboard(previous)
+
+
+def read_clipboard():
+    for command in (["wl-paste"], ["xclip", "-selection", "clipboard", "-o"], ["xsel", "--clipboard", "--output"]):
+        if shutil.which(command[0]):
+            result = subprocess.run(command, check=False, capture_output=True, text=True)
+            if result.returncode == 0:
+                return result.stdout
+    return None
+
+
+def write_clipboard(value):
+    for command in (["wl-copy"], ["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"]):
+        if shutil.which(command[0]):
+            subprocess.run(command, input=value, check=True, text=True)
+            return
+    raise RuntimeError("paste_text requires wl-copy, xclip, or xsel")
+
+
+def set_value(node, value):
+    if node is not None and bool(attempt(node.is_editable_text, False)):
+        editable = attempt(node.get_editable_text_iface)
+        if editable is not None and attempt(lambda: Atspi.EditableText.set_text_contents(editable, str(value)), False):
+            return True
+    value_iface = attempt(node.get_value_iface) if node is not None else None
+    if value_iface is not None:
+        return bool(attempt(lambda: Atspi.Value.set_current_value(value_iface, float(value)), False))
+    return False
+
+
+def run_operation(operation):
+    tool = operation.get("tool")
+    include_screenshot = not bool(operation.get("noScreenshot"))
+    if tool == "handshake":
+        return {"ok": True, "capabilities": handshake_response()}
+    if tool == "list_apps":
+        return {"ok": True, "apps": list_apps_response()}
+    if tool == "list_windows":
+        return {"ok": True, **list_windows_response(operation.get("app", ""))}
+    if tool == "get_app_state":
+        return {
+            "ok": True,
+            "snapshot": make_snapshot(
+                operation.get("app", ""),
+                include_screenshot,
+                operation.get("windowId"),
+                operation.get("windowIndex"),
+                bool(operation.get("restoreWindow")),
+            ),
+        }
+
+    app = find_app(operation.get("app", ""))
+    if operation.get("restoreWindow"):
+        restore_window(app)
+    _, window = choose_window(app, operation.get("windowId"), operation.get("windowIndex"))
+    if tool in {"type_text", "press_key", "hotkey", "paste_text"}:
+        require_keyboard_focus(window, operation)
+    bounds = screen_rect(window)
+    saved = operation.get("element")
+    node = find_element(app, saved)
+    from_node = find_element(app, operation.get("fromElement"))
+    to_node = find_element(app, operation.get("toElement"))
+    action = None
+
+    if tool == "click":
+        preferred = preferred_action(node)
+        click_count = int(operation.get("click_count", 1) or 1)
+        handled = operation.get("mouse_button", "left") == "left" and click_count <= 1 and perform_action(node, preferred)
+        if not handled:
+            click_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node), operation.get("mouse_button", "left"), operation.get("click_count", 1))
+            action = {"path": "synthetic", "actionName": None, "fallbackReason": "actionUnsupported"}
+        else:
+            labels = action_labels(node)
+            action = {"path": "accessibility", "actionName": labels[preferred] if preferred is not None and preferred < len(labels) else "action", "fallbackReason": None}
+    elif tool == "perform_secondary_action":
+        wanted = str(operation.get("action", "")).lower()
+        for index, label in enumerate(action_labels(node)):
+            if label.lower() == wanted and perform_action(node, index):
+                action = {"path": "accessibility", "actionName": label, "fallbackReason": None}
+                break
+        else:
+            raise RuntimeError(f'{operation.get("action", "")} is not a valid secondary action')
+    elif tool == "scroll":
+        scroll_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node), operation.get("direction"), operation.get("pages"))
+        action = {"path": "synthetic", "actionName": "scroll", "fallbackReason": None}
+    elif tool == "drag":
+        drag_between(
+            screen_point(bounds, operation.get("fromElement"), operation.get("from_x"), operation.get("from_y"), from_node),
+            screen_point(bounds, operation.get("toElement"), operation.get("to_x"), operation.get("to_y"), to_node),
+        )
+        action = {"path": "synthetic", "actionName": "drag", "fallbackReason": None}
+    elif tool == "type_text":
+        type_text(operation.get("text", ""))
+        action = {"path": "synthetic", "actionName": "typeText", "fallbackReason": None}
+    elif tool == "press_key":
+        press_key(operation.get("key", ""))
+        action = {"path": "synthetic", "actionName": "pressKey", "fallbackReason": None}
+    elif tool == "hotkey":
+        hotkey(operation.get("key", ""))
+        action = {"path": "synthetic", "actionName": "hotkey", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}
+    elif tool == "paste_text":
+        paste_text(operation.get("text", ""))
+        action = {"path": "clipboard", "actionName": "paste", "fallbackReason": None, "verification": {"state": "unverified", "reason": "clipboard_paste"}}
+    elif tool == "set_value":
+        if not set_value(node, operation.get("value", "")):
+            raise RuntimeError("element value is not settable")
+        action = {"path": "accessibility", "actionName": "setValue", "fallbackReason": None}
+    else:
+        raise RuntimeError("unknown tool: " + str(tool))
+
+    try:
+        snapshot = make_snapshot(
+            operation.get("app", ""),
+            include_screenshot,
+            operation.get("windowId"),
+            operation.get("windowIndex"),
+        )
+    except Exception:
+        if operation.get("windowId") is None and operation.get("windowIndex") is None:
+            raise
+        action.setdefault("verification", {"state": "unverified", "reason": "window_changed"})
+        snapshot = make_snapshot(operation.get("app", ""), include_screenshot, None, None)
+
+    return {"ok": True, "action": action, "snapshot": snapshot}
+
+
+def main():
+    try:
+        ensure_desktop_bus()
+        with open(sys.argv[1], "r", encoding="utf-8") as handle:
+            operation = json.load(handle)
+        print(json.dumps(run_operation(operation), separators=(",", ":")))
+    except Exception as exc:
+        print(json.dumps({"ok": False, "error": str(exc)}, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

native/computer-use-macos/Package.swift

@@ -0,0 +1,36 @@
+// swift-tools-version: 6.0
+
+import PackageDescription
+
+let package = Package(
+    name: "OrcaComputerUseMacOS",
+    platforms: [
+        .macOS(.v14)
+    ],
+    products: [
+        .library(
+            name: "OrcaComputerUseMacOSCore",
+            targets: ["OrcaComputerUseMacOSCore"]
+        ),
+        .executable(
+            name: "orca-computer-use-macos",
+            targets: ["OrcaComputerUseMacOS"]
+        )
+    ],
+    targets: [
+        .target(
+            name: "OrcaComputerUseMacOSCore",
+            path: "Sources/OrcaComputerUseMacOSCore"
+        ),
+        .executableTarget(
+            name: "OrcaComputerUseMacOS",
+            dependencies: ["OrcaComputerUseMacOSCore"],
+            path: "Sources/OrcaComputerUseMacOS"
+        ),
+        .testTarget(
+            name: "OrcaComputerUseMacOSTests",
+            dependencies: ["OrcaComputerUseMacOSCore"],
+            path: "Tests/OrcaComputerUseMacOSTests"
+        )
+    ]
+)

native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SnapshotRendering.swift

@@ -0,0 +1,236 @@
+import Foundation
+
+public struct SnapshotRenderNode: Equatable {
+    public var role: String
+    public var roleDescription: String?
+    public var title: String?
+    public var label: String?
+    public var linkText: String?
+    public var value: String?
+    public var placeholder: String?
+    public var url: String?
+    public var traits: [String]
+    public var rawActions: [String]
+    public var childCount: Int
+    public var summary: String?
+    public var rowSummary: String?
+    public var webAreaDepth: Int?
+
+    public init(
+        role: String,
+        roleDescription: String? = nil,
+        title: String? = nil,
+        label: String? = nil,
+        linkText: String? = nil,
+        value: String? = nil,
+        placeholder: String? = nil,
+        url: String? = nil,
+        traits: [String] = [],
+        rawActions: [String] = [],
+        childCount: Int = 0,
+        summary: String? = nil,
+        rowSummary: String? = nil,
+        webAreaDepth: Int? = nil
+    ) {
+        self.role = role
+        self.roleDescription = roleDescription
+        self.title = title
+        self.label = label
+        self.linkText = linkText
+        self.value = value
+        self.placeholder = placeholder
+        self.url = url
+        self.traits = traits
+        self.rawActions = rawActions
+        self.childCount = childCount
+        self.summary = summary
+        self.rowSummary = rowSummary
+        self.webAreaDepth = webAreaDepth
+    }
+}
+
+public enum SnapshotRenderHeuristics {
+    public static func displayName(_ node: SnapshotRenderNode) -> String? {
+        if let title = clean(node.title) {
+            return title
+        }
+        if node.role == "AXLink", let url = clean(node.url), let text = clean(node.linkText ?? node.label ?? node.value) {
+            return "[\(markdownEscaped(text))](\(url))"
+        }
+        if node.role == "AXWebArea" {
+            return clean(node.label) ?? clean(node.value)
+        }
+        if ["AXButton", "AXPopUpButton", "AXImage"].contains(node.role) {
+            return clean(node.label)
+        }
+        if ["AXRow", "AXCell", "AXOutlineRow"].contains(node.role) {
+            return clean(node.rowSummary)
+        }
+        return clean(node.label)
+    }
+
+    public static func meaningfulActions(_ rawActions: [String], role: String) -> [String] {
+        let noisy: Set<String> = [
+            "AXPress",
+            "AXShowDefaultUI",
+            "AXShowAlternateUI",
+            "AXShowMenu",
+            "AXScrollToVisible",
+            "AXConfirm",
+            "AXRaise",
+        ]
+        return rawActions.filter { action in
+            if noisy.contains(action) { return false }
+            if role == "AXMenu" || role == "AXMenuItem" {
+                return action != "AXCancel" && action != "AXPick"
+            }
+            if role == "AXScrollArea",
+               (rawActions.contains("AXScrollUpByPage") || rawActions.contains("AXScrollDownByPage")),
+               action == "AXScrollLeftByPage" || action == "AXScrollRightByPage" {
+                return false
+            }
+            return true
+        }
+    }
+
+    public static func shouldElide(_ node: SnapshotRenderNode) -> Bool {
+        guard node.role == "AXGroup" || node.role == "AXUnknown" else { return false }
+        guard displayName(node) == nil,
+              node.traits.isEmpty,
+              meaningfulActions(node.rawActions, role: node.role).isEmpty,
+              clean(node.summary) == nil
+        else {
+            return false
+        }
+        if node.webAreaDepth != nil, node.childCount > 1 {
+            return false
+        }
+        return true
+    }
+
+    public static func shouldSuppressChildren(_ node: SnapshotRenderNode) -> Bool {
+        if node.role == "AXMenuBarItem" {
+            return true
+        }
+        let name = displayName(node)
+        if node.role == "AXLink" && name?.hasPrefix("[") == true {
+            return true
+        }
+        let hasCompactLabel = name != nil || clean(node.value) != nil || clean(node.summary) != nil
+        return hasCompactLabel && compactControlRoles.contains(node.role)
+    }
+
+    public static func shouldSuppressChildren(role: String, name: String? = nil) -> Bool {
+        shouldSuppressChildren(SnapshotRenderNode(role: role, title: name))
+    }
+
+    public static func line(index: Int, node: SnapshotRenderNode) -> String {
+        let name = displayName(node)
+        let roleText = roleText(node)
+        let meaningful = meaningfulActions(node.rawActions, role: node.role)
+        var line = roleText.isEmpty ? "\(index)" : "\(index) \(roleText)"
+        if !node.traits.isEmpty { line += " (\(node.traits.joined(separator: ", ")))" }
+        if let name { line += " \(sanitize(name))" }
+        if node.role != "AXLink", let description = clean(node.label), description != name {
+            line += ", Description: \(sanitize(description))"
+        }
+        if let valueSegment = formattedValueSegment(roleText: roleText, name: name, value: node.value) {
+            line += valueSegment
+        }
+        if let placeholder = clean(node.placeholder), placeholder != name, placeholder != node.value {
+            line += name == nil && clean(node.value) == nil ? " Placeholder: \(sanitize(placeholder))" : ", Placeholder: \(sanitize(placeholder))"
+        }
+        if let summary = clean(node.summary), summary != name {
+            line += ", Text: \(sanitize(summary))"
+        } else if let rowSummary = clean(node.rowSummary), rowSummary != name {
+            line += ", Text: \(sanitize(rowSummary))"
+        }
+        if !meaningful.isEmpty {
+            line += ", Secondary Actions: \(meaningful.map(prettyAction).joined(separator: ", "))"
+        }
+        return line
+    }
+
+    public static func roleText(_ node: SnapshotRenderNode) -> String {
+        if node.role == "AXGroup" || node.role == "AXUnknown" {
+            return "container"
+        }
+        if node.role == "AXLink" {
+            return "link"
+        }
+        if node.role == "AXWebArea" {
+            return clean(node.roleDescription) ?? "html content"
+        }
+        if node.role == "AXMenuBarItem" {
+            return ""
+        }
+        if let value = clean(node.roleDescription) {
+            return value.lowercased()
+        }
+        if node.role.hasPrefix("AX") {
+            return splitCamelCase(String(node.role.dropFirst(2))).lowercased()
+        }
+        return node.role
+    }
+
+    public static func prettyAction(_ action: String) -> String {
+        if action == "AXZoomWindow" {
+            return "zoom the window"
+        }
+        let stripped = action.hasPrefix("AX") ? String(action.dropFirst(2)) : action
+        return splitCamelCase(stripped.replacingOccurrences(of: "ByPage", with: "")).lowercased()
+    }
+
+    public static func sanitize(_ value: String) -> String {
+        value.replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\r", with: " ")
+    }
+
+    private static func clean(_ value: String?) -> String? {
+        guard let value else { return nil }
+        let sanitized = sanitize(value)
+        return sanitized.isEmpty ? nil : sanitized
+    }
+
+    private static func formattedValueSegment(roleText: String, name: String?, value: String?) -> String? {
+        guard let value = clean(value), value != name else { return nil }
+        if roleText == "heading", Int(value) != nil {
+            return nil
+        }
+        let clean = sanitize(value)
+        if roleText == "text" || roleText == "text entry area" || roleText == "scroll bar" || roleText == "value indicator" {
+            return " \(clean)"
+        }
+        return ", Value: \(clean)"
+    }
+
+    private static func markdownEscaped(_ value: String) -> String {
+        sanitize(value)
+            .replacingOccurrences(of: "\\", with: "\\\\")
+            .replacingOccurrences(of: "[", with: "\\[")
+            .replacingOccurrences(of: "]", with: "\\]")
+    }
+
+    private static func splitCamelCase(_ value: String) -> String {
+        var result = ""
+        for character in value {
+            if character.isUppercase, !result.isEmpty {
+                result.append(" ")
+            }
+            result.append(character)
+        }
+        return result
+    }
+
+    private static let compactControlRoles: Set<String> = [
+        "AXButton",
+        "AXCheckBox",
+        "AXComboBox",
+        "AXDisclosureTriangle",
+        "AXHeading",
+        "AXMenuItem",
+        "AXPopUpButton",
+        "AXRadioButton",
+        "AXStaticText",
+        "AXTab",
+    ]
+}

native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SnapshotRenderingTests.swift

@@ -0,0 +1,85 @@
+import XCTest
+@testable import OrcaComputerUseMacOSCore
+
+final class SnapshotRenderingTests: XCTestCase {
+    func testElidesAnonymousWrappers() {
+        let node = SnapshotRenderNode(role: "AXGroup", childCount: 1)
+
+        XCTAssertTrue(SnapshotRenderHeuristics.shouldElide(node))
+    }
+
+    func testPreservesWebAreaContainersWithMultipleChildren() {
+        let node = SnapshotRenderNode(role: "AXGroup", childCount: 3, webAreaDepth: 1)
+
+        XCTAssertFalse(SnapshotRenderHeuristics.shouldElide(node))
+    }
+
+    func testRendersMarkdownLinksAndSuppressesTheirChildren() {
+        let node = SnapshotRenderNode(role: "AXLink", linkText: "Skip [main]", url: "https://example.com/path")
+
+        XCTAssertEqual(SnapshotRenderHeuristics.line(index: 7, node: node), "7 link [Skip \\[main\\]](https://example.com/path)")
+        XCTAssertTrue(SnapshotRenderHeuristics.shouldSuppressChildren(node))
+    }
+
+    func testSuppressesChildrenForNamedCompactControls() {
+        let button = SnapshotRenderNode(role: "AXButton", roleDescription: "button", label: "Install GitHub", childCount: 1)
+        let heading = SnapshotRenderNode(role: "AXHeading", roleDescription: "heading", label: "Repository navigation", value: "2", childCount: 1)
+
+        XCTAssertTrue(SnapshotRenderHeuristics.shouldSuppressChildren(button))
+        XCTAssertTrue(SnapshotRenderHeuristics.shouldSuppressChildren(heading))
+        XCTAssertEqual(SnapshotRenderHeuristics.line(index: 5, node: heading), "5 heading Repository navigation")
+    }
+
+    func testKeepsChildrenForRowsWithNestedControls() {
+        let row = SnapshotRenderNode(role: "AXRow", roleDescription: "row", childCount: 3, rowSummary: "Liked Songs")
+
+        XCTAssertFalse(SnapshotRenderHeuristics.shouldSuppressChildren(row))
+    }
+
+    func testFiltersNoisyActionsAndFormatsSecondaryActions() {
+        let node = SnapshotRenderNode(
+            role: "AXWindow",
+            title: "Document",
+            rawActions: ["AXPress", "AXShowMenu", "AXScrollToVisible", "AXZoomWindow"]
+        )
+
+        XCTAssertEqual(SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role), ["AXZoomWindow"])
+        XCTAssertEqual(SnapshotRenderHeuristics.line(index: 1, node: node), "1 window Document, Secondary Actions: zoom the window")
+    }
+
+    func testSuppressesHorizontalScrollWhenVerticalScrollExists() {
+        let node = SnapshotRenderNode(
+            role: "AXScrollArea",
+            rawActions: ["AXScrollUpByPage", "AXScrollDownByPage", "AXScrollLeftByPage", "AXScrollRightByPage"]
+        )
+
+        XCTAssertEqual(SnapshotRenderHeuristics.meaningfulActions(node.rawActions, role: node.role), ["AXScrollUpByPage", "AXScrollDownByPage"])
+    }
+
+    func testTextFieldsKeepDistinctValueAndPlaceholder() {
+        let node = SnapshotRenderNode(
+            role: "AXTextField",
+            roleDescription: "text field",
+            label: "Address",
+            value: "https://example.com",
+            placeholder: "Search"
+        )
+
+        XCTAssertEqual(
+            SnapshotRenderHeuristics.line(index: 3, node: node),
+            "3 text field Address, Value: https://example.com, Placeholder: Search"
+        )
+    }
+
+    func testStaticTextUsesCompactValueFormatting() {
+        let node = SnapshotRenderNode(role: "AXStaticText", roleDescription: "text", value: "Home")
+
+        XCTAssertEqual(SnapshotRenderHeuristics.line(index: 4, node: node), "4 text Home")
+    }
+
+    func testRowSummaryBecomesName() {
+        let node = SnapshotRenderNode(role: "AXRow", roleDescription: "row", rowSummary: "General Settings Enabled")
+
+        XCTAssertEqual(SnapshotRenderHeuristics.line(index: 9, node: node), "9 row General Settings Enabled")
+    }
+}

native/computer-use-windows/runtime.ps1

@@ -0,0 +1,879 @@
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$OperationPath
+)
+
+$ErrorActionPreference = "Stop"
+
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+Add-Type -AssemblyName System.Drawing
+Add-Type -AssemblyName System.Windows.Forms
+
+Add-Type -TypeDefinition @"
+using System;
+using System.Runtime.InteropServices;
+
+public static class OrcaDesktopWin32 {
+    [StructLayout(LayoutKind.Sequential)]
+    public struct RECT {
+        public int Left;
+        public int Top;
+        public int Right;
+        public int Bottom;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct POINT {
+        public int X;
+        public int Y;
+    }
+
+    [DllImport("user32.dll")]
+    public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);
+
+    [DllImport("user32.dll")]
+    public static extern bool ScreenToClient(IntPtr hwnd, ref POINT point);
+
+    [DllImport("user32.dll")]
+    public static extern bool PostMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam);
+
+    [DllImport("user32.dll")]
+    public static extern bool ShowWindow(IntPtr hwnd, int nCmdShow);
+
+    [DllImport("user32.dll")]
+    public static extern bool SetForegroundWindow(IntPtr hwnd);
+
+    [DllImport("user32.dll")]
+    public static extern IntPtr GetForegroundWindow();
+}
+"@
+
+$MaxNodes = 1200
+$MaxDepth = 64
+$TextLimit = 500
+$BlockedAppFragments = @(
+    "1password",
+    "bitwarden",
+    "dashlane",
+    "lastpass",
+    "nordpass",
+    "proton pass"
+)
+
+$WindowsMessages = @{
+    Char = 0x0102
+    KeyDown = 0x0100
+    KeyUp = 0x0101
+    MouseMove = 0x0200
+    LeftDown = 0x0201
+    LeftUp = 0x0202
+    RightDown = 0x0204
+    RightUp = 0x0205
+    MiddleDown = 0x0207
+    MiddleUp = 0x0208
+    Wheel = 0x020A
+}
+
+function Write-OrcaJson($Payload) {
+    $Payload | ConvertTo-Json -Depth 100 -Compress
+}
+
+function New-OrcaFrame([double]$X, [double]$Y, [double]$Width, [double]$Height) {
+    if ($Width -le 0 -or $Height -le 0) { return $null }
+    [pscustomobject]@{ x = $X; y = $Y; width = $Width; height = $Height }
+}
+
+function Read-OrcaOperation([string]$Path) {
+    Get-Content -Raw -Path $Path | ConvertFrom-Json
+}
+
+function ConvertTo-OrcaLParam([int]$X, [int]$Y) {
+    [IntPtr]((($Y -band 0xffff) -shl 16) -bor ($X -band 0xffff))
+}
+
+function ConvertTo-OrcaWheelParam([int]$Delta) {
+    [IntPtr](($Delta -band 0xffff) -shl 16)
+}
+
+function Get-OrcaWindowProcesses {
+    @(Get-Process | Where-Object { $_.MainWindowHandle -ne 0 } | Sort-Object ProcessName, Id)
+}
+
+function Find-OrcaProcess([string]$Query) {
+    $needle = ""
+    if ($null -ne $Query) { $needle = $Query.Trim() }
+    if ([string]::IsNullOrWhiteSpace($needle)) { throw 'appNotFound("")' }
+    if ($needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase)) {
+        $needle = $needle.Substring(4)
+    }
+
+    $pid = 0
+    $processes = Get-OrcaWindowProcesses
+    if ([int]::TryParse($needle, [ref]$pid)) {
+        $match = $processes | Where-Object { $_.Id -eq $pid } | Select-Object -First 1
+        if ($null -ne $match) {
+            Assert-OrcaProcessAllowed $match
+            return $match
+        }
+    }
+
+    $processNeedle = $needle
+    if ($processNeedle.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) {
+        $processNeedle = $processNeedle.Substring(0, $processNeedle.Length - 4)
+    }
+
+    $match = $processes | Where-Object {
+        $_.ProcessName -ieq $processNeedle -or
+        "$($_.ProcessName).exe" -ieq $needle -or
+        $_.MainWindowTitle -ieq $needle -or
+        $_.MainWindowTitle -ilike "*$needle*"
+    } | Select-Object -First 1
+    if ($null -ne $match) {
+        Assert-OrcaProcessAllowed $match
+        return $match
+    }
+
+    throw "appNotFound(`"$Query`")"
+}
+
+function Assert-OrcaProcessAllowed($Process) {
+    $values = @($Process.ProcessName, $Process.MainWindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() }
+    foreach ($fragment in $BlockedAppFragments) {
+        foreach ($value in $values) {
+            if ($value.Contains($fragment)) {
+                throw "appBlocked(`"$($Process.ProcessName)`")"
+            }
+        }
+    }
+}
+
+function Get-OrcaRootElement($Process) {
+    if ($Process.MainWindowHandle -eq 0) {
+        throw "No top-level UI Automation window is available for $($Process.ProcessName)."
+    }
+    [Windows.Automation.AutomationElement]::FromHandle([IntPtr]$Process.MainWindowHandle)
+}
+
+function Get-OrcaWindowFrame($Process, $RootElement) {
+    $rect = New-Object OrcaDesktopWin32+RECT
+    if ([OrcaDesktopWin32]::GetWindowRect([IntPtr]$Process.MainWindowHandle, [ref]$rect)) {
+        return New-OrcaFrame $rect.Left $rect.Top ($rect.Right - $rect.Left) ($rect.Bottom - $rect.Top)
+    }
+
+    try {
+        $bounds = $RootElement.Current.BoundingRectangle
+        if (-not $bounds.IsEmpty) {
+            return New-OrcaFrame $bounds.X $bounds.Y $bounds.Width $bounds.Height
+        }
+    } catch {}
+    $null
+}
+
+function Get-OrcaWindowId($Process) {
+    [int64]$Process.MainWindowHandle
+}
+
+function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) {
+    if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) {
+        throw "windowNotFound(`"$WindowIndex`")"
+    }
+    if ($null -ne $WindowId -and [int64]$WindowId -ne (Get-OrcaWindowId $Process)) {
+        throw "windowNotFound(`"$WindowId`")"
+    }
+}
+
+function Restore-OrcaWindow($Process) {
+    if ($Process.MainWindowHandle -eq 0) { return }
+    [void][OrcaDesktopWin32]::ShowWindow([IntPtr]$Process.MainWindowHandle, 9)
+    [void][OrcaDesktopWin32]::SetForegroundWindow([IntPtr]$Process.MainWindowHandle)
+}
+
+function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation) {
+    if ([bool]$Operation.restoreWindow) { return }
+    if ([OrcaDesktopWin32]::GetForegroundWindow() -eq $WindowHandle) { return }
+    throw "window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window"
+}
+
+function Get-OrcaElementFrame($Element, $WindowFrame) {
+    try {
+        $bounds = $Element.Current.BoundingRectangle
+        if ($bounds.IsEmpty) { return $null }
+        if ($null -eq $WindowFrame) {
+            return New-OrcaFrame $bounds.X $bounds.Y $bounds.Width $bounds.Height
+        }
+        New-OrcaFrame ($bounds.X - $WindowFrame.x) ($bounds.Y - $WindowFrame.y) $bounds.Width $bounds.Height
+    } catch {
+        $null
+    }
+}
+
+function Get-OrcaProperty($Element, [string]$Name) {
+    try { [string]$Element.Current.$Name } catch { "" }
+}
+
+function Get-OrcaRuntimeId($Element) {
+    try { @($Element.GetRuntimeId()) } catch { @() }
+}
+
+function Get-OrcaValueText($Element) {
+    try {
+        if ($Element.Current.IsPassword) { return "[redacted]" }
+        $pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
+        $rawValue = $pattern.Current.Value
+        $text = if ($null -eq $rawValue) { "" } else { [string]$rawValue }
+        if ($text.Length -gt $TextLimit) { return $text.Substring(0, $TextLimit) + "..." }
+        $text
+    } catch {
+        ""
+    }
+}
+
+function Get-OrcaActions($Element) {
+    $actions = New-Object System.Collections.Generic.List[string]
+    foreach ($pattern in $Element.GetSupportedPatterns()) {
+        $name = [string]$pattern.ProgrammaticName
+        if ($name -like "InvokePatternIdentifiers.Pattern") { $actions.Add("Invoke") }
+        elseif ($name -like "TogglePatternIdentifiers.Pattern") { $actions.Add("Toggle") }
+        elseif ($name -like "SelectionItemPatternIdentifiers.Pattern") { $actions.Add("Select") }
+        elseif ($name -like "ScrollPatternIdentifiers.Pattern") { $actions.Add("Scroll") }
+        elseif ($name -like "ValuePatternIdentifiers.Pattern") { $actions.Add("SetValue") }
+    }
+    @($actions | Select-Object -Unique)
+}
+
+function Get-OrcaMeaningfulActions($Actions) {
+    $noisy = @("Invoke", "ScrollToVisible", "ShowMenu")
+    @($Actions | Where-Object { $noisy -notcontains $_ })
+}
+
+function Format-OrcaSnapshotText([string]$Text) {
+    if ([string]::IsNullOrWhiteSpace($Text)) { return "" }
+    (($Text -replace "\s+", " ").Trim())
+}
+
+function Format-OrcaValueSegment([string]$RoleKey, [string]$Title, [string]$Value) {
+    $clean = Format-OrcaSnapshotText $Value
+    if ([string]::IsNullOrWhiteSpace($clean) -or $clean -eq $Title) { return "" }
+    if ($RoleKey -eq "heading" -and $clean -match "^\d+$") { return "" }
+    if ($RoleKey -in @("text", "edit", "document", "scroll bar", "progress bar")) {
+        return " $clean"
+    }
+    ", Value: $clean"
+}
+
+function Test-OrcaSuppressChildren([string]$RoleKey, [string]$Title, [string]$Value, [string]$Summary) {
+    $hasCompactLabel = -not [string]::IsNullOrWhiteSpace($Title) -or -not [string]::IsNullOrWhiteSpace((Format-OrcaSnapshotText $Value)) -or -not [string]::IsNullOrWhiteSpace((Format-OrcaSnapshotText $Summary))
+    $hasCompactLabel -and $RoleKey -in @(
+        "button",
+        "check box",
+        "combo box",
+        "heading",
+        "hyperlink",
+        "link",
+        "menu item",
+        "radio button",
+        "tab item"
+    )
+}
+
+function Get-OrcaTextSnippets($Element, [int]$Limit = 6, [int]$MaxDepth = 3) {
+    $values = New-Object System.Collections.Generic.List[string]
+    $seen = New-Object System.Collections.Generic.HashSet[string]
+
+    function Visit-OrcaText($Node, [int]$Depth) {
+        if ($values.Count -ge $Limit -or $Depth -gt $MaxDepth) { return }
+        $role = try { [string]$Node.Current.LocalizedControlType } catch { "" }
+        if ($role -match "text|link|label") {
+            foreach ($raw in @((Get-OrcaProperty $Node "Name"), (Get-OrcaValueText $Node))) {
+                $value = (($raw -replace "\s+", " ").Trim())
+                if (-not [string]::IsNullOrWhiteSpace($value) -and $seen.Add($value)) {
+                    if ($value.Length -gt 80) { $value = $value.Substring(0, 80) + "..." }
+                    $values.Add($value)
+                    if ($values.Count -ge $Limit) { return }
+                }
+            }
+        }
+        try {
+            $children = $Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition)
+            for ($i = 0; $i -lt $children.Count; $i++) {
+                Visit-OrcaText $children.Item($i) ($Depth + 1)
+                if ($values.Count -ge $Limit) { return }
+            }
+        } catch {}
+    }
+
+    Visit-OrcaText $Element 0
+    @($values.ToArray())
+}
+
+function Test-OrcaPlainTextSubtree($Element, [int]$MaxDepth = 4) {
+    $script:sawOrcaText = $false
+    $allowed = @("pane", "group", "custom", "unknown", "text", "link", "image")
+
+    function Visit-OrcaPlainText($Node, [int]$Depth) {
+        if ($Depth -gt $MaxDepth) { return $false }
+        $role = try { [string]$Node.Current.LocalizedControlType } catch { "" }
+        $roleKey = $role.ToLowerInvariant()
+        if ($allowed -notcontains $roleKey) { return $false }
+        if ($roleKey -match "text|link") { $script:sawOrcaText = $true }
+        if (@(Get-OrcaMeaningfulActions @(Get-OrcaActions $Node)).Count -gt 0) { return $false }
+        try {
+            $children = $Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition)
+            for ($i = 0; $i -lt $children.Count; $i++) {
+                if (-not (Visit-OrcaPlainText $children.Item($i) ($Depth + 1))) { return $false }
+            }
+        } catch {}
+        return $true
+    }
+
+    (Visit-OrcaPlainText $Element 0) -and $script:sawOrcaText
+}
+
+function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
+    $controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
+    $nativeWindowHandle = try { [int64]$Element.Current.NativeWindowHandle } catch { 0 }
+    [pscustomobject]@{
+        index = $Index
+        runtimeId = @(Get-OrcaRuntimeId $Element)
+        automationId = Get-OrcaProperty $Element "AutomationId"
+        name = Get-OrcaProperty $Element "Name"
+        controlType = $controlType
+        localizedControlType = Get-OrcaProperty $Element "LocalizedControlType"
+        className = Get-OrcaProperty $Element "ClassName"
+        value = Get-OrcaValueText $Element
+        nativeWindowHandle = $nativeWindowHandle
+        frame = Get-OrcaElementFrame $Element $WindowFrame
+        actions = @(Get-OrcaActions $Element)
+    }
+}
+
+function Render-OrcaTree($RootElement, $WindowFrame) {
+    $records = New-Object System.Collections.Generic.List[object]
+    $lines = New-Object System.Collections.Generic.List[string]
+    $seen = New-Object System.Collections.Generic.HashSet[string]
+    $truncation = [pscustomobject]@{
+        truncated = $false
+        maxNodes = $MaxNodes
+        maxDepth = $MaxDepth
+        maxDepthReached = $false
+    }
+
+    function Visit-OrcaNode($Node, [int]$Depth) {
+        if ($records.Count -ge $MaxNodes -or $Depth -gt $MaxDepth) {
+            $truncation.truncated = $true
+            if ($Depth -gt $MaxDepth) { $truncation.maxDepthReached = $true }
+            return
+        }
+        $identity = try { (@($Node.GetRuntimeId()) -join ".") } catch { [Guid]::NewGuid().ToString() }
+        if (-not $seen.Add($identity)) { return }
+
+        $record = New-OrcaElementRecord $Node $records.Count $WindowFrame
+        $children = @()
+        try {
+            $children = @($Node.FindAll([Windows.Automation.TreeScope]::Children, [Windows.Automation.Condition]::TrueCondition))
+        } catch {}
+        $meaningfulActions = @(Get-OrcaMeaningfulActions $record.actions)
+        $title = if ([string]::IsNullOrWhiteSpace($record.name)) { $record.automationId } else { $record.name }
+        $role = if ([string]::IsNullOrWhiteSpace($record.localizedControlType)) { $record.controlType } else { $record.localizedControlType }
+        $roleKey = $role.ToLowerInvariant()
+        $snippets = @(Get-OrcaTextSnippets $Node 8 4)
+        $genericSummary = $null
+        if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) {
+            $genericSummary = ($snippets -join " ")
+        }
+        if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) {
+            for ($i = 0; $i -lt $children.Count; $i++) {
+                Visit-OrcaNode $children.Item($i) $Depth
+            }
+            return
+        }
+
+        $records.Add($record)
+
+        $line = "$($record.index) $role $(Format-OrcaSnapshotText $title)".TrimEnd()
+        $line += Format-OrcaValueSegment $roleKey $title $record.value
+        if (-not [string]::IsNullOrWhiteSpace($genericSummary) -and $genericSummary -ne $title) {
+            $line += ", Text: " + (Format-OrcaSnapshotText $genericSummary)
+        } elseif ($roleKey -in @("row", "data item", "list item")) {
+            $rowSummary = @((Get-OrcaTextSnippets $Node 6 3)) -join " "
+            if (-not [string]::IsNullOrWhiteSpace($rowSummary) -and $rowSummary -ne $title) {
+                $line += ", Text: " + (Format-OrcaSnapshotText $rowSummary)
+            }
+        }
+        if ($meaningfulActions.Count -gt 0) {
+            $line += ", Secondary Actions: " + ($meaningfulActions -join ", ")
+        }
+        $lines.Add(("`t" * $Depth) + $line)
+
+        if (-not [string]::IsNullOrWhiteSpace($genericSummary) -or (Test-OrcaSuppressChildren $roleKey $title $record.value $genericSummary)) { return }
+        for ($i = 0; $i -lt $children.Count; $i++) {
+            Visit-OrcaNode $children.Item($i) ($Depth + 1)
+        }
+    }
+
+    Visit-OrcaNode $RootElement 0
+    [pscustomobject]@{ elements = @($records.ToArray()); lines = @($lines.ToArray()); truncation = $truncation }
+}
+
+function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame) {
+    if (-not $IncludeScreenshot -or $null -eq $WindowFrame) { return $null }
+    try {
+        $width = [int][Math]::Max(1, [Math]::Round($WindowFrame.width))
+        $height = [int][Math]::Max(1, [Math]::Round($WindowFrame.height))
+        $bitmap = New-Object System.Drawing.Bitmap $width, $height
+        $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+        $graphics.CopyFromScreen([int][Math]::Round($WindowFrame.x), [int][Math]::Round($WindowFrame.y), 0, 0, $bitmap.Size)
+        $stream = New-Object System.IO.MemoryStream
+        $bitmap.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png)
+        $bytes = $stream.ToArray()
+        $graphics.Dispose()
+        $bitmap.Dispose()
+        $stream.Dispose()
+        [Convert]::ToBase64String($bytes)
+    } catch {
+        $null
+    }
+}
+
+function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false) {
+    $process = Find-OrcaProcess $Query
+    if ($RestoreWindow) { Restore-OrcaWindow $process }
+    Assert-OrcaWindowTarget $process $WindowId $WindowIndex
+    $root = Get-OrcaRootElement $process
+    $windowFrame = Get-OrcaWindowFrame $process $root
+    $tree = Render-OrcaTree $root $windowFrame
+
+    [pscustomobject]@{
+        snapshotId = [guid]::NewGuid().ToString()
+        app = [pscustomobject]@{
+            name = $process.ProcessName
+            bundleIdentifier = $process.ProcessName
+            bundleId = $process.ProcessName
+            pid = [int]$process.Id
+        }
+        windowTitle = $process.MainWindowTitle
+        windowId = Get-OrcaWindowId $process
+        windowBounds = $windowFrame
+        screenshotPngBase64 = Get-OrcaScreenshot $IncludeScreenshot $windowFrame
+        coordinateSpace = "window"
+        truncation = $tree.truncation
+        treeLines = @($tree.lines)
+        focusedSummary = $null
+        focusedElementId = $null
+        selectedText = $null
+        elements = @($tree.elements)
+    }
+}
+
+function Get-OrcaAppList {
+    @(Get-OrcaWindowProcesses | ForEach-Object {
+        [pscustomobject]@{
+            name = $_.ProcessName
+            bundleIdentifier = $_.ProcessName
+            bundleId = $_.ProcessName
+            pid = [int]$_.Id
+        }
+    })
+}
+
+function Get-OrcaWindowList([string]$Query) {
+    $process = Find-OrcaProcess $Query
+    $root = Get-OrcaRootElement $process
+    $windowFrame = Get-OrcaWindowFrame $process $root
+    $x = $null
+    $y = $null
+    $width = 0
+    $height = 0
+    if ($null -ne $windowFrame) {
+        $x = [int][Math]::Round($windowFrame.x)
+        $y = [int][Math]::Round($windowFrame.y)
+        $width = [int][Math]::Max(0, [Math]::Round($windowFrame.width))
+        $height = [int][Math]::Max(0, [Math]::Round($windowFrame.height))
+    }
+    $app = [pscustomobject]@{
+        name = $process.ProcessName
+        bundleIdentifier = $process.ProcessName
+        bundleId = $process.ProcessName
+        pid = [int]$process.Id
+    }
+    [pscustomobject]@{
+        app = $app
+        windows = @([pscustomobject]@{
+            index = 0
+            app = $app
+            id = Get-OrcaWindowId $process
+            title = $process.MainWindowTitle
+            x = $x
+            y = $y
+            width = $width
+            height = $height
+            isMinimized = $false
+            isOffscreen = $false
+            screenIndex = $null
+            platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = Get-OrcaWindowId $process }
+        })
+    }
+}
+
+function Get-OrcaHandshake {
+    [pscustomobject]@{
+        platform = "win32"
+        provider = "orca-computer-use-windows"
+        providerVersion = "1.0.0"
+        protocolVersion = 1
+        supports = [pscustomobject]@{
+            apps = [pscustomobject]@{ list = $true; bundleIds = $false; pids = $true }
+            windows = [pscustomobject]@{ list = $true; targetById = $true; targetByIndex = $true; focus = $false; moveResize = $false }
+            observation = [pscustomobject]@{ screenshot = $true; annotatedScreenshot = $false; elementFrames = $true; ocr = $false }
+            actions = [pscustomobject]@{
+                click = $true
+                typeText = $true
+                pressKey = $true
+                hotkey = $true
+                pasteText = $true
+                scroll = $true
+                drag = $true
+                setValue = $true
+                performAction = $true
+            }
+            surfaces = [pscustomobject]@{ menus = $false; dialogs = $false; dock = $false; menubar = $false }
+        }
+    }
+}
+
+function Test-OrcaSameRuntimeId($Left, $Right) {
+    if ($null -eq $Left -or $null -eq $Right -or $Left.Count -ne $Right.Count) { return $false }
+    for ($i = 0; $i -lt $Left.Count; $i++) {
+        if ([int]$Left[$i] -ne [int]$Right[$i]) { return $false }
+    }
+    $true
+}
+
+function Find-OrcaElement($RootElement, $Record) {
+    if ($null -eq $Record) { return $null }
+    if ($Record.index -eq 0) { return $RootElement }
+
+    try {
+        $descendants = $RootElement.FindAll([Windows.Automation.TreeScope]::Descendants, [Windows.Automation.Condition]::TrueCondition)
+        for ($i = 0; $i -lt $descendants.Count; $i++) {
+            $candidate = $descendants.Item($i)
+            if (Test-OrcaSameRuntimeId @($candidate.GetRuntimeId()) @($Record.runtimeId)) {
+                return $candidate
+            }
+        }
+    } catch {}
+    $null
+}
+
+function Invoke-OrcaPrimaryAction($Element) {
+    foreach ($pattern in @(
+        [Windows.Automation.InvokePattern]::Pattern,
+        [Windows.Automation.SelectionItemPattern]::Pattern,
+        [Windows.Automation.TogglePattern]::Pattern
+    )) {
+        try {
+            $instance = $Element.GetCurrentPattern($pattern)
+            if ($pattern -eq [Windows.Automation.InvokePattern]::Pattern) { $instance.Invoke(); return $true }
+            if ($pattern -eq [Windows.Automation.SelectionItemPattern]::Pattern) { $instance.Select(); return $true }
+            if ($pattern -eq [Windows.Automation.TogglePattern]::Pattern) { $instance.Toggle(); return $true }
+        } catch {}
+    }
+    $false
+}
+
+function Invoke-OrcaNamedAction($Element, [string]$Action) {
+    $wanted = ""
+    if ($null -ne $Action) { $wanted = $Action.Trim().ToLowerInvariant() }
+    switch ($wanted) {
+        "invoke" {
+            $pattern = $Element.GetCurrentPattern([Windows.Automation.InvokePattern]::Pattern)
+            $pattern.Invoke()
+            return $true
+        }
+        "select" {
+            $pattern = $Element.GetCurrentPattern([Windows.Automation.SelectionItemPattern]::Pattern)
+            $pattern.Select()
+            return $true
+        }
+        "toggle" {
+            $pattern = $Element.GetCurrentPattern([Windows.Automation.TogglePattern]::Pattern)
+            $pattern.Toggle()
+            return $true
+        }
+        default {
+            return $false
+        }
+    }
+}
+
+function Set-OrcaElementValue($Element, [string]$Value) {
+    try {
+        $pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
+        if (-not $pattern.Current.IsReadOnly) {
+            $pattern.SetValue($Value)
+            return $true
+        }
+    } catch {}
+    $false
+}
+
+function Get-OrcaScreenPoint($Operation, $WindowFrame) {
+    if ($null -ne $Operation.element) {
+        throw "stale element frame; run get-app-state again and use a fresh element index"
+    }
+    @{
+        x = [int][Math]::Round($WindowFrame.x + [double]$Operation.x)
+        y = [int][Math]::Round($WindowFrame.y + [double]$Operation.y)
+    }
+}
+
+function Get-OrcaElementScreenPoint($Element) {
+    if ($null -eq $Element) { return $null }
+    try {
+        $rect = $Element.Current.BoundingRectangle
+        if ($rect.Width -gt 0 -and $rect.Height -gt 0) {
+            return @{
+                x = [int][Math]::Round($rect.X + ($rect.Width / 2))
+                y = [int][Math]::Round($rect.Y + ($rect.Height / 2))
+            }
+        }
+    } catch {}
+    $null
+}
+
+function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count) {
+    $point = New-Object OrcaDesktopWin32+POINT
+    $point.X = $ScreenX
+    $point.Y = $ScreenY
+    [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$point)
+
+    $down = $WindowsMessages.LeftDown
+    $up = $WindowsMessages.LeftUp
+    $flag = 1
+    if ($Button -eq "right") {
+        $down = $WindowsMessages.RightDown
+        $up = $WindowsMessages.RightUp
+        $flag = 2
+    } elseif ($Button -eq "middle") {
+        $down = $WindowsMessages.MiddleDown
+        $up = $WindowsMessages.MiddleUp
+        $flag = 16
+    }
+
+    $position = ConvertTo-OrcaLParam $point.X $point.Y
+    for ($i = 0; $i -lt [Math]::Max(1, $Count); $i++) {
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, $position)
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $down, [IntPtr]$flag, $position)
+        Start-Sleep -Milliseconds 35
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $up, [IntPtr]::Zero, $position)
+    }
+}
+
+function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {
+    $start = New-Object OrcaDesktopWin32+POINT
+    $start.X = [int]$From.x
+    $start.Y = [int]$From.y
+    [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$start)
+
+    $end = New-Object OrcaDesktopWin32+POINT
+    $end.X = [int]$To.x
+    $end.Y = [int]$To.y
+    [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$end)
+
+    [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, (ConvertTo-OrcaLParam $start.X $start.Y))
+    [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftDown, [IntPtr]1, (ConvertTo-OrcaLParam $start.X $start.Y))
+    for ($step = 1; $step -le 12; $step++) {
+        $x = [int][Math]::Round($start.X + (($end.X - $start.X) * $step / 12))
+        $y = [int][Math]::Round($start.Y + (($end.Y - $start.Y) * $step / 12))
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]1, (ConvertTo-OrcaLParam $x $y))
+        Start-Sleep -Milliseconds 20
+    }
+    [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftUp, [IntPtr]::Zero, (ConvertTo-OrcaLParam $end.X $end.Y))
+}
+
+function Send-OrcaText([IntPtr]$WindowHandle, [string]$Text) {
+    foreach ($character in $Text.ToCharArray()) {
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.Char, [IntPtr][int][char]$character, [IntPtr]::Zero)
+        Start-Sleep -Milliseconds 8
+    }
+}
+
+function Get-OrcaVirtualKey([string]$Key) {
+    $normalized = $Key.ToLowerInvariant()
+    $map = @{
+        "return" = 0x0D; "enter" = 0x0D; "tab" = 0x09; "escape" = 0x1B; "esc" = 0x1B
+        "backspace" = 0x08; "delete" = 0x2E; "space" = 0x20; "left" = 0x25
+        "up" = 0x26; "right" = 0x27; "down" = 0x28; "home" = 0x24; "end" = 0x23
+    }
+    if ($map.ContainsKey($normalized)) { return $map[$normalized] }
+    if ($normalized.Length -eq 1) { return [int][char]$normalized.ToUpperInvariant()[0] }
+    throw "Unsupported key: $Key"
+}
+
+function Send-OrcaKey([IntPtr]$WindowHandle, [string]$Key) {
+    $virtualKey = Get-OrcaVirtualKey $Key
+    [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$virtualKey, [IntPtr]::Zero)
+    Start-Sleep -Milliseconds 25
+    [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$virtualKey, [IntPtr]::Zero)
+}
+
+function Get-OrcaModifierVirtualKey([string]$Modifier) {
+    switch ($Modifier.ToLowerInvariant()) {
+        { $_ -in @("ctrl", "control", "cmdorctrl", "commandorcontrol") } { return 0x11 }
+        { $_ -in @("shift") } { return 0x10 }
+        { $_ -in @("alt", "option") } { return 0x12 }
+        { $_ -in @("meta", "super", "win", "cmd", "command") } { return 0x5B }
+        default { throw "Unsupported modifier: $Modifier" }
+    }
+}
+
+function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) {
+    $parts = @($KeySpec.Split("+") | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
+    if ($parts.Count -eq 0) { throw "Unsupported key: $KeySpec" }
+    $key = $parts[$parts.Count - 1]
+    $modifiers = @()
+    if ($parts.Count -gt 1) {
+        $modifiers = @($parts[0..($parts.Count - 2)] | ForEach-Object { Get-OrcaModifierVirtualKey $_ })
+    }
+    foreach ($modifier in $modifiers) {
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$modifier, [IntPtr]::Zero)
+    }
+    Send-OrcaKey $WindowHandle $key
+    for ($i = $modifiers.Count - 1; $i -ge 0; $i--) {
+        [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$modifiers[$i], [IntPtr]::Zero)
+    }
+}
+
+function Send-OrcaPasteText([IntPtr]$WindowHandle, [string]$Text) {
+    $previous = $null
+    $hadPrevious = $false
+    try { $previous = [System.Windows.Forms.Clipboard]::GetDataObject() } catch {}
+    $hadPrevious = $null -ne $previous
+    try {
+        Set-Clipboard -Value $Text
+        Send-OrcaHotkey $WindowHandle "Ctrl+v"
+    } finally {
+        if ($hadPrevious) {
+            [System.Windows.Forms.Clipboard]::SetDataObject($previous, $true)
+        } else {
+            [System.Windows.Forms.Clipboard]::Clear()
+        }
+    }
+}
+
+function Invoke-OrcaOperation($Operation) {
+    $includeScreenshot = -not [bool]$Operation.noScreenshot
+    if ($Operation.tool -eq "handshake") {
+        return [pscustomobject]@{ ok = $true; capabilities = Get-OrcaHandshake }
+    }
+    if ($Operation.tool -eq "list_apps") {
+        return [pscustomobject]@{ ok = $true; apps = @(Get-OrcaAppList) }
+    }
+    if ($Operation.tool -eq "list_windows") {
+        $list = Get-OrcaWindowList $Operation.app
+        return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) }
+    }
+    if ($Operation.tool -eq "get_app_state") {
+        return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) }
+    }
+
+    $process = Find-OrcaProcess $Operation.app
+    if ([bool]$Operation.restoreWindow) { Restore-OrcaWindow $process }
+    Assert-OrcaWindowTarget $process $Operation.windowId $Operation.windowIndex
+    $root = Get-OrcaRootElement $process
+    $windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $process $root }
+    $element = Find-OrcaElement $root $Operation.element
+    $fromElement = Find-OrcaElement $root $Operation.fromElement
+    $toElement = Find-OrcaElement $root $Operation.toElement
+    $handle = [IntPtr]$process.MainWindowHandle
+    if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text")) {
+        Assert-OrcaKeyboardFocus $handle $Operation
+    }
+    $action = $null
+
+    switch ($Operation.tool) {
+        "click" {
+            $handledByPattern = $false
+            if ($null -ne $element -and $Operation.mouse_button -ne "right" -and $Operation.mouse_button -ne "middle" -and [int]$Operation.click_count -le 1) {
+                $handledByPattern = Invoke-OrcaPrimaryAction $element
+            }
+            if (-not $handledByPattern) {
+                $point = Get-OrcaElementScreenPoint $element
+                if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
+                Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button ([int]$Operation.click_count)
+                $action = [pscustomobject]@{ path = "synthetic"; actionName = $null; fallbackReason = "actionUnsupported" }
+            } else {
+                $action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }
+            }
+        }
+        "perform_secondary_action" {
+            if ($null -eq $element) { throw "unknown element_index" }
+            if (-not (Invoke-OrcaNamedAction $element $Operation.action)) {
+                throw "$($Operation.action) is not a valid secondary action"
+            }
+            $action = [pscustomobject]@{ path = "accessibility"; actionName = $Operation.action; fallbackReason = $null }
+        }
+        "scroll" {
+            $delta = 120 * [int][Math]::Max(1, [Math]::Ceiling([double]$Operation.pages))
+            if ($Operation.direction -eq "down" -or $Operation.direction -eq "right") { $delta = -1 * $delta }
+            $point = Get-OrcaElementScreenPoint $element
+            if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
+            [void][OrcaDesktopWin32]::PostMessage($handle, $WindowsMessages.Wheel, (ConvertTo-OrcaWheelParam $delta), (ConvertTo-OrcaLParam $point.x $point.y))
+            $action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }
+        }
+        "drag" {
+            $from = Get-OrcaElementScreenPoint $fromElement
+            if ($null -eq $from -and $null -ne $Operation.fromElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
+            if ($null -eq $from) { $from = @{ x = $windowFrame.x + [double]$Operation.from_x; y = $windowFrame.y + [double]$Operation.from_y } }
+            $to = Get-OrcaElementScreenPoint $toElement
+            if ($null -eq $to -and $null -ne $Operation.toElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
+            if ($null -eq $to) { $to = @{ x = $windowFrame.x + [double]$Operation.to_x; y = $windowFrame.y + [double]$Operation.to_y } }
+            Send-OrcaDrag $handle $from $to
+            $action = [pscustomobject]@{ path = "synthetic"; actionName = "drag"; fallbackReason = $null }
+        }
+        "type_text" {
+            Send-OrcaText $handle ([string]$Operation.text)
+            $action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null }
+        }
+        "press_key" {
+            Send-OrcaKey $handle ([string]$Operation.key)
+            $action = [pscustomobject]@{ path = "synthetic"; actionName = "pressKey"; fallbackReason = $null }
+        }
+        "hotkey" {
+            Send-OrcaHotkey $handle ([string]$Operation.key)
+            $action = [pscustomobject]@{ path = "synthetic"; actionName = "hotkey"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
+        }
+        "paste_text" {
+            Send-OrcaPasteText $handle ([string]$Operation.text)
+            $action = [pscustomobject]@{ path = "clipboard"; actionName = "paste"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "clipboard_paste" } }
+        }
+        "set_value" {
+            if ($null -eq $element -or -not (Set-OrcaElementValue $element ([string]$Operation.value))) {
+                throw "element value is not settable"
+            }
+            $action = [pscustomobject]@{ path = "accessibility"; actionName = "setValue"; fallbackReason = $null }
+        }
+        default {
+            throw "unsupported tool: $($Operation.tool)"
+        }
+    }
+
+    try {
+        $snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex
+    } catch {
+        if ($null -eq $Operation.windowId -and $null -eq $Operation.windowIndex) { throw }
+        if ($null -eq $action.verification) {
+            $action | Add-Member -NotePropertyName verification -NotePropertyValue ([pscustomobject]@{ state = "unverified"; reason = "window_changed" })
+        }
+        $snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $null $null
+    }
+    [pscustomobject]@{ ok = $true; action = $action; snapshot = $snapshot }
+}
+
+try {
+    $operation = Read-OrcaOperation $OperationPath
+    Write-OrcaJson (Invoke-OrcaOperation $operation)
+} catch {
+    Write-OrcaJson ([pscustomobject]@{ ok = $false; error = [string]$_.Exception.Message })
+}

package.json

@@ -25,10 +25,13 @@
     "start": "electron-vite preview",
     "dev": "node config/scripts/run-electron-vite-dev.mjs",
     "build:relay": "node config/scripts/build-relay.mjs",
+    "build:computer-macos": "node config/scripts/build-computer-macos.mjs",
+    "smoke:computer": "node config/scripts/computer-use-smoke.mjs",
+    "verify:computer-native": "node config/scripts/verify-computer-native.mjs",
     "build:cli": "tsc -p config/tsconfig.cli.json --outDir out --composite false --incremental false && node config/scripts/install-dev-cli.mjs",
     "build:electron-vite": "node config/scripts/run-electron-vite-build.mjs",
-    "build": "pnpm run typecheck && pnpm run build:relay && pnpm run build:electron-vite && pnpm run build:cli",
-    "build:release": "pnpm run build:relay && pnpm run build:electron-vite && pnpm run build:cli",
+    "build": "pnpm run typecheck && pnpm run build:relay && pnpm run build:computer-macos && pnpm run build:electron-vite && pnpm run build:cli",
+    "build:release": "pnpm run build:relay && pnpm run build:computer-macos && pnpm run verify:computer-native && pnpm run build:electron-vite && pnpm run build:cli",
     "postinstall": "pnpm rebuild electron && node config/scripts/rebuild-native-deps.mjs",
     "rebuild:electron": "node config/scripts/rebuild-native-deps.mjs",
     "build:unpack": "pnpm run build && electron-builder --config config/electron-builder.config.cjs --dir",
@@ -38,7 +41,8 @@
     "build:mac:release": "node config/scripts/verify-macos-release-env.mjs && ORCA_MAC_RELEASE=1 pnpm run build && ORCA_MAC_RELEASE=1 electron-builder --config config/electron-builder.config.cjs --mac",
     "build:linux": "pnpm run build && electron-builder --config config/electron-builder.config.cjs --linux",
     "test:e2e": "npx playwright test --config tests/playwright.config.ts --project electron-headless",
-    "test:e2e:headful": "npx playwright test --config tests/playwright.config.ts --project electron-headful"
+    "test:e2e:headful": "npx playwright test --config tests/playwright.config.ts --project electron-headful",
+    "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts"
   },
   "dependencies": {
     "@dnd-kit/core": "^6.3.1",

resources/build/entitlements.computer-use.mac.plist

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+</dict>
+</plist>

skills/computer-use/SKILL.md

@@ -0,0 +1,192 @@
+---
+name: computer-use
+description: Use Orca's computer-use CLI to inspect and control local desktop apps through accessibility trees, screenshots, and safe UI actions. Use when an agent needs to list desktop apps, get an app state, read visible UI, click, type, press keys, scroll, drag, set values, or perform app accessibility actions. Triggers include "computer use", "orca computer", "list apps", "get app state", "read Spotify", "read Slack", "click app", "type text", "press key", "set value", "scroll app", "drag app", and desktop app interaction tasks.
+---
+
+# Computer Use
+
+Use this skill when the task should operate through Orca's desktop computer-use surface rather than native Codex computer tools, raw AppleScript, ad hoc screenshots, or direct app internals.
+
+## Preconditions
+
+- Prefer the public `orca computer ...` command.
+- In this Orca worktree, use `./config/scripts/orca-dev computer ...` when testing the local dev runtime.
+- Prefer `--json` for agent-driven calls. Screenshot image bytes are omitted from JSON and written to `screenshot.path` when present.
+- Do not push, submit forms, send messages, buy items, delete data, or change account settings unless the user explicitly asked for that specific action.
+- If an app contains sensitive content, read only what the user requested and avoid unnecessary screenshots or logs.
+
+Check runtime availability first:
+
+```bash
+orca status --json
+orca computer capabilities --json
+```
+
+For local development against this worktree:
+
+```bash
+./config/scripts/orca-dev status --json
+```
+
+## Core Workflow
+
+Use a snapshot-act-snapshot loop:
+
+1. Discover apps:
+
+```bash
+orca computer list-apps --json
+```
+
+2. Get a fresh state for the target app:
+
+```bash
+orca computer get-app-state --app com.spotify.client --json
+```
+
+3. Choose an element from that state.
+
+4. Perform one action:
+
+```bash
+orca computer click --app com.spotify.client --element-index 42 --json
+```
+
+5. Inspect the action result before deciding whether to act again. Actions return a fresh state:
+
+```bash
+orca computer click --app com.spotify.client --element-index 42 --json
+```
+
+Element indexes are scoped to the current app state. They can go stale after navigation, focus changes, scrolling, window changes, or app re-rendering. Never carry indexes across unrelated steps without refreshing state.
+
+## App Selectors
+
+Prefer bundle IDs returned by `list-apps`:
+
+```bash
+orca computer get-app-state --app com.microsoft.edgemac --json
+orca computer get-app-state --app com.spotify.client --json
+```
+
+Names are acceptable when unambiguous:
+
+```bash
+orca computer get-app-state --app Spotify --json
+```
+
+Use `pid:<number>` only when bundle ID or name matching is ambiguous:
+
+```bash
+orca computer get-app-state --app pid:12345 --json
+```
+
+## Commands
+
+```bash
+orca computer permissions --json
+orca computer capabilities --json
+orca computer list-apps --json
+orca computer list-windows --app <app> --json
+orca computer get-app-state --app <app> --json
+orca computer click --app <app> --element-index <index> --json
+orca computer perform-secondary-action --app <app> --element-index <index> --action <name> --json
+orca computer set-value --app <app> --element-index <index> --value "text" --json
+orca computer type-text --app <app> --text "text" --json
+orca computer press-key --app <app> --key Return --json
+orca computer hotkey --app <app> --key CmdOrCtrl+A --json
+orca computer paste-text --app <app> --text "text" --json
+orca computer scroll --app <app> (--element-index <index> | --x <x> --y <y>) --direction down --json
+orca computer drag --app <app> --from-x 100 --from-y 100 --to-x 300 --to-y 300 --json
+```
+
+Use `--no-screenshot` only when pixels are not needed. Screenshots are often the only useful signal for Electron, WebView, or canvas-heavy apps with shallow accessibility trees.
+
+Coordinates are window-local. Use coordinates from the latest screenshot/state for the same target window.
+
+Use `--text-stdin` or `--value-stdin` for sensitive text so payloads do not land in shell history.
+On Linux and Windows, action payloads still pass through a short-lived local operation file.
+
+```bash
+printf '%s' "$TEXT" | orca computer set-value --app <app> --element-index <index> --value-stdin --json
+```
+
+## Choosing Actions
+
+Prefer semantic actions over raw keyboard input:
+
+- Use `set-value` for known editable fields.
+- Use `click` for buttons, tabs, menu items, checkboxes, and other direct controls.
+- Use `perform-secondary-action` only when the state lists a concrete action name and the user intent matches it.
+- Use `type-text` after focusing a field and confirming the app has a focused text receiver.
+- Use `press-key` for navigation keys, Return, Escape, shortcuts, or submitting a field after the state confirms the right target is active.
+
+Why: keyboard input is process-targeted on macOS, but it still depends on the target app having a valid focused receiver. `set-value` targets the accessibility element directly and is more reliable when supported.
+
+## Foreground And Background
+
+Some actions work while the app is in the background. Treat this as app-dependent:
+
+- `set-value` can work in the background when the app exposes a writable accessibility value.
+- `click` and accessibility actions may work in the background for some native controls.
+- `type-text` and `press-key` are targeted to the app process on macOS, but the app may ignore them unless it owns focus or already has an active text receiver.
+
+If an action returns success but the UI did not change, do not repeat the same action blindly. Run `get-app-state` again, inspect the screenshot/tree, then switch to a more semantic action or bring/focus the target if needed.
+
+## Screenshots
+
+`get-app-state` returns an accessibility tree and, by default, a screenshot. Use both:
+
+- Trust the tree for element indexes, names, roles, values, and actions.
+- Trust the screenshot for visual confirmation, especially in Electron and WebView apps.
+- If the tree is shallow, use screenshot evidence before deciding whether any action is safe.
+- If screenshot capture fails or returns no image, the app may be hidden, minimized, off-screen, or have no visible window.
+
+Use restore only when appropriate for the task:
+
+```bash
+orca computer get-app-state --app <app> --restore-window --json
+```
+
+## App-Specific Notes
+
+### Browsers
+
+For Edge, Chrome, and similar browsers, prefer setting the address/search field directly:
+
+```bash
+orca computer get-app-state --app com.microsoft.edgemac --json
+orca computer set-value --app com.microsoft.edgemac --element-index <addressBarIndex> --value "test123" --json
+orca computer press-key --app com.microsoft.edgemac --key Return --json
+orca computer get-app-state --app com.microsoft.edgemac --json
+```
+
+Do not assume raw typing went to the address bar. Confirm the field or page changed after pressing Return.
+
+### Spotify
+
+Spotify state can update asynchronously after playback or network-backed search. After a playback click, run `get-app-state` before clicking again.
+
+For search, prefer `set-value` on the search combobox, usually named like `What do you want to play?`. `type-text` may only work when Spotify owns focus and that field is already focused.
+
+### Slack
+
+Slack may expose a shallow accessibility tree while the screenshot contains the useful information. Reading visible Slack UI is acceptable when requested, but do not send messages or trigger workflows unless explicitly asked.
+
+## Error Handling
+
+- `app_not_found`: run `list-apps` and retry with the bundle ID.
+- `element_not_found`: the index is stale; run `get-app-state` again.
+- `action_failed`: inspect the element role/actions and try a more semantic action.
+- Empty tree or no screenshot: the app may have no visible window, be minimized, or be blocked by permissions.
+- Permission errors: the user needs to grant Accessibility or Screen Recording to `Orca Computer Use`. Run `orca computer permissions --json`, use the setup UI, then retry `orca computer get-app-state --app <bundle> --json`.
+
+## Safety Checks
+
+Before acting, classify the action:
+
+- Safe: read state, list apps, inspect screenshot, focus a search box, scroll, open a harmless tab.
+- Needs care: typing into a focused field, pressing Return, clicking a primary button.
+- Requires explicit user permission: sending messages, posting, purchasing, deleting, submitting forms, changing settings, signing in, or exposing secrets.
+
+When uncertain, stop after `get-app-state` and report what is visible instead of acting.

src/cli/args.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, it } from 'vitest'
+
+import { parseArgs } from './args'
+
+describe('parseArgs', () => {
+  it('keeps an empty string as a flag value', () => {
+    const parsed = parseArgs(['computer', 'set-value', '--value', '', '--json'])
+
+    expect(parsed.commandPath).toEqual(['computer', 'set-value'])
+    expect(parsed.flags.get('value')).toBe('')
+    expect(parsed.flags.get('json')).toBe(true)
+  })
+})

src/cli/args.ts

@@ -28,8 +28,9 @@ export function parseArgs(argv: string[]): ParsedArgs {
     }
 
     const flag = token.slice(2)
+    const hasNext = i + 1 < argv.length
     const next = argv[i + 1]
-    if (!next || next.startsWith('--')) {
+    if (!hasNext || next.startsWith('--')) {
       flags.set(flag, true)
       continue
     }
@@ -61,7 +62,7 @@ export function supportsBrowserPageFlag(commandPath: string[]): boolean {
   if (['open', 'status'].includes(commandPath[0])) {
     return false
   }
-  if (['repo', 'worktree', 'terminal'].includes(commandPath[0])) {
+  if (['repo', 'worktree', 'terminal', 'computer'].includes(commandPath[0])) {
     return false
   }
   return ![
@@ -90,7 +91,8 @@ export function isCommandGroup(commandPath: string[]): boolean {
         'clipboard',
         'dialog',
         'storage',
-        'orchestration'
+        'orchestration',
+        'computer'
       ].includes(commandPath[0])) ||
     (commandPath.length === 2 &&
       commandPath[0] === 'storage' &&

src/cli/dispatch.ts

@@ -13,6 +13,7 @@ import { BROWSER_CAPTURE_HANDLERS } from './handlers/browser-capture'
 import { BROWSER_ENV_HANDLERS } from './handlers/browser-env'
 import { BROWSER_STORAGE_HANDLERS } from './handlers/browser-storage'
 import { ORCHESTRATION_HANDLERS } from './handlers/orchestration'
+import { COMPUTER_HANDLERS } from './handlers/computer'
 
 export type HandlerContext = {
   flags: Map<string, string | boolean>
@@ -38,7 +39,8 @@ function buildHandlers(): Map<string, CommandHandler> {
     BROWSER_CAPTURE_HANDLERS,
     BROWSER_ENV_HANDLERS,
     BROWSER_STORAGE_HANDLERS,
-    ORCHESTRATION_HANDLERS
+    ORCHESTRATION_HANDLERS,
+    COMPUTER_HANDLERS
   ]
   for (const group of groups) {
     for (const [key, handler] of Object.entries(group)) {

src/cli/flags.test.ts

@@ -0,0 +1,11 @@
+import { describe, expect, it } from 'vitest'
+
+import { getRequiredStringFlagAllowingEmpty } from './flags'
+
+describe('CLI flags', () => {
+  it('allows required string flags to be empty when the command opts in', () => {
+    const flags = new Map<string, string | boolean>([['value', '']])
+
+    expect(getRequiredStringFlagAllowingEmpty(flags, 'value')).toBe('')
+  })
+})

src/cli/flags.ts

@@ -8,6 +8,17 @@ export function getRequiredStringFlag(flags: Map<string, string | boolean>, name
   throw new RuntimeClientError('invalid_argument', `Missing required --${name}`)
 }
 
+export function getRequiredStringFlagAllowingEmpty(
+  flags: Map<string, string | boolean>,
+  name: string
+): string {
+  const value = flags.get(name)
+  if (typeof value === 'string') {
+    return value
+  }
+  throw new RuntimeClientError('invalid_argument', `Missing required --${name}`)
+}
+
 export function getOptionalStringFlag(
   flags: Map<string, string | boolean>,
   name: string

src/cli/format.ts

@@ -1,3 +1,7 @@
+/* eslint-disable max-lines -- Why: CLI result formatters are centralized so handlers can stay thin RPC glue. */
+import { chmodSync, lstatSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
 import type {
   BrowserProfileListResult,
   BrowserTabCurrentResult,
@@ -7,6 +11,11 @@ import type {
   BrowserTabProfileCloneResult,
   BrowserTabProfileShowResult,
   BrowserTabShowResult,
+  ComputerActionResult,
+  ComputerActionVerification,
+  ComputerListAppsResult,
+  ComputerListWindowsResult,
+  ComputerSnapshotResult,
   CliStatusResult,
   RuntimeRepoList,
   RuntimeRepoSearchRefs,
@@ -33,7 +42,7 @@ export function printResult<TResult>(
   formatter: (value: TResult) => string
 ): void {
   if (json) {
-    console.log(JSON.stringify(response, null, 2))
+    console.log(JSON.stringify(prepareCliJsonResult(response), null, 2))
     return
   }
   console.log(formatter(response.result))
@@ -298,3 +307,193 @@ export function formatTabProfileShow(result: BrowserTabProfileShowResult): strin
 export function formatTabProfileClone(result: BrowserTabProfileCloneResult): string {
   return `Cloned ${result.sourceBrowserPageId} to ${result.browserPageId} (${result.profileLabel ?? result.profileId ?? 'default'})`
 }
+
+export function formatGetAppState(result: ComputerSnapshotResult): string {
+  const app = result.snapshot.app
+  const bundle = app.bundleId ? `, ${app.bundleId}` : ''
+  const focused =
+    result.snapshot.focusedElementId === null ? 'none' : `#${result.snapshot.focusedElementId}`
+  const windowId =
+    result.snapshot.window.id === null || result.snapshot.window.id === undefined
+      ? ''
+      : ` id:${result.snapshot.window.id}`
+  const origin =
+    result.snapshot.window.x === null ||
+    result.snapshot.window.x === undefined ||
+    result.snapshot.window.y === null ||
+    result.snapshot.window.y === undefined
+      ? ''
+      : ` @ ${result.snapshot.window.x},${result.snapshot.window.y}`
+  const truncation = result.snapshot.truncation?.truncated
+    ? `  Truncated: yes (max nodes ${result.snapshot.truncation.maxNodes ?? 'unknown'}, max depth ${result.snapshot.truncation.maxDepth ?? 'unknown'})`
+    : '  Truncated: no'
+  return [
+    `${app.name} (pid ${app.pid}${bundle})`,
+    `  Window:${windowId} "${result.snapshot.window.title}" (${result.snapshot.window.width}x${result.snapshot.window.height}${origin})`,
+    `  Elements: ${result.snapshot.elementCount}  Focused: ${focused}  Coordinates: ${result.snapshot.coordinateSpace}`,
+    truncation,
+    `  ${formatComputerScreenshotStatus(result)}`,
+    '',
+    result.snapshot.treeText
+  ].join('\n')
+}
+
+function prepareCliJsonResult<TResult>(
+  response: RuntimeRpcSuccess<TResult>
+): RuntimeRpcSuccess<TResult> {
+  const record = response as RuntimeRpcSuccess<TResult> & {
+    result?: { screenshot?: { data?: unknown; format?: unknown; path?: unknown } | null }
+  }
+  const screenshot = record.result?.screenshot
+  if (!screenshot || typeof screenshot.data !== 'string' || screenshot.data.length === 0) {
+    return response
+  }
+  const extension = screenshot.format === 'png' ? 'png' : 'img'
+  const outputDir = computerScreenshotTempDir()
+  cleanupComputerScreenshots(outputDir)
+  const outputPath = join(outputDir, `${safeCliFileStem(response.id)}-screenshot.${extension}`)
+  writeFileSync(outputPath, Buffer.from(screenshot.data, 'base64'), { mode: 0o600 })
+  const expiresAt = new Date(Date.now() + COMPUTER_SCREENSHOT_TTL_MS).toISOString()
+  return {
+    ...response,
+    result: {
+      ...record.result,
+      screenshot: {
+        ...screenshot,
+        data: undefined,
+        path: outputPath,
+        dataOmitted: true,
+        expiresAt
+      }
+    }
+  } as RuntimeRpcSuccess<TResult>
+}
+
+const COMPUTER_SCREENSHOT_TTL_MS = 24 * 60 * 60 * 1000
+
+function computerScreenshotTempDir(): string {
+  const outputDir = join(tmpdir(), 'orca-computer-use')
+  mkdirSync(outputDir, { recursive: true, mode: 0o700 })
+  const stat = lstatSync(outputDir)
+  if (!stat.isDirectory() || stat.isSymbolicLink()) {
+    throw new Error(`Unsafe computer screenshot temp path: ${outputDir}`)
+  }
+  if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) {
+    throw new Error(`Computer screenshot temp path is not owned by the current user: ${outputDir}`)
+  }
+  chmodSync(outputDir, 0o700)
+  return outputDir
+}
+
+function cleanupComputerScreenshots(outputDir: string): void {
+  const cutoff = Date.now() - COMPUTER_SCREENSHOT_TTL_MS
+  for (const entry of readdirSync(outputDir)) {
+    if (!entry.endsWith('-screenshot.png') && !entry.endsWith('-screenshot.img')) {
+      continue
+    }
+    const path = join(outputDir, entry)
+    try {
+      if (statSync(path).mtimeMs < cutoff) {
+        rmSync(path, { force: true })
+      }
+    } catch {
+      // Best-effort cleanup only; formatting should not fail because a temp file raced.
+    }
+  }
+}
+
+function safeCliFileStem(value: string): string {
+  return value.replaceAll(/[^a-zA-Z0-9._-]/g, '_')
+}
+
+export function formatListApps(result: ComputerListAppsResult): string {
+  if (result.apps.length === 0) {
+    return 'No apps found.'
+  }
+  return result.apps
+    .map((app) => {
+      const bundle = app.bundleId ? `  ${app.bundleId}` : ''
+      return `${app.name}  pid:${app.pid}${bundle}`
+    })
+    .join('\n')
+}
+
+export function formatListWindows(result: ComputerListWindowsResult): string {
+  if (result.windows.length === 0) {
+    return `No windows found for ${result.app.name}.`
+  }
+  return result.windows
+    .map((window) => {
+      const id = window.id === null || window.id === undefined ? 'none' : String(window.id)
+      const origin =
+        window.x === null || window.x === undefined || window.y === null || window.y === undefined
+          ? ''
+          : ` @ ${window.x},${window.y}`
+      const screen =
+        window.screenIndex === null || window.screenIndex === undefined
+          ? ''
+          : ` screen:${window.screenIndex}`
+      const state = [
+        window.isMinimized ? 'minimized' : null,
+        window.isOffscreen ? 'offscreen' : null
+      ].filter(Boolean)
+      const stateText = state.length > 0 ? ` ${state.join(',')}` : ''
+      return `[${window.index}] id:${id} "${window.title}" (${window.width}x${window.height}${origin})${screen}${stateText}`
+    })
+    .join('\n')
+}
+
+export function formatComputerAction(verb: string, result: ComputerActionResult): string {
+  const path = result.action?.path ? ` via ${result.action.path}` : ''
+  const verification = formatActionVerification(result.action?.verification)
+  const app = shellQuote(result.snapshot.app.bundleId ?? result.snapshot.app.name)
+  const windowId =
+    result.snapshot.window.id === null || result.snapshot.window.id === undefined
+      ? ''
+      : ` --window-id ${result.snapshot.window.id}`
+  return `${formatActionVerb(verb)} completed${path}${verification}; ${result.snapshot.elementCount} elements in current window. Use \`orca computer get-app-state --app ${app}${windowId}\` to inspect.`
+}
+
+function formatActionVerification(verification: ComputerActionVerification | undefined): string {
+  if (!verification) {
+    return ''
+  }
+  if (verification.state === 'verified') {
+    return `, verified ${verification.property}`
+  }
+  return `, unverified (${verification.reason.replaceAll('_', ' ')})`
+}
+
+function formatComputerScreenshotStatus(result: ComputerSnapshotResult): string {
+  if (result.screenshotStatus.state === 'captured' && result.screenshot) {
+    const bytes = result.screenshot.data
+      ? `${Math.round(result.screenshot.data.length * 0.75)} bytes`
+      : `saved to ${result.screenshot.path ?? 'temporary file'}`
+    const engine = result.screenshotStatus.metadata?.engine
+    const detail = engine
+      ? `${result.screenshot.format}, ${bytes}, ${engine}`
+      : `${result.screenshot.format}, ${bytes}`
+    return `Screenshot captured (${detail})`
+  }
+  if (result.screenshotStatus.state === 'skipped') {
+    return 'Screenshot skipped (--no-screenshot)'
+  }
+  if (result.screenshotStatus.state === 'failed') {
+    return `Screenshot failed (${result.screenshotStatus.code}): ${result.screenshotStatus.message}`
+  }
+  return 'Screenshot was not captured'
+}
+
+function shellQuote(value: string): string {
+  if (/^[a-zA-Z0-9._:/@-]+$/.test(value)) {
+    return value
+  }
+  return `'${value.replaceAll("'", "'\\''")}'`
+}
+
+function formatActionVerb(verb: string): string {
+  return verb
+    .split('-')
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join(' ')
+}

src/cli/handlers/computer.test.ts

@@ -0,0 +1,435 @@
+/* eslint-disable max-lines -- Why: computer CLI coverage shares one mocked runtime setup across command contracts. */
+import path from 'path'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const callMock = vi.fn()
+
+vi.mock('../runtime-client', () => {
+  class RuntimeClient {
+    call = callMock
+    getCliStatus = vi.fn()
+    openOrca = vi.fn()
+  }
+
+  class RuntimeClientError extends Error {
+    readonly code: string
+
+    constructor(code: string, message: string) {
+      super(message)
+      this.code = code
+    }
+  }
+
+  class RuntimeRpcFailureError extends RuntimeClientError {
+    readonly response: unknown
+
+    constructor(response: unknown) {
+      super('runtime_error', 'runtime_error')
+      this.response = response
+    }
+  }
+
+  return {
+    RuntimeClient,
+    RuntimeClientError,
+    RuntimeRpcFailureError
+  }
+})
+
+import { main } from '../index'
+import { buildWorktree, okFixture, queueFixtures, worktreeListFixture } from '../test-fixtures'
+
+describe('orca computer CLI handlers', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks()
+    callMock.mockReset()
+    vi.spyOn(console, 'log').mockImplementation(() => {})
+  })
+
+  it('prints group help with all computer subcommands', async () => {
+    await main(['computer', '--help'], '/tmp/repo')
+
+    const output = vi.mocked(console.log).mock.calls[0][0]
+    expect(output).toContain('get-app-state')
+    expect(output).toContain('hotkey')
+    expect(output).toContain('capabilities')
+    expect(output).toContain('list-apps')
+    expect(output).toContain('list-windows')
+    expect(output).toContain('paste-text')
+    expect(output).toContain('permissions')
+    expect(output).toContain('set-value')
+  })
+
+  it('passes list-apps through with a resolved worktree', async () => {
+    queueFixtures(
+      callMock,
+      worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+      okFixture('req_apps', { apps: [] })
+    )
+
+    await main(['computer', 'list-apps', '--json'], '/tmp/repo/src')
+
+    expect(callMock).toHaveBeenNthCalledWith(1, 'worktree.list', { limit: 10_000 })
+    expect(callMock).toHaveBeenNthCalledWith(2, 'computer.listApps', {
+      worktree: `path:${path.resolve('/tmp/repo')}`
+    })
+  })
+
+  it('prints provider capabilities without resolving a worktree', async () => {
+    queueFixtures(
+      callMock,
+      okFixture('req_capabilities', {
+        platform: 'darwin',
+        provider: 'orca-computer-use-macos',
+        providerVersion: '1.0.0',
+        protocolVersion: 1,
+        supports: {
+          apps: { list: true, bundleIds: true, pids: true },
+          windows: { list: true, targetById: true, targetByIndex: true },
+          observation: { screenshot: true, elementFrames: true, annotatedScreenshot: false },
+          actions: { click: true, setValue: true, pasteText: true },
+          surfaces: {}
+        }
+      })
+    )
+
+    await main(['computer', 'capabilities'], '/tmp/repo/src')
+
+    expect(callMock).toHaveBeenCalledTimes(1)
+    expect(callMock).toHaveBeenCalledWith('computer.capabilities', {})
+    expect(vi.mocked(console.log).mock.calls[0][0]).toContain('orca-computer-use-macos')
+  })
+
+  it('opens computer permission setup without resolving a worktree', async () => {
+    queueFixtures(
+      callMock,
+      okFixture('req_permissions', {
+        platform: 'darwin',
+        helperAppPath: '/Applications/Orca Computer Use.app',
+        openedSettings: false,
+        launchedHelper: true
+      })
+    )
+
+    await main(['computer', 'permissions'], '/tmp/repo/src')
+
+    expect(callMock).toHaveBeenCalledTimes(1)
+    expect(callMock).toHaveBeenCalledWith('computer.permissions', {})
+    const output = vi.mocked(console.log).mock.calls[0][0]
+    expect(output).toContain('Opened Orca Computer Use permission setup')
+    expect(output).toContain('/Applications/Orca Computer Use.app')
+  })
+
+  it('passes get-app-state target and observe flags', async () => {
+    queueFixtures(
+      callMock,
+      worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+      okFixture('req_state', sampleSnapshot())
+    )
+
+    await main(
+      [
+        'computer',
+        'get-app-state',
+        '--app',
+        'Finder',
+        '--no-screenshot',
+        '--restore-window',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenNthCalledWith(2, 'computer.getAppState', {
+      app: 'Finder',
+      worktree: `path:${path.resolve('/tmp/repo')}`,
+      noScreenshot: true,
+      restoreWindow: true
+    })
+  })
+
+  it('passes list-windows target and formats window IDs', async () => {
+    queueFixtures(
+      callMock,
+      worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+      okFixture('req_windows', {
+        app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+        windows: [
+          {
+            app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+            index: 0,
+            id: 42,
+            title: 'Recents',
+            x: 10,
+            y: 20,
+            width: 800,
+            height: 600,
+            isMinimized: false,
+            isOffscreen: false,
+            screenIndex: 0
+          }
+        ]
+      })
+    )
+
+    await main(['computer', 'list-windows', '--app', 'Finder'], '/tmp/repo/src')
+
+    expect(callMock).toHaveBeenNthCalledWith(2, 'computer.listWindows', {
+      app: 'Finder',
+      worktree: `path:${path.resolve('/tmp/repo')}`
+    })
+    const output = vi.mocked(console.log).mock.calls[0][0]
+    expect(output).toContain('[0] id:42 "Recents"')
+  })
+
+  it('does not resolve worktree when --session is explicit', async () => {
+    queueFixtures(callMock, okFixture('req_click', sampleSnapshot()))
+
+    await main(
+      [
+        'computer',
+        'click',
+        '--session',
+        'manual',
+        '--app',
+        'Finder',
+        '--element-index',
+        '3',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenCalledTimes(1)
+    expect(callMock).toHaveBeenCalledWith('computer.click', {
+      session: 'manual',
+      app: 'Finder',
+      elementIndex: 3,
+      x: undefined,
+      y: undefined,
+      clickCount: undefined,
+      mouseButton: undefined,
+      noScreenshot: undefined
+    })
+  })
+
+  it('maps action command flags to RPC payloads', async () => {
+    queueFixtures(
+      callMock,
+      worktreeListFixture([buildWorktree('/tmp/repo', 'feature')]),
+      okFixture('req_drag', sampleSnapshot())
+    )
+
+    await main(
+      [
+        'computer',
+        'drag',
+        '--app',
+        'Finder',
+        '--from-x',
+        '1',
+        '--from-y',
+        '2',
+        '--to-x',
+        '3',
+        '--to-y',
+        '4',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenNthCalledWith(2, 'computer.drag', {
+      app: 'Finder',
+      worktree: `path:${path.resolve('/tmp/repo')}`,
+      fromElementIndex: undefined,
+      toElementIndex: undefined,
+      fromX: 1,
+      fromY: 2,
+      toX: 3,
+      toY: 4,
+      noScreenshot: undefined
+    })
+  })
+
+  it('maps coordinate scroll flags to RPC payloads', async () => {
+    queueFixtures(callMock, okFixture('req_scroll', sampleSnapshot()))
+
+    await main(
+      [
+        'computer',
+        'scroll',
+        '--session',
+        'manual',
+        '--app',
+        'Finder',
+        '--x',
+        '10',
+        '--y',
+        '20',
+        '--direction',
+        'down',
+        '--pages',
+        '0.5',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenCalledWith('computer.scroll', {
+      session: 'manual',
+      app: 'Finder',
+      elementIndex: undefined,
+      x: 10,
+      y: 20,
+      direction: 'down',
+      pages: 0.5,
+      noScreenshot: undefined
+    })
+  })
+
+  it('maps hotkey and paste-text command flags to RPC payloads', async () => {
+    queueFixtures(callMock, okFixture('req_hotkey', sampleSnapshot()))
+    await main(
+      [
+        'computer',
+        'hotkey',
+        '--session',
+        'manual',
+        '--app',
+        'Finder',
+        '--key',
+        'CmdOrCtrl+L',
+        '--no-screenshot',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenCalledWith('computer.hotkey', {
+      session: 'manual',
+      app: 'Finder',
+      key: 'CmdOrCtrl+L',
+      noScreenshot: true
+    })
+
+    callMock.mockReset()
+    vi.mocked(console.log).mockClear()
+    queueFixtures(callMock, okFixture('req_paste', sampleSnapshot()))
+    await main(
+      [
+        'computer',
+        'paste-text',
+        '--session',
+        'manual',
+        '--app',
+        'Finder',
+        '--text',
+        'hello world',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenCalledWith('computer.pasteText', {
+      session: 'manual',
+      app: 'Finder',
+      text: 'hello world',
+      noScreenshot: undefined
+    })
+  })
+
+  it('formats get-app-state without printing screenshot bytes in pretty mode', async () => {
+    queueFixtures(callMock, okFixture('req_state', sampleSnapshot()))
+
+    await main(['computer', 'get-app-state', '--session', 'manual', '--app', 'Finder'], '/tmp/repo')
+
+    const output = vi.mocked(console.log).mock.calls[0][0]
+    expect(output).toContain('Finder (pid 100, com.apple.finder)')
+    expect(output).toContain('App=com.apple.finder')
+    expect(output).not.toContain('base64-data')
+  })
+
+  it('omits screenshot bytes from JSON output and writes a screenshot path', async () => {
+    queueFixtures(callMock, okFixture('req_state', sampleSnapshot()))
+
+    await main(
+      ['computer', 'get-app-state', '--session', 'manual', '--app', 'Finder', '--json'],
+      '/tmp/repo'
+    )
+
+    const output = vi.mocked(console.log).mock.calls[0][0]
+    expect(output).not.toContain('base64-data')
+    const parsed = JSON.parse(output)
+    expect(parsed.result.screenshot).toMatchObject({
+      dataOmitted: true,
+      format: 'png',
+      expiresAt: expect.any(String),
+      path: expect.stringContaining('orca-computer-use/req_state-screenshot.png')
+    })
+  })
+
+  it('shows coordinate space and truncation in pretty state output', async () => {
+    queueFixtures(callMock, okFixture('req_state', sampleSnapshot()))
+
+    await main(['computer', 'get-app-state', '--session', 'manual', '--app', 'Finder'], '/tmp/repo')
+
+    const output = vi.mocked(console.log).mock.calls[0][0]
+    expect(output).toContain('Coordinates: window')
+    expect(output).toContain('Truncated: no')
+  })
+
+  it('passes get-app-state window target flags through', async () => {
+    queueFixtures(callMock, okFixture('req_snapshot', sampleSnapshot()))
+
+    await main(
+      [
+        'computer',
+        'get-app-state',
+        '--session',
+        'manual',
+        '--app',
+        'Finder',
+        '--window-id',
+        '42',
+        '--window-index',
+        '0',
+        '--no-screenshot',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenCalledWith('computer.getAppState', {
+      session: 'manual',
+      app: 'Finder',
+      noScreenshot: true,
+      restoreWindow: undefined,
+      windowId: 42,
+      windowIndex: 0
+    })
+  })
+})
+
+function sampleSnapshot() {
+  return {
+    snapshot: {
+      id: 'snap-test',
+      app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+      window: { title: 'Finder', width: 800, height: 600 },
+      coordinateSpace: 'window',
+      truncation: { truncated: false, maxNodes: 1200, maxDepth: 64, maxDepthReached: false },
+      treeText: 'App=com.apple.finder (pid 100)\n0 standard window Finder',
+      elementCount: 1,
+      focusedElementId: 0
+    },
+    screenshot: {
+      data: 'base64-data',
+      format: 'png',
+      width: 800,
+      height: 600,
+      scale: 1
+    },
+    screenshotStatus: { state: 'captured' }
+  }
+}

src/cli/handlers/computer.ts

@@ -0,0 +1,278 @@
+import type {
+  ComputerActionResult,
+  ComputerListAppsResult,
+  ComputerListWindowsResult,
+  ComputerProviderCapabilities,
+  ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import type { CommandHandler } from '../dispatch'
+import {
+  getOptionalNumberFlag,
+  getOptionalNonNegativeIntegerFlag,
+  getOptionalPositiveIntegerFlag,
+  getOptionalStringFlag,
+  getRequiredStringFlagAllowingEmpty,
+  getRequiredStringFlag
+} from '../flags'
+import {
+  formatComputerAction,
+  formatGetAppState,
+  formatListApps,
+  formatListWindows,
+  printResult
+} from '../format'
+import { RuntimeClientError } from '../runtime-client'
+import { getComputerCommandTarget } from '../selectors'
+
+export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
+  'computer capabilities': async ({ client, json }) => {
+    const result = await client.call<ComputerProviderCapabilities>('computer.capabilities', {})
+    printResult(result, json, formatComputerCapabilities)
+  },
+  'computer list-apps': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerListAppsResult>('computer.listApps', {
+      worktree: target.worktree
+    })
+    printResult(result, json, formatListApps)
+  },
+  'computer permissions': async ({ client, json }) => {
+    const result = await client.call<{
+      platform: NodeJS.Platform
+      helperAppPath: string | null
+      openedSettings: boolean
+      launchedHelper: boolean
+      permissions?: { id: string; status: string }[]
+      nextStep?: string | null
+    }>('computer.permissions', {})
+    printResult(result, json, (value) => {
+      if (value.platform !== 'darwin') {
+        return 'Computer-use permission setup is only required on macOS.'
+      }
+      const firstLine = value.launchedHelper
+        ? 'Opened Orca Computer Use permission setup.'
+        : 'Computer Use permissions checked.'
+      return [
+        firstLine,
+        `Helper app: ${value.helperAppPath}`,
+        `Permissions: ${value.permissions?.map((permission) => `${permission.id}=${permission.status}`).join(', ') ?? 'unknown'}`,
+        value.nextStep
+          ? `Next: ${value.nextStep}`
+          : 'Computer Use permissions are already granted.',
+        value.launchedHelper
+          ? 'Use the Allow buttons or drag "Orca Computer Use" into the macOS permission list.'
+          : null
+      ]
+        .filter((line) => line !== null)
+        .join('\n')
+    })
+  },
+  'computer list-windows': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerListWindowsResult>('computer.listWindows', target)
+    printResult(result, json, formatListWindows)
+  },
+  'computer get-app-state': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerSnapshotResult>('computer.getAppState', {
+      ...target,
+      noScreenshot: flags.has('no-screenshot') ? true : undefined,
+      restoreWindow: flags.has('restore-window') ? true : undefined,
+      windowId: getOptionalNumberFlag(flags, 'window-id'),
+      windowIndex: getOptionalNonNegativeIntegerFlag(flags, 'window-index')
+    })
+    printResult(result, json, formatGetAppState)
+  },
+  'computer click': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.click', {
+      ...target,
+      elementIndex: getOptionalNonNegativeIntegerFlag(flags, 'element-index'),
+      x: getOptionalNumberFlag(flags, 'x'),
+      y: getOptionalNumberFlag(flags, 'y'),
+      clickCount: getOptionalPositiveIntegerFlag(flags, 'click-count'),
+      mouseButton: getOptionalStringFlag(flags, 'mouse-button'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('click', value))
+  },
+  'computer perform-secondary-action': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.performSecondaryAction', {
+      ...target,
+      elementIndex: getRequiredNonNegativeIntegerFlag(flags, 'element-index'),
+      action: getRequiredStringFlag(flags, 'action'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('perform-secondary-action', value))
+  },
+  'computer scroll': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.scroll', {
+      ...target,
+      elementIndex: getOptionalNonNegativeIntegerFlag(flags, 'element-index'),
+      x: getOptionalNumberFlag(flags, 'x'),
+      y: getOptionalNumberFlag(flags, 'y'),
+      direction: getRequiredStringFlag(flags, 'direction'),
+      pages: getOptionalPositiveNumberFlag(flags, 'pages'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('scroll', value))
+  },
+  'computer drag': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.drag', {
+      ...target,
+      fromElementIndex: getOptionalNonNegativeIntegerFlag(flags, 'from-element-index'),
+      toElementIndex: getOptionalNonNegativeIntegerFlag(flags, 'to-element-index'),
+      fromX: getOptionalNumberFlag(flags, 'from-x'),
+      fromY: getOptionalNumberFlag(flags, 'from-y'),
+      toX: getOptionalNumberFlag(flags, 'to-x'),
+      toY: getOptionalNumberFlag(flags, 'to-y'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('drag', value))
+  },
+  'computer type-text': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.typeText', {
+      ...target,
+      text: await getTextPayload(flags, 'text'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('type-text', value))
+  },
+  'computer press-key': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.pressKey', {
+      ...target,
+      key: getRequiredStringFlag(flags, 'key'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('press-key', value))
+  },
+  'computer hotkey': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.hotkey', {
+      ...target,
+      key: getRequiredStringFlag(flags, 'key'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('hotkey', value))
+  },
+  'computer paste-text': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.pasteText', {
+      ...target,
+      text: await getTextPayload(flags, 'text'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('paste-text', value))
+  },
+  'computer set-value': async ({ flags, client, cwd, json }) => {
+    const target = await getComputerCommandTarget(flags, cwd, client)
+    const result = await client.call<ComputerActionResult>('computer.setValue', {
+      ...target,
+      elementIndex: getRequiredNonNegativeIntegerFlag(flags, 'element-index'),
+      value: await getTextPayload(flags, 'value'),
+      ...getComputerActionObserveFlags(flags)
+    })
+    printResult(result, json, (value) => formatComputerAction('set-value', value))
+  }
+}
+
+async function getTextPayload(
+  flags: Map<string, string | boolean>,
+  name: 'text' | 'value'
+): Promise<string> {
+  const stdinFlag = `${name}-stdin`
+  if (flags.has(stdinFlag)) {
+    if (flags.has(name)) {
+      throw new RuntimeClientError(
+        'invalid_argument',
+        `Use either --${name} or --${stdinFlag}, not both`
+      )
+    }
+    return await readStdin()
+  }
+  return name === 'value'
+    ? getRequiredStringFlagAllowingEmpty(flags, name)
+    : getRequiredStringFlag(flags, name)
+}
+
+async function readStdin(): Promise<string> {
+  if (process.stdin.isTTY) {
+    throw new RuntimeClientError('invalid_argument', 'stdin payload requested but stdin is a TTY')
+  }
+  const chunks: Buffer[] = []
+  for await (const chunk of process.stdin) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)))
+  }
+  return Buffer.concat(chunks).toString('utf8')
+}
+
+function getOptionalPositiveNumberFlag(
+  flags: Map<string, string | boolean>,
+  name: string
+): number | undefined {
+  const value = getOptionalNumberFlag(flags, name)
+  if (value === undefined) {
+    return undefined
+  }
+  if (value <= 0) {
+    throw new RuntimeClientError('invalid_argument', `Invalid positive number for --${name}`)
+  }
+  return value
+}
+
+function formatComputerCapabilities(value: ComputerProviderCapabilities): string {
+  return [
+    `${value.provider} (${value.platform}, protocol ${value.protocolVersion})`,
+    `  Apps: list=${value.supports.apps.list} bundleIds=${value.supports.apps.bundleIds} pids=${value.supports.apps.pids}`,
+    `  Windows: list=${value.supports.windows.list} targetById=${value.supports.windows.targetById} targetByIndex=${value.supports.windows.targetByIndex}`,
+    `  Observation: screenshot=${value.supports.observation.screenshot} elementFrames=${value.supports.observation.elementFrames} annotatedScreenshot=${value.supports.observation.annotatedScreenshot}`,
+    `  Actions: ${Object.entries(value.supports.actions)
+      .filter(([, enabled]) => enabled)
+      .map(([name]) => name)
+      .join(', ')}`
+  ].join('\n')
+}
+
+function getRequiredNonNegativeIntegerFlag(
+  flags: Map<string, string | boolean>,
+  name: string
+): number {
+  const value = getOptionalNonNegativeIntegerFlag(flags, name)
+  if (value === undefined) {
+    throw new RuntimeClientError('invalid_argument', `Missing required --${name}`)
+  }
+  return value
+}
+
+function getComputerActionObserveFlags(flags: Map<string, string | boolean>): {
+  noScreenshot?: boolean
+  restoreWindow?: boolean
+  windowId?: number
+  windowIndex?: number
+} {
+  const observeFlags: {
+    noScreenshot?: boolean
+    restoreWindow?: boolean
+    windowId?: number
+    windowIndex?: number
+  } = {
+    noScreenshot: flags.has('no-screenshot') ? true : undefined
+  }
+  if (flags.has('restore-window')) {
+    observeFlags.restoreWindow = true
+  }
+  const windowId = getOptionalNumberFlag(flags, 'window-id')
+  if (windowId !== undefined) {
+    observeFlags.windowId = windowId
+  }
+  const windowIndex = getOptionalNonNegativeIntegerFlag(flags, 'window-index')
+  if (windowIndex !== undefined) {
+    observeFlags.windowIndex = windowIndex
+  }
+  return observeFlags
+}

src/cli/help.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-lines -- Why: root and generated command help text live together so CLI discovery stays greppable. */
 import type { CommandSpec } from './args'
 import { findCommandSpec, isCommandGroup, supportsBrowserPageFlag } from './args'
 
@@ -56,6 +57,21 @@ Orchestration:
   orchestration gate-list   List decision gates
   orchestration reset       Reset orchestration state
 
+Computer Use:
+  computer permissions      Open the macOS permission setup for computer-use
+  computer list-apps        List running apps available to computer-use
+  computer list-windows     List visible windows for a target app
+  computer get-app-state    Capture a compact accessibility snapshot of an app
+  computer click            Click an app element or window coordinate
+  computer perform-secondary-action Run an advertised accessibility action
+  computer scroll           Scroll an app element
+  computer drag             Drag between app elements or window coordinates
+  computer type-text        Type literal text at the current app focus
+  computer press-key        Press a key using xdotool-style syntax
+  computer hotkey           Press a shortcut combination such as CmdOrCtrl+A
+  computer paste-text       Paste text through the native clipboard path
+  computer set-value        Set the value of a settable app element
+
 Browser Automation:
   tab create                Create a new browser tab (navigates to --url)
   tab list                  List open browser tabs
@@ -290,27 +306,48 @@ export function formatFlagHelp(flag: string): string {
     command: '--command <text>       Command to run in the terminal on startup',
     comment: '--comment <text>       Comment stored in Orca metadata',
     cursor: '--cursor <n>           Line cursor from a previous read (returns only new output)',
-    direction: '--direction <dir>      Direction: horizontal|vertical (split) or up|down (scroll)',
+    action: '--action <name>       Secondary accessibility action name',
+    app: '--app <app>            App name, bundle ID, or pid:N',
+    direction:
+      '--direction <dir>      Direction: up|down|left|right for scroll, horizontal|vertical for split',
     'display-name': '--display-name <name>  Override the Orca display name',
+    'element-index': '--element-index <n>   Element index from get-app-state',
     title: '--title <text>         Custom title for the terminal tab (omit to reset)',
     enter: '--enter                Append Enter after sending text',
     force: '--force                Force worktree removal when supported',
     for: '--for exit|tui-idle    Wait condition to satisfy',
+    'from-element-index': '--from-element-index <n> Source element index from get-app-state',
+    'from-x': '--from-x <x>           Source window-local x coordinate',
+    'from-y': '--from-y <y>           Source window-local y coordinate',
     help: '--help                 Show this help message',
     interrupt: '--interrupt            Send as an interrupt-style input when supported',
     issue: '--issue <number|null>  Linked GitHub issue number',
     json: '--json                 Emit machine-readable JSON',
+    key: '--key <key>            Key or combo to press, e.g. Escape or CmdOrCtrl+L',
     limit: '--limit <n>            Maximum number of rows to return',
+    'mouse-button': '--mouse-button <btn>   Mouse button: left, right, or middle',
     name: '--name <name>          Name for the new worktree',
+    'no-screenshot': '--no-screenshot       Skip screenshot capture after the operation',
+    pages: '--pages <n>           Number of scroll pages',
     path: '--path <path>          Filesystem path to the repo',
     query: '--query <text>        Search text for matching refs',
     ref: '--ref <ref>            Base ref to persist for the repo',
     repo: '--repo <selector>      Repo selector such as id:<id>, name:<name>, or path:<path>',
+    'restore-window':
+      '--restore-window     Bring the target app/window forward before the operation',
+    session: '--session <id>        Snapshot namespace for a related computer-use workflow',
     terminal: '--terminal <handle>  Runtime-issued terminal handle',
-    text: '--text <text>          Text to send to the terminal',
+    text: '--text <text>          Text payload to send or type',
+    'text-stdin': '--text-stdin          Read text payload from stdin',
     'timeout-ms': '--timeout-ms <ms>     Maximum wait time before timing out',
+    'to-element-index': '--to-element-index <n> Destination element index from get-app-state',
+    'to-x': '--to-x <x>             Destination window-local x coordinate',
+    'to-y': '--to-y <y>             Destination window-local y coordinate',
     worktree:
       '--worktree <selector>  Worktree selector such as id:<id>, branch:<branch>, issue:<number>, path:<path>, or active/current',
+    'value-stdin': '--value-stdin         Read set-value payload from stdin',
+    'window-id': '--window-id <id>      Target a window id from list-windows',
+    'window-index': '--window-index <n>   Target a window index from list-windows',
     // Browser automation flags
     element: '--element <ref>        Element ref from snapshot (e.g. e3)',
     url: '--url <url>            URL to navigate to',

src/cli/selectors.ts

@@ -1,5 +1,5 @@
 import { isAbsolute, relative, resolve as resolvePath } from 'path'
-import type { RuntimeWorktreeListResult } from '../shared/runtime-types'
+import type { ComputerAppQuery, RuntimeWorktreeListResult } from '../shared/runtime-types'
 import type { RuntimeClient } from './runtime-client'
 import { RuntimeClientError } from './runtime-client'
 import { getOptionalStringFlag, getRequiredStringFlag } from './flags'
@@ -9,6 +9,12 @@ export type BrowserCliTarget = {
   page?: string
 }
 
+export type ComputerCliTarget = {
+  session?: string
+  worktree?: string
+  app?: ComputerAppQuery
+}
+
 export function buildCurrentWorktreeSelector(cwd: string): string {
   return `path:${resolvePath(cwd)}`
 }
@@ -150,3 +156,19 @@ export async function getBrowserCommandTarget(
     worktree: normalizeWorktreeSelector(explicitWorktree, cwd)
   }
 }
+
+export async function getComputerCommandTarget(
+  flags: Map<string, string | boolean>,
+  cwd: string,
+  client: RuntimeClient
+): Promise<ComputerCliTarget> {
+  const app = getOptionalStringFlag(flags, 'app')
+  const session = getOptionalStringFlag(flags, 'session')
+  if (session) {
+    return { session, app }
+  }
+  return {
+    app,
+    worktree: await getBrowserWorktreeSelector(flags, cwd, client)
+  }
+}

src/cli/specs/computer.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest'
+
+import { COMPUTER_COMMAND_SPECS } from './computer'
+
+describe('computer command specs', () => {
+  it('allows explicit window targeting on action commands', () => {
+    const actionSpecs = COMPUTER_COMMAND_SPECS.filter((spec) =>
+      [
+        'computer click',
+        'computer drag',
+        'computer hotkey',
+        'computer paste-text',
+        'computer perform-secondary-action',
+        'computer press-key',
+        'computer scroll',
+        'computer set-value',
+        'computer type-text'
+      ].includes(spec.path.join(' '))
+    )
+
+    expect(actionSpecs).not.toHaveLength(0)
+    for (const spec of actionSpecs) {
+      expect(spec.allowedFlags).toEqual(expect.arrayContaining(['window-id', 'window-index']))
+    }
+  })
+})

src/cli/specs/computer.ts

@@ -0,0 +1,124 @@
+import type { CommandSpec } from '../args'
+import { GLOBAL_FLAGS } from '../args'
+
+const COMPUTER_FLAGS = [...GLOBAL_FLAGS, 'worktree', 'session', 'app']
+const COMPUTER_WINDOW_TARGET_FLAGS = ['window-id', 'window-index']
+const COMPUTER_ACTION_FLAGS = [
+  ...COMPUTER_FLAGS,
+  ...COMPUTER_WINDOW_TARGET_FLAGS,
+  'restore-window',
+  'no-screenshot'
+]
+
+export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
+  {
+    path: ['computer', 'capabilities'],
+    summary: 'Show computer-use provider capabilities',
+    usage: 'orca computer capabilities [--json]',
+    allowedFlags: [...GLOBAL_FLAGS]
+  },
+  {
+    path: ['computer', 'list-apps'],
+    summary: 'List running apps available to computer-use',
+    usage: 'orca computer list-apps [--worktree <selector>] [--json]',
+    allowedFlags: [...GLOBAL_FLAGS, 'worktree']
+  },
+  {
+    path: ['computer', 'permissions'],
+    summary: 'Open computer-use permission setup',
+    usage: 'orca computer permissions [--json]',
+    allowedFlags: [...GLOBAL_FLAGS]
+  },
+  {
+    path: ['computer', 'list-windows'],
+    summary: 'List windows for an app available to computer-use',
+    usage:
+      'orca computer list-windows --app <name|bundle|pid:N> [--worktree <selector> | --session <id>] [--json]',
+    allowedFlags: COMPUTER_FLAGS
+  },
+  {
+    path: ['computer', 'get-app-state'],
+    summary: 'Capture a compact accessibility snapshot of an app',
+    usage:
+      'orca computer get-app-state --app <name|bundle|pid:N> [--window-id <id> | --window-index <n>] [--worktree <selector> | --session <id>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, ...COMPUTER_WINDOW_TARGET_FLAGS, 'restore-window']
+  },
+  {
+    path: ['computer', 'click'],
+    summary: 'Click an app element or window coordinate',
+    usage:
+      'orca computer click --app <app> (--element-index <n> | --x <x> --y <y>) [--window-id <id> | --window-index <n>] [--click-count <n>] [--mouse-button <left|right|middle>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [
+      ...COMPUTER_ACTION_FLAGS,
+      'element-index',
+      'x',
+      'y',
+      'click-count',
+      'mouse-button'
+    ]
+  },
+  {
+    path: ['computer', 'perform-secondary-action'],
+    summary: 'Perform an advertised secondary accessibility action',
+    usage:
+      'orca computer perform-secondary-action --app <app> --element-index <n> --action <name> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'action']
+  },
+  {
+    path: ['computer', 'scroll'],
+    summary: 'Scroll an app element or window coordinate',
+    usage:
+      'orca computer scroll --app <app> (--element-index <n> | --x <x> --y <y>) --direction <up|down|left|right> [--window-id <id> | --window-index <n>] [--pages <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'x', 'y', 'direction', 'pages']
+  },
+  {
+    path: ['computer', 'drag'],
+    summary: 'Drag between app elements or window coordinates',
+    usage:
+      'orca computer drag --app <app> (--from-element-index <n> --to-element-index <n> | --from-x <x> --from-y <y> --to-x <x> --to-y <y>) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [
+      ...COMPUTER_ACTION_FLAGS,
+      'from-element-index',
+      'to-element-index',
+      'from-x',
+      'from-y',
+      'to-x',
+      'to-y'
+    ]
+  },
+  {
+    path: ['computer', 'type-text'],
+    summary: 'Type literal text at the current app focus',
+    usage:
+      'orca computer type-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
+  },
+  {
+    path: ['computer', 'press-key'],
+    summary: 'Press a key using xdotool-style syntax',
+    usage:
+      'orca computer press-key --app <app> --key <key> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
+  },
+  {
+    path: ['computer', 'hotkey'],
+    summary: 'Press a platform-aware key combination',
+    usage:
+      'orca computer hotkey --app <app> --key <key-combo> [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'key']
+  },
+  {
+    path: ['computer', 'paste-text'],
+    summary: 'Paste exact text at the current app focus',
+    usage:
+      'orca computer paste-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
+  },
+  {
+    path: ['computer', 'set-value'],
+    summary: 'Set the value of a settable app element',
+    usage:
+      'orca computer set-value --app <app> --element-index <n> (--value <text> | --value-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--json]',
+    allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'value', 'value-stdin']
+  }
+]

src/cli/specs/index.ts

@@ -3,10 +3,12 @@ import { BROWSER_ADVANCED_COMMAND_SPECS } from './browser-advanced'
 import { BROWSER_BASIC_COMMAND_SPECS } from './browser-basic'
 import { CORE_COMMAND_SPECS } from './core'
 import { ORCHESTRATION_COMMAND_SPECS } from './orchestration'
+import { COMPUTER_COMMAND_SPECS } from './computer'
 
 export const COMMAND_SPECS: CommandSpec[] = [
   ...CORE_COMMAND_SPECS,
   ...BROWSER_BASIC_COMMAND_SPECS,
   ...BROWSER_ADVANCED_COMMAND_SPECS,
-  ...ORCHESTRATION_COMMAND_SPECS
+  ...ORCHESTRATION_COMMAND_SPECS,
+  ...COMPUTER_COMMAND_SPECS
 ]

src/main/computer/desktop-script-provider-client.test.ts

@@ -0,0 +1,453 @@
+/* eslint-disable max-lines -- Why: desktop provider contract coverage shares one mocked bridge harness. */
+import { execFile } from 'child_process'
+import { readFileSync } from 'fs'
+import { describe, expect, it, vi } from 'vitest'
+import { DesktopScriptProviderClient } from './desktop-script-provider-client'
+
+vi.mock('child_process', () => ({
+  execFile: vi.fn()
+}))
+
+describe('DesktopScriptProviderClient', () => {
+  it('normalizes list-apps responses', async () => {
+    mockBridgeResponse({
+      ok: true,
+      apps: [{ name: 'Notepad', bundleIdentifier: 'notepad', pid: 42 }]
+    })
+
+    const client = new DesktopScriptProviderClient('windows', '/tmp/runtime.ps1')
+
+    await expect(client.listApps()).resolves.toEqual({
+      apps: [
+        {
+          name: 'Notepad',
+          bundleId: 'notepad',
+          pid: 42,
+          isRunning: true,
+          lastUsedAt: null,
+          useCount: null
+        }
+      ]
+    })
+  })
+
+  it('normalizes snapshots and remembers elements for follow-up actions', async () => {
+    mockBridgeResponse({
+      ok: true,
+      snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+    })
+    mockBridgeResponse({
+      ok: true,
+      snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+    })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    const initial = await client.snapshot({ app: 'Text Editor' })
+    expect(initial.snapshot.elementCount).toBe(1)
+    expect(initial.snapshot.window).toMatchObject({
+      title: 'Text Editor',
+      id: 99,
+      x: 10,
+      y: 20,
+      width: 300,
+      height: 200
+    })
+    expect(initial.screenshotStatus).toEqual({
+      state: 'captured',
+      metadata: { engine: 'unknown', windowId: 99 }
+    })
+    expect(initial.snapshot.treeText).toContain('initial')
+    expect(publicSnapshotKeys(initial.snapshot)).toEqual([
+      'app',
+      'coordinateSpace',
+      'elementCount',
+      'focusedElementId',
+      'id',
+      'treeText',
+      'truncation',
+      'window'
+    ])
+
+    await client.action('setValue', {
+      app: 'Text Editor',
+      elementIndex: 0,
+      value: 'changed',
+      noScreenshot: true
+    })
+
+    const secondCall = vi.mocked(execFile).mock.calls[1]
+    const operationPath = secondCall[1]?.at(-1)
+    expect(typeof operationPath).toBe('string')
+  })
+
+  it('targets cached elements by session and explicit window id', async () => {
+    mockBridgeResponse({
+      ok: true,
+      snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+    })
+    mockBridgeResponse(
+      {
+        ok: true,
+        snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+      },
+      (operation) => {
+        expect(operation).toMatchObject({
+          tool: 'drag',
+          app: 'Text Editor',
+          windowId: 99,
+          windowIndex: 0,
+          fromElement: expect.objectContaining({ index: 0 }),
+          toElement: expect.objectContaining({ index: 0 })
+        })
+        expect(operation.from_x).toBeUndefined()
+        expect(operation.to_x).toBeUndefined()
+        expect(operation.windowBounds).toBeNull()
+      }
+    )
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await client.snapshot({ app: 'Text Editor', session: 'agent-a', windowId: 99 })
+    await client.action('drag', {
+      app: 'Text Editor',
+      session: 'agent-a',
+      windowId: 99,
+      windowIndex: 0,
+      fromElementIndex: 0,
+      toElementIndex: 0
+    })
+  })
+
+  it('forwards restore-window to desktop providers', async () => {
+    mockBridgeResponse(
+      {
+        ok: true,
+        snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+      },
+      (operation) => {
+        expect(operation).toMatchObject({
+          tool: 'get_app_state',
+          restoreWindow: true
+        })
+      }
+    )
+    mockBridgeResponse(
+      {
+        ok: true,
+        snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+      },
+      (operation) => {
+        expect(operation).toMatchObject({
+          tool: 'set_value',
+          restoreWindow: true
+        })
+      }
+    )
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await client.snapshot({ app: 'Text Editor', restoreWindow: true })
+    const result = await client.action('setValue', {
+      app: 'Text Editor',
+      elementIndex: 0,
+      value: 'changed',
+      restoreWindow: true,
+      noScreenshot: true
+    })
+
+    expect(result.action).toMatchObject({
+      path: 'accessibility',
+      actionName: 'setValue'
+    })
+  })
+
+  it('keeps window-index snapshots scoped to the matching session', async () => {
+    mockBridgeResponse({
+      ok: true,
+      snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+    })
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await client.snapshot({ app: 'Text Editor', session: 'agent-a', windowIndex: 0 })
+
+    await expect(
+      client.action('click', {
+        app: 'Text Editor',
+        session: 'agent-b',
+        windowIndex: 0,
+        elementIndex: 0
+      })
+    ).rejects.toMatchObject({ code: 'element_not_found' })
+  })
+
+  it('explains screenshot capture failures while keeping accessibility state usable', async () => {
+    mockBridgeResponse({
+      ok: true,
+      snapshot: {
+        ...sampleBridgeSnapshot('Text Editor', 'initial'),
+        screenshotPngBase64: null
+      }
+    })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await expect(client.snapshot({ app: 'Text Editor' })).resolves.toMatchObject({
+      snapshot: {
+        elementCount: 1
+      },
+      screenshotStatus: {
+        state: 'failed',
+        code: 'screenshot_failed',
+        message: expect.stringContaining('--no-screenshot')
+      }
+    })
+  })
+
+  it('normalizes list-windows responses after provider handshake', async () => {
+    mockBridgeResponse({
+      ok: true,
+      capabilities: {
+        platform: 'linux',
+        provider: 'orca-computer-use-linux',
+        providerVersion: '1.0.0',
+        protocolVersion: 1,
+        supports: {
+          apps: { list: true, bundleIds: false, pids: true },
+          windows: {
+            list: true,
+            targetById: true,
+            targetByIndex: true,
+            focus: false,
+            moveResize: false
+          },
+          observation: {
+            screenshot: true,
+            annotatedScreenshot: false,
+            elementFrames: true,
+            ocr: false
+          },
+          actions: {
+            click: true,
+            typeText: true,
+            pressKey: true,
+            hotkey: false,
+            pasteText: false,
+            scroll: true,
+            drag: true,
+            setValue: true,
+            performAction: true
+          },
+          surfaces: { menus: false, dialogs: false, dock: false, menubar: false }
+        }
+      }
+    })
+    mockBridgeResponse({
+      ok: true,
+      app: { name: 'Text Editor', bundleIdentifier: 'Text Editor', pid: 100 },
+      windows: [
+        {
+          index: 0,
+          app: { name: 'Text Editor', bundleIdentifier: 'Text Editor', pid: 100 },
+          id: 99,
+          title: 'Document',
+          x: 10,
+          y: 20,
+          width: 300,
+          height: 200,
+          isMinimized: false,
+          isOffscreen: false,
+          screenIndex: null,
+          platform: { backend: 'at-spi' }
+        }
+      ]
+    })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await expect(client.listWindows({ app: 'Text Editor' })).resolves.toMatchObject({
+      app: { name: 'Text Editor', bundleId: 'Text Editor', pid: 100 },
+      windows: [{ index: 0, id: 99, title: 'Document' }]
+    })
+  })
+
+  it('maps bridge app errors to RuntimeClientError codes', async () => {
+    mockBridgeResponse({ ok: false, error: 'appNotFound("Missing")' })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await expect(client.snapshot({ app: 'Missing' })).rejects.toMatchObject({
+      code: 'app_not_found'
+    })
+  })
+
+  it('maps blocked app bridge errors to a policy error', async () => {
+    mockBridgeResponse({ ok: false, error: 'appBlocked("1Password")' })
+
+    const client = new DesktopScriptProviderClient('windows', '/tmp/runtime.ps1')
+
+    await expect(client.snapshot({ app: '1Password' })).rejects.toMatchObject({
+      code: 'app_blocked'
+    })
+  })
+
+  it('maps action-specific bridge errors to actionable codes', async () => {
+    mockBridgeResponse({ ok: false, error: 'element value is not settable' })
+    mockBridgeResponse({ ok: false, error: 'Raise is not a valid secondary action' })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
+      code: 'value_not_settable'
+    })
+    await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
+      code: 'action_not_supported'
+    })
+  })
+
+  it('rejects actions that the provider does not advertise', async () => {
+    mockBridgeResponse({
+      ok: true,
+      capabilities: sampleCapabilities({
+        hotkey: false,
+        pasteText: false
+      })
+    })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await expect(
+      client.action('pasteText', { app: 'Text Editor', text: 'hello' })
+    ).rejects.toMatchObject({
+      code: 'unsupported_capability',
+      message: expect.stringContaining('actions.pasteText')
+    })
+  })
+
+  it('uses provider action metadata when bridge reports the actual path', async () => {
+    mockBridgeResponse({
+      ok: true,
+      snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
+    })
+    mockBridgeResponse({
+      ok: true,
+      action: {
+        path: 'accessibility',
+        actionName: 'Press',
+        fallbackReason: null
+      },
+      snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
+    })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await client.snapshot({ app: 'Text Editor' })
+    await expect(
+      client.action('click', {
+        app: 'Text Editor',
+        elementIndex: 0,
+        noScreenshot: true
+      })
+    ).resolves.toMatchObject({
+      action: {
+        path: 'accessibility',
+        actionName: 'Press'
+      }
+    })
+  })
+
+  it('explains that missing element indexes require a fresh snapshot', async () => {
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    await expect(
+      client.action('click', { app: 'Text Editor', elementIndex: 4 })
+    ).rejects.toMatchObject({
+      code: 'element_not_found',
+      message: expect.stringContaining('run get-app-state again')
+    })
+  })
+})
+
+function mockBridgeResponse(
+  response: unknown,
+  inspectOperation?: (operation: Record<string, unknown>) => void
+): void {
+  vi.mocked(execFile).mockImplementationOnce((_command, _args, _options, callback) => {
+    const operationPath = _args?.at(-1)
+    if (inspectOperation && typeof operationPath === 'string') {
+      inspectOperation(JSON.parse(readFileSync(operationPath, 'utf8')) as Record<string, unknown>)
+    }
+    const done = callback as (error: Error | null, stdout: string, stderr: string) => void
+    done(null, JSON.stringify(response), '')
+    return null as never
+  })
+}
+
+function sampleBridgeSnapshot(name: string, value: string) {
+  return {
+    app: { name, bundleIdentifier: name, pid: 100 },
+    snapshotId: 'snap-test',
+    windowTitle: name,
+    windowId: 99,
+    windowBounds: { x: 10, y: 20, width: 300, height: 200 },
+    screenshotPngBase64: 'iVBORw0KGgo=',
+    coordinateSpace: 'window',
+    truncation: { truncated: false, maxNodes: 1200, maxDepth: 64, maxDepthReached: false },
+    treeLines: [`0 text entry area, Value: ${value}`],
+    focusedSummary: 'text entry area',
+    elements: [
+      {
+        index: 0,
+        runtimeId: [0, 0],
+        name: 'Body',
+        controlType: 'text',
+        localizedControlType: 'text entry area',
+        value,
+        frame: { x: 1, y: 2, width: 100, height: 20 },
+        actions: ['SetValue']
+      }
+    ]
+  }
+}
+
+function sampleCapabilities(actions: Partial<Record<string, boolean>> = {}) {
+  return {
+    platform: 'linux',
+    provider: 'orca-computer-use-linux',
+    providerVersion: '1.0.0',
+    protocolVersion: 1,
+    supports: {
+      apps: { list: true, bundleIds: false, pids: true },
+      windows: {
+        list: true,
+        targetById: true,
+        targetByIndex: true,
+        focus: false,
+        moveResize: false
+      },
+      observation: {
+        screenshot: true,
+        annotatedScreenshot: false,
+        elementFrames: true,
+        ocr: false
+      },
+      actions: {
+        click: true,
+        typeText: true,
+        pressKey: true,
+        hotkey: true,
+        pasteText: true,
+        scroll: true,
+        drag: true,
+        setValue: true,
+        performAction: true,
+        ...actions
+      },
+      surfaces: { menus: false, dialogs: false, dock: false, menubar: false }
+    }
+  }
+}
+
+function publicSnapshotKeys(snapshot: unknown): string[] {
+  return Object.keys(snapshot as Record<string, unknown>).sort()
+}

src/main/computer/desktop-script-provider-client.ts

@@ -0,0 +1,742 @@
+/* eslint-disable max-lines -- Why: bridge schema, request mapping, and result normalization stay together so platform scripts have one audited contract. */
+import { execFile } from 'child_process'
+import { mkdtemp, rm, writeFile } from 'fs/promises'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import type {
+  ComputerActionMetadata,
+  ComputerActionResult,
+  ComputerListAppsResult,
+  ComputerListWindowsResult,
+  ComputerProviderCapabilities,
+  ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import {
+  desktopScriptPlatform,
+  resolveDesktopScriptProviderPath,
+  type DesktopScriptPlatform
+} from './desktop-script-provider-paths'
+import { RuntimeClientError } from './runtime-client-error'
+
+type NativeMethod =
+  | 'handshake'
+  | 'listApps'
+  | 'listWindows'
+  | 'getAppState'
+  | 'click'
+  | 'performSecondaryAction'
+  | 'scroll'
+  | 'drag'
+  | 'typeText'
+  | 'pressKey'
+  | 'hotkey'
+  | 'pasteText'
+  | 'setValue'
+
+type NativeActionMethod = Exclude<
+  NativeMethod,
+  'handshake' | 'listApps' | 'listWindows' | 'getAppState'
+>
+
+type BridgeFrame = {
+  x: number
+  y: number
+  width: number
+  height: number
+}
+
+type BridgeElement = {
+  index: number
+  runtimeId?: unknown
+  automationId?: string
+  name?: string
+  controlType?: string
+  localizedControlType?: string
+  className?: string
+  value?: string
+  nativeWindowHandle?: number
+  frame?: BridgeFrame | null
+  actions?: string[]
+}
+
+type BridgeSnapshot = {
+  snapshotId?: string
+  app: {
+    name: string
+    bundleIdentifier?: string
+    bundleId?: string
+    pid: number
+  }
+  windowTitle?: string
+  windowId?: number | null
+  windowBounds?: BridgeFrame | null
+  screenshotPngBase64?: string | null
+  coordinateSpace?: 'window'
+  truncation?: {
+    truncated?: boolean
+    maxNodes?: number
+    maxDepth?: number
+    maxDepthReached?: boolean
+  }
+  treeLines?: string[]
+  focusedSummary?: string | null
+  focusedElementId?: number | null
+  selectedText?: string | null
+  elements?: BridgeElement[]
+}
+
+type BridgeWindow = {
+  index: number
+  app: {
+    name: string
+    bundleIdentifier?: string | null
+    bundleId?: string | null
+    pid: number
+  }
+  id?: number | null
+  title: string
+  x?: number | null
+  y?: number | null
+  width: number
+  height: number
+  isMinimized?: boolean | null
+  isOffscreen?: boolean | null
+  screenIndex?: number | null
+  platform?: Record<string, unknown>
+}
+
+type BridgeResponse = {
+  ok: boolean
+  error?: string
+  capabilities?: ComputerProviderCapabilities
+  apps?: {
+    name: string
+    bundleIdentifier?: string | null
+    bundleId?: string | null
+    pid: number
+  }[]
+  app?: {
+    name: string
+    bundleIdentifier?: string | null
+    bundleId?: string | null
+    pid: number
+  }
+  windows?: BridgeWindow[]
+  snapshot?: BridgeSnapshot
+  action?: ComputerActionMetadata
+}
+
+type BridgeRequest = {
+  tool: string
+  app?: string
+  element?: BridgeElement
+  fromElement?: BridgeElement
+  toElement?: BridgeElement
+  x?: number
+  y?: number
+  from_x?: number
+  from_y?: number
+  to_x?: number
+  to_y?: number
+  click_count?: number
+  mouse_button?: string
+  action?: string
+  direction?: string
+  pages?: number
+  text?: string
+  key?: string
+  value?: string
+  windowBounds?: BridgeFrame | null
+  windowId?: number
+  windowIndex?: number
+  noScreenshot?: boolean
+  restoreWindow?: boolean
+}
+
+const REQUEST_TIMEOUT_MS = 30_000
+
+export function shouldUseDesktopScriptProvider(): boolean {
+  return desktopScriptPlatform() !== null && resolveDesktopScriptProviderPath() !== null
+}
+
+export class DesktopScriptProviderClient {
+  private readonly snapshots = new Map<string, BridgeSnapshot>()
+  private providerCapabilities: ComputerProviderCapabilities | null = null
+
+  constructor(
+    private readonly platform: DesktopScriptPlatform = requiredPlatform(),
+    private readonly scriptPath: string = requiredScriptPath()
+  ) {}
+
+  async listApps(): Promise<ComputerListAppsResult> {
+    const response = await this.callBridge({ tool: 'list_apps' })
+    return {
+      apps: (response.apps ?? []).map((app) => ({
+        name: app.name,
+        bundleId: app.bundleId ?? app.bundleIdentifier ?? null,
+        pid: app.pid,
+        isRunning: true,
+        lastUsedAt: null,
+        useCount: null
+      }))
+    }
+  }
+
+  async capabilities(): Promise<ComputerProviderCapabilities> {
+    return await this.readCapabilities()
+  }
+
+  async listWindows(params: Record<string, unknown>): Promise<ComputerListWindowsResult> {
+    const capabilities = await this.readCapabilities()
+    if (!capabilities.supports.windows.list) {
+      throw new RuntimeClientError(
+        'unsupported_capability',
+        `${capabilities.provider} does not support windows.list`
+      )
+    }
+    const response = await this.callBridge({
+      tool: 'list_windows',
+      app: stringParam(params, 'app')
+    })
+    return {
+      app: normalizeBridgeApp(response.app),
+      windows: (response.windows ?? []).map((window) => ({
+        index: window.index,
+        app: normalizeBridgeApp(window.app),
+        id: window.id ?? null,
+        title: window.title,
+        x: window.x ?? null,
+        y: window.y ?? null,
+        width: window.width,
+        height: window.height,
+        isMinimized: window.isMinimized ?? null,
+        isOffscreen: window.isOffscreen ?? null,
+        screenIndex: window.screenIndex ?? null,
+        platform: window.platform
+      }))
+    }
+  }
+
+  async snapshot(params: Record<string, unknown>): Promise<ComputerSnapshotResult> {
+    const app = stringParam(params, 'app')
+    const response = await this.callBridge({
+      tool: 'get_app_state',
+      app,
+      windowId: optionalNumberParam(params, 'windowId'),
+      windowIndex: optionalNumberParam(params, 'windowIndex'),
+      noScreenshot: params.noScreenshot === true,
+      restoreWindow: params.restoreWindow === true
+    })
+    return this.rememberAndRender(app, response, params.noScreenshot === true, params)
+  }
+
+  async action(
+    method: NativeActionMethod,
+    params: Record<string, unknown>
+  ): Promise<ComputerActionResult> {
+    const app = stringParam(params, 'app')
+    await this.ensureActionSupported(method)
+    const explicitWindowId = optionalNumberParam(params, 'windowId')
+    const explicitWindowIndex = optionalNumberParam(params, 'windowIndex')
+    const current = this.currentSnapshot(app, explicitWindowId, params)
+    const element = elementParam(current, optionalNumberParam(params, 'elementIndex'))
+    const fromElement = elementParam(current, optionalNumberParam(params, 'fromElementIndex'))
+    const toElement = elementParam(current, optionalNumberParam(params, 'toElementIndex'))
+    const response = await this.callBridge({
+      tool: bridgeTool(method),
+      app,
+      element,
+      fromElement,
+      toElement,
+      x: optionalNumberParam(params, 'x'),
+      y: optionalNumberParam(params, 'y'),
+      from_x: optionalNumberParam(params, 'fromX'),
+      from_y: optionalNumberParam(params, 'fromY'),
+      to_x: optionalNumberParam(params, 'toX'),
+      to_y: optionalNumberParam(params, 'toY'),
+      click_count: optionalNumberParam(params, 'clickCount'),
+      mouse_button: optionalStringParam(params, 'mouseButton'),
+      action: optionalStringParam(params, 'action'),
+      direction: optionalStringParam(params, 'direction'),
+      pages: optionalNumberParam(params, 'pages'),
+      text: optionalStringParam(params, 'text'),
+      key: optionalStringParam(params, 'key'),
+      value: optionalStringParam(params, 'value'),
+      windowBounds: null,
+      windowId: explicitWindowId ?? current?.windowId ?? undefined,
+      windowIndex: explicitWindowIndex,
+      noScreenshot: params.noScreenshot === true,
+      restoreWindow: params.restoreWindow === true
+    })
+    return {
+      ...this.rememberAndRender(app, response, params.noScreenshot === true, params),
+      action:
+        response.action ??
+        desktopActionMetadata(method, response.snapshot?.windowId ?? current?.windowId ?? null)
+    }
+  }
+
+  private async ensureActionSupported(method: NativeActionMethod): Promise<void> {
+    if (method !== 'hotkey' && method !== 'pasteText') {
+      return
+    }
+    const capabilities = await this.readCapabilities()
+    const actionKey = actionCapabilityKey(method)
+    if (!capabilities.supports.actions[actionKey]) {
+      throw new RuntimeClientError(
+        'unsupported_capability',
+        `${capabilities.provider} does not support actions.${actionKey}`
+      )
+    }
+  }
+
+  private async callBridge(request: BridgeRequest): Promise<BridgeResponse> {
+    const operationDirectory = await mkdtemp(join(tmpdir(), 'orca-computer-use-'))
+    const operationPath = join(operationDirectory, 'operation.json')
+    try {
+      await writeFile(operationPath, JSON.stringify(request), { encoding: 'utf8', mode: 0o600 })
+      const { stdout, stderr } = await execBridge(this.platform, this.scriptPath, operationPath)
+      let response: BridgeResponse
+      try {
+        response = JSON.parse(stdout) as BridgeResponse
+      } catch (error) {
+        throw new RuntimeClientError(
+          'accessibility_error',
+          `desktop provider returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`
+        )
+      }
+      if (!response.ok) {
+        throw mapBridgeError(response.error ?? stderr)
+      }
+      return response
+    } finally {
+      await rm(operationDirectory, { force: true, recursive: true })
+    }
+  }
+
+  private async readCapabilities(): Promise<ComputerProviderCapabilities> {
+    if (this.providerCapabilities) {
+      return this.providerCapabilities
+    }
+    const response = await this.callBridge({ tool: 'handshake' })
+    if (!response.capabilities) {
+      throw new RuntimeClientError(
+        'accessibility_error',
+        'desktop provider returned no capabilities'
+      )
+    }
+    this.providerCapabilities = response.capabilities
+    return response.capabilities
+  }
+
+  private rememberAndRender(
+    app: string,
+    response: BridgeResponse,
+    noScreenshot: boolean,
+    params: Record<string, unknown>
+  ): ComputerSnapshotResult {
+    if (!response.snapshot) {
+      throw new RuntimeClientError('accessibility_error', 'desktop provider returned no snapshot')
+    }
+    this.rememberSnapshot(app, response.snapshot, params)
+    return renderSnapshot(response.snapshot, noScreenshot)
+  }
+
+  private rememberSnapshot(
+    query: string,
+    snapshot: BridgeSnapshot,
+    params: Record<string, unknown>
+  ): void {
+    const namespace = snapshotNamespace(params)
+    for (const key of [
+      query,
+      snapshot.app.name,
+      snapshot.app.bundleId,
+      snapshot.app.bundleIdentifier,
+      String(snapshot.app.pid),
+      ...snapshotKeysForWindow(query, snapshot),
+      ...snapshotKeysForWindow(snapshot.app.name, snapshot),
+      ...(snapshot.app.bundleId ? snapshotKeysForWindow(snapshot.app.bundleId, snapshot) : []),
+      ...(snapshot.app.bundleIdentifier
+        ? snapshotKeysForWindow(snapshot.app.bundleIdentifier, snapshot)
+        : []),
+      ...snapshotKeysForWindowIndex(query, params),
+      ...snapshotKeysForWindowIndex(snapshot.app.name, params),
+      ...(snapshot.app.bundleId ? snapshotKeysForWindowIndex(snapshot.app.bundleId, params) : []),
+      ...(snapshot.app.bundleIdentifier
+        ? snapshotKeysForWindowIndex(snapshot.app.bundleIdentifier, params)
+        : [])
+    ]) {
+      if (key) {
+        if (!isExplicitSnapshotNamespace(namespace)) {
+          this.snapshots.set(key.toLowerCase(), snapshot)
+        }
+        this.snapshots.set(namespacedSnapshotKey(namespace, key), snapshot)
+      }
+    }
+  }
+
+  private currentSnapshot(
+    app: string,
+    windowId: number | undefined,
+    params: Record<string, unknown>
+  ): BridgeSnapshot | null {
+    const namespace = snapshotNamespace(params)
+    if (windowId !== undefined) {
+      const windowKey = snapshotWindowKey(app, windowId)
+      return (
+        this.snapshots.get(namespacedSnapshotKey(namespace, windowKey)) ??
+        (isExplicitSnapshotNamespace(namespace) ? undefined : this.snapshots.get(windowKey)) ??
+        null
+      )
+    }
+    const windowIndex = optionalNumberParam(params, 'windowIndex')
+    if (windowIndex !== undefined) {
+      const windowIndexKey = snapshotWindowIndexKey(app, windowIndex)
+      return (
+        this.snapshots.get(namespacedSnapshotKey(namespace, windowIndexKey)) ??
+        (isExplicitSnapshotNamespace(namespace) ? undefined : this.snapshots.get(windowIndexKey)) ??
+        null
+      )
+    }
+    return (
+      this.snapshots.get(namespacedSnapshotKey(namespace, app)) ??
+      (isExplicitSnapshotNamespace(namespace)
+        ? undefined
+        : this.snapshots.get(app.toLowerCase())) ??
+      null
+    )
+  }
+}
+
+function execBridge(
+  platform: DesktopScriptPlatform,
+  scriptPath: string,
+  operationPath: string
+): Promise<{ stdout: string; stderr: string }> {
+  const command = platform === 'windows' ? 'powershell.exe' : 'python3'
+  const args =
+    platform === 'windows'
+      ? [
+          '-NoProfile',
+          '-NonInteractive',
+          '-ExecutionPolicy',
+          'Bypass',
+          '-File',
+          scriptPath,
+          operationPath
+        ]
+      : [scriptPath, operationPath]
+  return new Promise((resolve, reject) => {
+    execFile(
+      command,
+      args,
+      {
+        env: process.env,
+        maxBuffer: 20 * 1024 * 1024,
+        timeout: REQUEST_TIMEOUT_MS,
+        windowsHide: true
+      },
+      (error, stdout, stderr) => {
+        if (error) {
+          const message = stderr.trim() || stdout.trim() || error.message
+          reject(
+            error.killed
+              ? new RuntimeClientError('action_timeout', message)
+              : mapBridgeError(message)
+          )
+          return
+        }
+        resolve({ stdout, stderr })
+      }
+    )
+  })
+}
+
+function renderSnapshot(snapshot: BridgeSnapshot, noScreenshot: boolean): ComputerSnapshotResult {
+  const bounds = snapshot.windowBounds
+  const treeText = renderTreeText(snapshot)
+  const screenshot = snapshot.screenshotPngBase64
+    ? {
+        data: snapshot.screenshotPngBase64,
+        format: 'png' as const,
+        width: Math.max(1, Math.round(bounds?.width ?? 1)),
+        height: Math.max(1, Math.round(bounds?.height ?? 1)),
+        scale: 1
+      }
+    : null
+  return {
+    snapshot: {
+      // Why: bridge elements can be large; keep them cached internally for actions
+      // instead of returning duplicated metadata in every agent-facing snapshot.
+      id: snapshot.snapshotId ?? fallbackSnapshotId(snapshot),
+      app: {
+        name: snapshot.app.name,
+        bundleId: snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? null,
+        pid: snapshot.app.pid
+      },
+      window: {
+        title: snapshot.windowTitle ?? snapshot.app.name,
+        id: snapshot.windowId ?? null,
+        x: bounds ? Math.round(bounds.x) : null,
+        y: bounds ? Math.round(bounds.y) : null,
+        width: Math.max(0, Math.round(bounds?.width ?? 0)),
+        height: Math.max(0, Math.round(bounds?.height ?? 0)),
+        isMinimized: null,
+        isOffscreen: null,
+        screenIndex: null
+      },
+      coordinateSpace: snapshot.coordinateSpace ?? 'window',
+      treeText,
+      elementCount: snapshot.elements?.length ?? 0,
+      focusedElementId: snapshot.focusedElementId ?? null,
+      truncation: {
+        truncated: snapshot.truncation?.truncated === true,
+        maxNodes: snapshot.truncation?.maxNodes,
+        maxDepth: snapshot.truncation?.maxDepth,
+        maxDepthReached: snapshot.truncation?.maxDepthReached === true
+      }
+    },
+    screenshot,
+    screenshotStatus: screenshot
+      ? {
+          state: 'captured',
+          metadata: { engine: 'unknown', windowId: snapshot.windowId ?? null }
+        }
+      : noScreenshot
+        ? { state: 'skipped', reason: 'no_screenshot_flag' }
+        : {
+            state: 'failed',
+            code: 'screenshot_failed',
+            message:
+              'desktop provider returned no image; grant screen capture permission or pass --no-screenshot to inspect accessibility state only.'
+          }
+  }
+}
+
+function fallbackSnapshotId(snapshot: BridgeSnapshot): string {
+  const appRef = snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? snapshot.app.name
+  return `${appRef}:${snapshot.app.pid}:${snapshot.windowId ?? 'window'}`
+}
+
+function renderTreeText(snapshot: BridgeSnapshot): string {
+  const appRef = snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? snapshot.app.name
+  const lines = [
+    `App=${appRef} (pid ${snapshot.app.pid})`,
+    `Window: "${sanitize(snapshot.windowTitle ?? snapshot.app.name)}", App: ${sanitize(snapshot.app.name)}.`,
+    '',
+    ...(snapshot.treeLines ?? [])
+  ]
+  if (snapshot.selectedText) {
+    lines.push('', `Selected text: [${sanitize(snapshot.selectedText)}]`)
+  } else if (snapshot.focusedSummary) {
+    lines.push('', `The focused UI element is ${sanitize(snapshot.focusedSummary)}.`)
+  }
+  return lines.join('\n')
+}
+
+function bridgeTool(method: NativeActionMethod): string {
+  return {
+    click: 'click',
+    performSecondaryAction: 'perform_secondary_action',
+    scroll: 'scroll',
+    drag: 'drag',
+    typeText: 'type_text',
+    pressKey: 'press_key',
+    hotkey: 'hotkey',
+    pasteText: 'paste_text',
+    setValue: 'set_value'
+  }[method]
+}
+
+function actionCapabilityKey(
+  method: NativeActionMethod
+): keyof ComputerProviderCapabilities['supports']['actions'] {
+  const keys = {
+    click: 'click',
+    performSecondaryAction: 'performAction',
+    scroll: 'scroll',
+    drag: 'drag',
+    typeText: 'typeText',
+    pressKey: 'pressKey',
+    hotkey: 'hotkey',
+    pasteText: 'pasteText',
+    setValue: 'setValue'
+  } satisfies Record<NativeActionMethod, keyof ComputerProviderCapabilities['supports']['actions']>
+  return keys[method]
+}
+
+function desktopActionMetadata(method: NativeActionMethod, targetWindowId: number | null) {
+  const path =
+    method === 'pasteText'
+      ? ('clipboard' as const)
+      : method === 'setValue' || method === 'performSecondaryAction'
+        ? ('accessibility' as const)
+        : ('synthetic' as const)
+  return {
+    path,
+    actionName:
+      method === 'hotkey'
+        ? 'hotkey'
+        : method === 'pasteText'
+          ? 'paste'
+          : method === 'setValue'
+            ? 'setValue'
+            : method === 'performSecondaryAction'
+              ? 'performSecondaryAction'
+              : null,
+    fallbackReason: null,
+    targetWindowId,
+    verification:
+      method === 'hotkey' || method === 'pasteText'
+        ? {
+            state: 'unverified' as const,
+            reason:
+              method === 'pasteText' ? ('clipboard_paste' as const) : ('synthetic_input' as const)
+          }
+        : undefined
+  }
+}
+
+function normalizeBridgeApp(app: BridgeResponse['app'] | BridgeWindow['app'] | undefined) {
+  if (!app) {
+    throw new RuntimeClientError('accessibility_error', 'desktop provider returned no app')
+  }
+  return {
+    name: app.name,
+    bundleId: app.bundleId ?? app.bundleIdentifier ?? null,
+    pid: app.pid
+  }
+}
+
+function elementParam(
+  snapshot: BridgeSnapshot | null,
+  index: number | undefined
+): BridgeElement | undefined {
+  if (index === undefined) {
+    return undefined
+  }
+  const element = snapshot?.elements?.find((candidate) => candidate.index === index)
+  if (!element) {
+    throw new RuntimeClientError(
+      'element_not_found',
+      `element ${index} is not in the current cached snapshot; run get-app-state again and use a fresh element index`
+    )
+  }
+  return element
+}
+
+function snapshotKeysForWindow(query: string, snapshot: BridgeSnapshot): string[] {
+  return snapshot.windowId === null || snapshot.windowId === undefined
+    ? []
+    : [snapshotWindowKey(query, snapshot.windowId)]
+}
+
+function snapshotWindowKey(query: string, windowId: number): string {
+  return `${query.toLowerCase()}#window:${windowId}`
+}
+
+function snapshotKeysForWindowIndex(query: string, params: Record<string, unknown>): string[] {
+  const windowIndex = optionalNumberParam(params, 'windowIndex')
+  return windowIndex === undefined ? [] : [snapshotWindowIndexKey(query, windowIndex)]
+}
+
+function snapshotWindowIndexKey(query: string, windowIndex: number): string {
+  return `${query.toLowerCase()}#window-index:${windowIndex}`
+}
+
+function snapshotNamespace(params: Record<string, unknown>): string {
+  const session = optionalStringParam(params, 'session')
+  const worktree = optionalStringParam(params, 'worktree')
+  return session ? `session:${session}` : worktree ? `worktree:${worktree}` : 'default'
+}
+
+function namespacedSnapshotKey(namespace: string, key: string): string {
+  return `${namespace}:${key.toLowerCase()}`
+}
+
+function isExplicitSnapshotNamespace(namespace: string): boolean {
+  return namespace !== 'default'
+}
+
+function mapBridgeError(message: string): RuntimeClientError {
+  const text = message.trim() || 'desktop provider failed'
+  if (/appNotFound|app not found/i.test(text)) {
+    return new RuntimeClientError('app_not_found', text)
+  }
+  if (/appBlocked|app blocked/i.test(text)) {
+    return new RuntimeClientError('app_blocked', text)
+  }
+  if (/unsupported capability|hotkey.*require|paste_text requires/i.test(text)) {
+    return new RuntimeClientError('unsupported_capability', text)
+  }
+  if (/ModuleNotFoundError: No module named 'gi'|PyGObject|python3-gi/i.test(text)) {
+    return new RuntimeClientError(
+      'unsupported_capability',
+      'Linux Computer Use requires python3-gi and AT-SPI packages. Install python3-gi gir1.2-atspi-2.0 at-spi2-core, then retry.'
+    )
+  }
+  if (/not a valid secondary action|action.*not supported/i.test(text)) {
+    return new RuntimeClientError('action_not_supported', text)
+  }
+  if (/value is not settable|not settable/i.test(text)) {
+    return new RuntimeClientError('value_not_settable', text)
+  }
+  if (/stale element|fresh element index/i.test(text)) {
+    return new RuntimeClientError('element_not_found', text)
+  }
+  if (/windowStale|window stale/i.test(text)) {
+    return new RuntimeClientError('window_stale', text)
+  }
+  if (/No top-level|No .*window|window/i.test(text)) {
+    return new RuntimeClientError('window_not_found', text)
+  }
+  if (/permission|desktop session|DBUS|XDG_RUNTIME_DIR|AT-SPI/i.test(text)) {
+    return new RuntimeClientError('permission_denied', text)
+  }
+  if (/element|element_index/i.test(text)) {
+    return new RuntimeClientError('element_not_found', text)
+  }
+  return new RuntimeClientError('accessibility_error', text)
+}
+
+function requiredPlatform(): DesktopScriptPlatform {
+  const platform = desktopScriptPlatform()
+  if (!platform) {
+    throw new RuntimeClientError('accessibility_error', 'desktop script provider is not available')
+  }
+  return platform
+}
+
+function requiredScriptPath(): string {
+  const scriptPath = resolveDesktopScriptProviderPath()
+  if (!scriptPath) {
+    throw new RuntimeClientError(
+      'accessibility_error',
+      'desktop script provider script was not found'
+    )
+  }
+  return scriptPath
+}
+
+function stringParam(params: Record<string, unknown>, key: string): string {
+  const value = params[key]
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new RuntimeClientError('invalid_argument', `missing ${key}`)
+  }
+  return value
+}
+
+function optionalStringParam(params: Record<string, unknown>, key: string): string | undefined {
+  const value = params[key]
+  return typeof value === 'string' ? value : undefined
+}
+
+function optionalNumberParam(params: Record<string, unknown>, key: string): number | undefined {
+  const value = params[key]
+  return typeof value === 'number' && Number.isFinite(value) ? value : undefined
+}
+
+function sanitize(value: string): string {
+  return value.replaceAll('\n', ' ').replaceAll('\r', ' ')
+}

src/main/computer/desktop-script-provider-paths.ts

@@ -0,0 +1,39 @@
+import { existsSync } from 'fs'
+import { join, resolve } from 'path'
+
+export type DesktopScriptPlatform = 'linux' | 'windows'
+
+export function desktopScriptPlatform(): DesktopScriptPlatform | null {
+  if (process.platform === 'linux') {
+    return 'linux'
+  }
+  if (process.platform === 'win32') {
+    return 'windows'
+  }
+  return null
+}
+
+export function resolveDesktopScriptProviderPath(
+  platform = desktopScriptPlatform()
+): string | null {
+  const override = process.env.ORCA_COMPUTER_DESKTOP_SCRIPT_PROVIDER_PATH
+  if (override && existsSync(override)) {
+    return override
+  }
+  if (!platform) {
+    return null
+  }
+
+  const filename = platform === 'windows' ? 'runtime.ps1' : 'runtime.py'
+  const directory = platform === 'windows' ? 'computer-use-windows' : 'computer-use-linux'
+  const sourceDirectory =
+    platform === 'windows' ? 'native/computer-use-windows' : 'native/computer-use-linux'
+  const packaged = [join(process.resourcesPath ?? '', directory, filename)]
+  const dev = [
+    join(process.cwd(), sourceDirectory, filename),
+    resolve(__dirname, '../../', sourceDirectory, filename)
+  ]
+  const candidates = process.resourcesPath ? [...packaged, ...dev] : dev
+
+  return candidates.find((candidate) => candidate && existsSync(candidate)) ?? null
+}

src/main/computer/key-mapping.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest'
+import { parseKey } from './key-mapping'
+
+describe('parseKey', () => {
+  it.each([
+    ['a', { key: 'a', modifiers: [] }],
+    ['7', { key: '7', modifiers: [] }],
+    [';', { key: ';', modifiers: [] }],
+    ['Return', { key: 'Enter', modifiers: [] }],
+    ['Escape', { key: 'Escape', modifiers: [] }],
+    ['BackSpace', { key: 'Backspace', modifiers: [] }],
+    ['Tab', { key: 'Tab', modifiers: [] }],
+    ['space', { key: 'Space', modifiers: [] }],
+    ['Page_Up', { key: 'PageUp', modifiers: [] }],
+    ['Page_Down', { key: 'PageDown', modifiers: [] }],
+    ['Home', { key: 'Home', modifiers: [] }],
+    ['End', { key: 'End', modifiers: [] }],
+    ['Up', { key: 'ArrowUp', modifiers: [] }],
+    ['Down', { key: 'ArrowDown', modifiers: [] }],
+    ['Left', { key: 'ArrowLeft', modifiers: [] }],
+    ['Right', { key: 'ArrowRight', modifiers: [] }],
+    ['Delete', { key: 'Delete', modifiers: [] }],
+    ['Insert', { key: 'Insert', modifiers: [] }],
+    ['F1', { key: 'F1', modifiers: [] }],
+    ['F24', { key: 'F24', modifiers: [] }],
+    ['KP_0', { key: 'Numpad0', modifiers: [] }],
+    ['KP_9', { key: 'Numpad9', modifiers: [] }],
+    ['KP_Add', { key: 'NumpadAdd', modifiers: [] }],
+    ['KP_Subtract', { key: 'NumpadSubtract', modifiers: [] }],
+    ['KP_Multiply', { key: 'NumpadMultiply', modifiers: [] }],
+    ['KP_Divide', { key: 'NumpadDivide', modifiers: [] }],
+    ['KP_Enter', { key: 'NumpadEnter', modifiers: [] }]
+  ])('maps %s', (input, expected) => {
+    expect(parseKey(input)).toEqual(expected)
+  })
+
+  it.each([
+    ['ctrl+a', { key: 'a', modifiers: ['Ctrl'] }],
+    ['control+a', { key: 'a', modifiers: ['Ctrl'] }],
+    ['ctrl+shift+t', { key: 't', modifiers: ['Ctrl', 'Shift'] }],
+    ['alt+F4', { key: 'F4', modifiers: ['Alt'] }],
+    ['cmd+a', { key: 'a', modifiers: ['Meta'] }],
+    ['command+a', { key: 'a', modifiers: ['Meta'] }],
+    ['CmdOrCtrl+a', { key: 'a', modifiers: [process.platform === 'darwin' ? 'Meta' : 'Ctrl'] }],
+    ['super+Left', { key: 'ArrowLeft', modifiers: ['Meta'] }],
+    ['win+Right', { key: 'ArrowRight', modifiers: ['Meta'] }]
+  ])('maps chord %s', (input, expected) => {
+    expect(parseKey(input)).toEqual(expected)
+  })
+
+  it('rejects unknown modifiers and key names', () => {
+    expect(() => parseKey('hyper+a')).toThrow(expect.objectContaining({ code: 'invalid_argument' }))
+    expect(() => parseKey('NotAKey')).toThrow(expect.objectContaining({ code: 'invalid_argument' }))
+    expect(() => parseKey('F25')).toThrow(expect.objectContaining({ code: 'invalid_argument' }))
+  })
+})

src/main/computer/key-mapping.ts

@@ -0,0 +1,109 @@
+import { RuntimeClientError } from './runtime-client-error'
+
+export type KeyChord = {
+  key: string
+  modifiers: string[]
+}
+
+const MODIFIER_NAMES: Record<string, string> = {
+  ctrl: 'Ctrl',
+  control: 'Ctrl',
+  shift: 'Shift',
+  alt: 'Alt',
+  meta: 'Meta',
+  super: 'Meta',
+  cmd: 'Meta',
+  command: 'Meta',
+  win: 'Meta'
+}
+
+const PLATFORM_MODIFIER_NAMES: Record<string, () => string> = {
+  cmdorctrl: () => (process.platform === 'darwin' ? 'Meta' : 'Ctrl'),
+  commandorcontrol: () => (process.platform === 'darwin' ? 'Meta' : 'Ctrl')
+}
+
+const KEY_NAMES: Record<string, string> = {
+  Return: 'Enter',
+  Enter: 'Enter',
+  Escape: 'Escape',
+  Esc: 'Escape',
+  BackSpace: 'Backspace',
+  Backspace: 'Backspace',
+  Tab: 'Tab',
+  space: 'Space',
+  Space: 'Space',
+  Page_Up: 'PageUp',
+  Page_Down: 'PageDown',
+  Home: 'Home',
+  End: 'End',
+  Up: 'ArrowUp',
+  Down: 'ArrowDown',
+  Left: 'ArrowLeft',
+  Right: 'ArrowRight',
+  Delete: 'Delete',
+  Insert: 'Insert',
+  KP_Add: 'NumpadAdd',
+  KP_Subtract: 'NumpadSubtract',
+  KP_Multiply: 'NumpadMultiply',
+  KP_Divide: 'NumpadDivide',
+  KP_Enter: 'NumpadEnter'
+}
+
+export function parseKey(input: string): KeyChord {
+  const parts = input
+    .split('+')
+    .map((part) => part.trim())
+    .filter(Boolean)
+  if (parts.length === 0) {
+    throw new RuntimeClientError('invalid_argument', 'key must not be empty')
+  }
+
+  const keyPart = parts.at(-1)!
+  const modifiers = parts.slice(0, -1).map(parseModifier)
+  return {
+    key: parseBaseKey(keyPart),
+    modifiers: dedupeModifiers(modifiers)
+  }
+}
+
+function parseModifier(input: string): string {
+  const platformModifier = PLATFORM_MODIFIER_NAMES[input.toLowerCase()]
+  if (platformModifier) {
+    return platformModifier()
+  }
+  const modifier = MODIFIER_NAMES[input.toLowerCase()]
+  if (!modifier) {
+    throw new RuntimeClientError('invalid_argument', `unknown modifier '${input}'`)
+  }
+  return modifier
+}
+
+function parseBaseKey(input: string): string {
+  const mapped = KEY_NAMES[input]
+  if (mapped) {
+    return mapped
+  }
+  if (/^F([1-9]|1[0-9]|2[0-4])$/.test(input)) {
+    return input
+  }
+  const keypadDigit = input.match(/^KP_([0-9])$/)
+  if (keypadDigit) {
+    return `Numpad${keypadDigit[1]}`
+  }
+  if (isPrintableAscii(input)) {
+    return input
+  }
+  throw new RuntimeClientError('invalid_argument', `unknown key '${input}'`)
+}
+
+function isPrintableAscii(input: string): boolean {
+  if (input.length !== 1) {
+    return false
+  }
+  const code = input.charCodeAt(0)
+  return code >= 0x20 && code <= 0x7e
+}
+
+function dedupeModifiers(modifiers: string[]): string[] {
+  return [...new Set(modifiers)]
+}

src/main/computer/macos-app-catalog.ts

@@ -0,0 +1,121 @@
+import { execFile } from 'child_process'
+import type { ComputerAppInfo } from '../../shared/runtime-types'
+
+type MacOSRunningApp = ComputerAppInfo & {
+  activationPolicy: number
+  isFrontmost: boolean
+}
+
+const JXA_LIST_RUNNING_APPS = `
+ObjC.import('AppKit');
+const apps = $.NSWorkspace.sharedWorkspace.runningApplications;
+const out = [];
+for (let i = 0; i < apps.count; i += 1) {
+  const app = apps.objectAtIndex(i);
+  if (app.isTerminated) continue;
+  const name = ObjC.unwrap(app.localizedName) || '';
+  const bundleId = ObjC.unwrap(app.bundleIdentifier) || null;
+  const pid = Number(app.processIdentifier);
+  const activationPolicy = Number(app.activationPolicy);
+  if (!name || !Number.isInteger(pid) || pid <= 0) continue;
+  out.push({
+    name,
+    bundleId,
+    pid,
+    activationPolicy,
+    isFrontmost: Boolean(app.active)
+  });
+}
+JSON.stringify(out);
+`
+
+export async function listMacOSApps(): Promise<ComputerAppInfo[]> {
+  const { stdout } = await execFilePromise(
+    'osascript',
+    ['-l', 'JavaScript', '-e', JXA_LIST_RUNNING_APPS],
+    {
+      encoding: 'utf8',
+      timeout: 5000,
+      maxBuffer: 1024 * 1024
+    }
+  )
+  const apps = parseMacOSApps(stdout).filter(hasLivePid)
+  return dedupeApps(apps.filter(isUserFacingApp)).map(({ name, bundleId, pid }) => ({
+    name,
+    bundleId,
+    pid
+  }))
+}
+
+function parseMacOSApps(stdout: string): MacOSRunningApp[] {
+  const parsed: unknown = JSON.parse(stdout)
+  if (!Array.isArray(parsed)) {
+    return []
+  }
+  return parsed.flatMap((value): MacOSRunningApp[] => {
+    if (!value || typeof value !== 'object') {
+      return []
+    }
+    const record = value as Record<string, unknown>
+    const name = typeof record.name === 'string' ? record.name : ''
+    const bundleId = typeof record.bundleId === 'string' ? record.bundleId : null
+    const pid = typeof record.pid === 'number' && Number.isInteger(record.pid) ? record.pid : 0
+    const activationPolicy =
+      typeof record.activationPolicy === 'number' && Number.isInteger(record.activationPolicy)
+        ? record.activationPolicy
+        : -1
+    const isFrontmost = record.isFrontmost === true
+    return name && pid > 0 ? [{ name, bundleId, pid, activationPolicy, isFrontmost }] : []
+  })
+}
+
+function isUserFacingApp(app: MacOSRunningApp): boolean {
+  return app.activationPolicy === 0
+}
+
+function hasLivePid(app: MacOSRunningApp): boolean {
+  try {
+    process.kill(app.pid, 0)
+    return true
+  } catch {
+    // NSWorkspace can briefly retain terminated apps; filter them before provider calls.
+    return false
+  }
+}
+
+function dedupeApps(apps: MacOSRunningApp[]): MacOSRunningApp[] {
+  const seen = new Set<string>()
+  const sorted = [...apps].sort((a, b) => {
+    if (a.isFrontmost !== b.isFrontmost) {
+      return a.isFrontmost ? -1 : 1
+    }
+    return a.name.localeCompare(b.name)
+  })
+  const result: MacOSRunningApp[] = []
+  for (const app of sorted) {
+    const key = app.bundleId ? `bundle:${app.bundleId.toLowerCase()}` : `pid:${app.pid}`
+    if (seen.has(key)) {
+      continue
+    }
+    seen.add(key)
+    result.push(app)
+  }
+  return result
+}
+
+function execFilePromise(
+  file: string,
+  args: string[],
+  options: { encoding: 'utf8'; timeout: number; maxBuffer: number }
+): Promise<{ stdout: string }> {
+  return new Promise((resolve, reject) => {
+    execFile(file, args, options, (error, stdout, stderr) => {
+      if (error) {
+        const detail = typeof stderr === 'string' && stderr.trim() ? `: ${stderr.trim()}` : ''
+        reject(new Error(`${error.message}${detail}`))
+        return
+      }
+      resolve({ stdout })
+    })
+  })
+}

src/main/computer/macos-computer-use-permissions.test.ts

@@ -0,0 +1,180 @@
+import { spawn, spawnSync } from 'child_process'
+import { existsSync, readFileSync, rmSync } from 'fs'
+import type * as Fs from 'fs'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { openComputerUsePermissions } from './macos-computer-use-permissions'
+
+const resolveHelperAppPathMock = vi.hoisted(() => vi.fn())
+
+vi.mock('child_process', () => ({
+  spawn: vi.fn(() => ({ unref: vi.fn() })),
+  spawnSync: vi.fn()
+}))
+
+vi.mock('fs', async (importOriginal) => {
+  const actual = (await importOriginal()) as typeof Fs
+  return {
+    ...actual,
+    existsSync: vi.fn(),
+    readFileSync: vi.fn(),
+    rmSync: vi.fn()
+  }
+})
+
+vi.mock('./macos-native-provider-paths', () => ({
+  resolveMacOSComputerUseAppPath: resolveHelperAppPathMock
+}))
+
+describe('openComputerUsePermissions', () => {
+  const originalPlatform = process.platform
+
+  beforeEach(() => {
+    vi.mocked(spawn).mockClear()
+    vi.mocked(spawnSync).mockClear()
+    vi.mocked(existsSync).mockReset()
+    vi.mocked(readFileSync).mockReset()
+    vi.mocked(rmSync).mockReset()
+    resolveHelperAppPathMock.mockReset()
+    mockPermissionStatus('{"accessibility":"granted","screenshots":"granted"}')
+    setPlatform('darwin')
+  })
+
+  afterEach(() => {
+    setPlatform(originalPlatform)
+  })
+
+  it('does not launch the setup helper when all permissions are granted', () => {
+    resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+
+    expect(openComputerUsePermissions()).toEqual({
+      platform: 'darwin',
+      helperAppPath: '/Applications/Orca Computer Use.app',
+      permissionId: undefined,
+      openedSettings: false,
+      launchedHelper: false,
+      permissions: [
+        { id: 'accessibility', status: 'granted' },
+        { id: 'screenshots', status: 'granted' }
+      ],
+      nextStep: null
+    })
+    expect(spawn).not.toHaveBeenCalled()
+  })
+
+  it('launches the helper app in permissions mode', () => {
+    resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+    mockPermissionStatus('{"accessibility":"granted","screenshots":"not-granted"}')
+
+    expect(openComputerUsePermissions()).toEqual({
+      platform: 'darwin',
+      helperAppPath: '/Applications/Orca Computer Use.app',
+      permissionId: undefined,
+      openedSettings: false,
+      launchedHelper: true,
+      permissions: [
+        { id: 'accessibility', status: 'granted' },
+        { id: 'screenshots', status: 'not-granted' }
+      ],
+      nextStep: 'Grant Screen Recording to Orca Computer Use, then retry get-app-state.'
+    })
+    expect(spawn).toHaveBeenCalledTimes(1)
+    expect(spawnSync).toHaveBeenCalledWith(
+      '/usr/bin/pkill',
+      ['-f', 'orca-computer-use-macos --permission'],
+      { stdio: 'ignore' }
+    )
+    expect(spawnSync).toHaveBeenCalledWith(
+      '/usr/bin/pkill',
+      ['-f', 'orca-computer-use-macos --permissions'],
+      { stdio: 'ignore' }
+    )
+    expect(spawn).toHaveBeenCalledWith(
+      '/usr/bin/open',
+      ['-n', '/Applications/Orca Computer Use.app', '--args', '--permissions'],
+      { detached: true, stdio: 'ignore' }
+    )
+  })
+
+  it('launches a targeted permission helper flow', () => {
+    resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+    mockPermissionStatus('{"accessibility":"not-granted","screenshots":"not-granted"}')
+
+    expect(openComputerUsePermissions('accessibility')).toEqual({
+      platform: 'darwin',
+      helperAppPath: '/Applications/Orca Computer Use.app',
+      permissionId: 'accessibility',
+      openedSettings: true,
+      launchedHelper: true,
+      permissions: [
+        { id: 'accessibility', status: 'not-granted' },
+        { id: 'screenshots', status: 'not-granted' }
+      ],
+      nextStep: 'Grant Accessibility to Orca Computer Use, then retry get-app-state.'
+    })
+    expect(spawn).toHaveBeenCalledWith(
+      '/usr/bin/open',
+      ['-n', '/Applications/Orca Computer Use.app', '--args', '--permission', 'accessibility'],
+      { detached: true, stdio: 'ignore' }
+    )
+  })
+
+  it('returns a no-op result on non-macOS platforms', () => {
+    setPlatform('linux')
+
+    expect(openComputerUsePermissions()).toEqual({
+      platform: 'linux',
+      helperAppPath: null,
+      permissionId: undefined,
+      openedSettings: false,
+      launchedHelper: false,
+      permissions: [
+        { id: 'accessibility', status: 'unsupported' },
+        { id: 'screenshots', status: 'unsupported' }
+      ],
+      nextStep: null
+    })
+    expect(spawn).not.toHaveBeenCalled()
+  })
+
+  it('throws when the helper app is missing on macOS', () => {
+    resolveHelperAppPathMock.mockReturnValue(null)
+
+    expect(() => openComputerUsePermissions()).toThrow('Orca Computer Use.app was not found')
+  })
+
+  it('reads permission status through the helper app bundle', async () => {
+    const { getComputerUsePermissionStatus } = await import('./macos-computer-use-permissions')
+    resolveHelperAppPathMock.mockReturnValue('/Applications/Orca Computer Use.app')
+    mockPermissionStatus('{"accessibility":"granted","screenshots":"not-granted"}')
+
+    expect(getComputerUsePermissionStatus()).toEqual({
+      platform: 'darwin',
+      permissions: [
+        { id: 'accessibility', status: 'granted' },
+        { id: 'screenshots', status: 'not-granted' }
+      ]
+    })
+    expect(spawnSync).toHaveBeenCalledWith(
+      '/usr/bin/open',
+      [
+        '-n',
+        '/Applications/Orca Computer Use.app',
+        '--args',
+        '--permission-status-file',
+        expect.stringContaining('/status.json')
+      ],
+      { stdio: 'ignore' }
+    )
+    expect(rmSync).toHaveBeenCalled()
+  })
+})
+
+function mockPermissionStatus(json: string): void {
+  vi.mocked(spawnSync).mockReturnValue({} as ReturnType<typeof spawnSync>)
+  vi.mocked(existsSync).mockReturnValue(true)
+  vi.mocked(readFileSync).mockReturnValue(json)
+}
+
+function setPlatform(platform: NodeJS.Platform): void {
+  Object.defineProperty(process, 'platform', { configurable: true, value: platform })
+}

src/main/computer/macos-computer-use-permissions.ts

@@ -0,0 +1,153 @@
+import { spawn, spawnSync } from 'child_process'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import { RuntimeClientError } from './runtime-client-error'
+import { resolveMacOSComputerUseAppPath } from './macos-native-provider-paths'
+import type {
+  ComputerUsePermissionId,
+  ComputerUsePermissionSetupResult,
+  ComputerUsePermissionStatus,
+  ComputerUsePermissionStatusResult
+} from '../../shared/computer-use-permissions-types'
+
+export function openComputerUsePermissions(
+  permissionId?: ComputerUsePermissionId
+): ComputerUsePermissionSetupResult {
+  if (process.platform !== 'darwin') {
+    return {
+      platform: process.platform,
+      helperAppPath: null,
+      permissionId,
+      openedSettings: false,
+      launchedHelper: false,
+      permissions: [
+        { id: 'accessibility', status: 'unsupported' },
+        { id: 'screenshots', status: 'unsupported' }
+      ],
+      nextStep: null
+    }
+  }
+
+  const helperAppPath = resolveMacOSComputerUseAppPath()
+  if (!helperAppPath) {
+    throw new RuntimeClientError('accessibility_error', 'Orca Computer Use.app was not found')
+  }
+  const status = getComputerUsePermissionStatus()
+  const nextStep = nextPermissionStep(status.permissions)
+
+  if (!permissionId && !nextStep) {
+    return {
+      platform: process.platform,
+      helperAppPath,
+      permissionId,
+      openedSettings: false,
+      launchedHelper: false,
+      permissions: status.permissions,
+      nextStep
+    }
+  }
+
+  closeExistingPermissionHelpers()
+  const helperArgs = permissionId ? ['--permission', permissionId] : ['--permissions']
+  const helper = spawn('/usr/bin/open', ['-n', helperAppPath, '--args', ...helperArgs], {
+    detached: true,
+    stdio: 'ignore'
+  })
+  helper.unref()
+
+  return {
+    platform: process.platform,
+    helperAppPath,
+    permissionId,
+    openedSettings: permissionId !== undefined,
+    launchedHelper: true,
+    permissions: status.permissions,
+    nextStep
+  }
+}
+
+function closeExistingPermissionHelpers(): void {
+  spawnSync('/usr/bin/pkill', ['-f', 'orca-computer-use-macos --permission'], {
+    stdio: 'ignore'
+  })
+  spawnSync('/usr/bin/pkill', ['-f', 'orca-computer-use-macos --permissions'], {
+    stdio: 'ignore'
+  })
+}
+
+export function getComputerUsePermissionStatus(): ComputerUsePermissionStatusResult {
+  if (process.platform !== 'darwin') {
+    return {
+      platform: process.platform,
+      permissions: [
+        { id: 'accessibility', status: 'unsupported' },
+        { id: 'screenshots', status: 'unsupported' }
+      ]
+    }
+  }
+
+  const helperAppPath = resolveMacOSComputerUseAppPath()
+  if (!helperAppPath) {
+    throw new RuntimeClientError('accessibility_error', 'Orca Computer Use.app was not found')
+  }
+
+  const raw = readPermissionStatusFromHelperApp(helperAppPath)
+
+  return {
+    platform: process.platform,
+    permissions: [
+      { id: 'accessibility', status: raw.accessibility ?? 'not-granted' },
+      { id: 'screenshots', status: raw.screenshots ?? 'not-granted' }
+    ]
+  }
+}
+
+function readPermissionStatusFromHelperApp(
+  helperAppPath: string
+): Partial<Record<ComputerUsePermissionId, ComputerUsePermissionStatus>> {
+  const directory = mkdtempSync(join(tmpdir(), 'orca-computer-permissions-'))
+  const statusPath = join(directory, 'status.json')
+  try {
+    // Why: TCC can attribute direct executable probes to the parent shell;
+    // launch the app bundle so status uses the same identity as real actions.
+    const result = spawnSync(
+      '/usr/bin/open',
+      ['-n', helperAppPath, '--args', '--permission-status-file', statusPath],
+      { stdio: 'ignore' }
+    )
+    if (result.error) {
+      throw result.error
+    }
+    waitForStatusFile(statusPath)
+    return JSON.parse(readFileSync(statusPath, 'utf8')) as Partial<
+      Record<ComputerUsePermissionId, ComputerUsePermissionStatus>
+    >
+  } finally {
+    rmSync(directory, { force: true, recursive: true })
+  }
+}
+
+function waitForStatusFile(statusPath: string): void {
+  const deadline = Date.now() + 3_000
+  while (Date.now() < deadline) {
+    if (existsSync(statusPath)) {
+      return
+    }
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50)
+  }
+  throw new RuntimeClientError(
+    'action_timeout',
+    'Orca Computer Use.app did not report permission status'
+  )
+}
+
+function nextPermissionStep(
+  permissions: ComputerUsePermissionStatusResult['permissions']
+): string | null {
+  const missing = permissions.find((permission) => permission.status !== 'granted')
+  if (!missing) {
+    return null
+  }
+  return `Grant ${missing.id === 'accessibility' ? 'Accessibility' : 'Screen Recording'} to Orca Computer Use, then retry get-app-state.`
+}

src/main/computer/macos-native-provider-client.ts

@@ -0,0 +1,282 @@
+/* eslint-disable max-lines -- Why: the macOS provider transport owns one lifecycle across stdio fallback and helper-app socket mode. */
+import { spawn } from 'child_process'
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'fs'
+import type net from 'net'
+import { release, tmpdir } from 'os'
+import { join } from 'path'
+import { randomUUID } from 'crypto'
+import type {
+  ComputerActionResult,
+  ComputerListAppsResult,
+  ComputerListWindowsResult,
+  ComputerProviderCapabilities,
+  ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import {
+  assertMacOSProviderCapability,
+  REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION,
+  type NativeActionMethod,
+  type NativeMethod,
+  type NativeResponse,
+  type PendingNativeRequest,
+  writeNativeProviderLine
+} from './macos-native-provider-contract'
+import { resolveMacOSComputerUseAppPath } from './macos-native-provider-paths'
+import { connectMacOSProviderSocket } from './macos-native-provider-socket'
+import { RuntimeClientError } from './runtime-client-error'
+
+const REQUEST_TIMEOUT_MS = 60_000
+const HELPER_CONNECT_TIMEOUT_MS = 10_000
+
+export function shouldUseMacOSNativeProvider(): boolean {
+  return (
+    process.platform === 'darwin' && isMacOS14OrNewer() && resolveMacOSComputerUseAppPath() !== null
+  )
+}
+
+export class MacOSNativeProviderClient {
+  private socket: net.Socket | null = null
+  private socketStartPromise: Promise<net.Socket> | null = null
+  private socketPath: string | null = null
+  private socketDirectory: string | null = null
+  private socketTokenPath: string | null = null
+  private socketToken: string | null = null
+  private nextId = 1
+  private pending = new Map<number, PendingNativeRequest>()
+  private socketBuffer = ''
+  private providerCapabilities: ComputerProviderCapabilities | null = null
+  async listApps(): Promise<ComputerListAppsResult> {
+    return (await this.call('listApps', {})) as ComputerListAppsResult
+  }
+  async capabilities(): Promise<ComputerProviderCapabilities> {
+    await this.ensureCompatible()
+    return this.providerCapabilities!
+  }
+  async listWindows(params: unknown): Promise<ComputerListWindowsResult> {
+    await this.ensureCapability('windows', 'list')
+    return (await this.call('listWindows', params)) as ComputerListWindowsResult
+  }
+  async snapshot(params: unknown): Promise<ComputerSnapshotResult> {
+    return (await this.call('getAppState', params)) as ComputerSnapshotResult
+  }
+  async action(method: NativeActionMethod, params: unknown): Promise<ComputerActionResult> {
+    return (await this.call(method, params)) as ComputerActionResult
+  }
+  shutdown(): void {
+    const socket = this.socket
+    const token = this.socketToken
+    this.socket = null
+    this.socketStartPromise = null
+    this.providerCapabilities = null
+    if (socket && !socket.destroyed) {
+      const id = this.nextId++
+      socket.write(`${JSON.stringify({ id, method: 'terminate', params: {}, token })}\n`)
+      socket.end()
+    }
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timer)
+      pending.reject(
+        new RuntimeClientError('accessibility_error', 'native macOS provider shut down')
+      )
+      this.pending.delete(id)
+    }
+    this.cleanupSocketDirectory()
+  }
+  private async call(method: NativeMethod, params: unknown): Promise<unknown> {
+    if (method !== 'handshake') {
+      await this.ensureCompatible()
+    }
+    return await this.send(method, params)
+  }
+  private async send(method: NativeMethod, params: unknown): Promise<unknown> {
+    const id = this.nextId++
+    const helperAppPath = resolveMacOSComputerUseAppPath()
+    if (!helperAppPath) {
+      throw new RuntimeClientError('accessibility_error', 'Orca Computer Use.app was not found')
+    }
+    const transport = await this.ensureSocketStarted(helperAppPath)
+    const token = this.socketToken
+    const line = `${JSON.stringify({ id, method, params, token })}\n`
+    const result = new Promise<unknown>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id)
+        this.shutdown()
+        reject(
+          new RuntimeClientError('action_timeout', `native macOS provider ${method} timed out`)
+        )
+      }, REQUEST_TIMEOUT_MS)
+
+      this.pending.set(id, { resolve, reject, timer })
+    })
+    try {
+      await writeNativeProviderLine(transport, line)
+    } catch (error) {
+      const pending = this.pending.get(id)
+      if (pending) {
+        clearTimeout(pending.timer)
+        this.pending.delete(id)
+      }
+      throw new RuntimeClientError(
+        'accessibility_error',
+        error instanceof Error ? error.message : String(error)
+      )
+    }
+    return await result
+  }
+  private async ensureCompatible(): Promise<void> {
+    if (this.providerCapabilities) {
+      return
+    }
+    const capabilities = await this.readCapabilities()
+    if (capabilities.protocolVersion === REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION) {
+      this.providerCapabilities = capabilities
+      return
+    }
+    this.shutdown()
+    const restarted = await this.readCapabilities()
+    if (restarted.protocolVersion !== REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION) {
+      throw new RuntimeClientError(
+        'provider_incompatible',
+        `native macOS provider protocol ${restarted.protocolVersion} is incompatible with required protocol ${REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION}`
+      )
+    }
+    this.providerCapabilities = restarted
+  }
+  private async readCapabilities(): Promise<ComputerProviderCapabilities> {
+    return (await this.send('handshake', {})) as ComputerProviderCapabilities
+  }
+  private async ensureCapability(
+    group: keyof ComputerProviderCapabilities['supports'],
+    capability: string
+  ): Promise<void> {
+    await this.ensureCompatible()
+    if (assertMacOSProviderCapability(this.providerCapabilities, group, capability)) {
+      return
+    }
+    throw new RuntimeClientError(
+      'unsupported_capability',
+      `native macOS provider does not support ${String(group)}.${capability}`
+    )
+  }
+  private async ensureSocketStarted(helperAppPath: string): Promise<net.Socket> {
+    if (this.socket && !this.socket.destroyed) {
+      return this.socket
+    }
+    if (this.socketStartPromise) {
+      return await this.socketStartPromise
+    }
+    this.socketStartPromise = this.startSocket(helperAppPath)
+    try {
+      return await this.socketStartPromise
+    } finally {
+      this.socketStartPromise = null
+    }
+  }
+  private async startSocket(helperAppPath: string): Promise<net.Socket> {
+    this.socketDirectory = mkdtempSync(join(tmpdir(), 'orca-computer-use-'))
+    chmodSync(this.socketDirectory, 0o700)
+    this.socketPath = join(this.socketDirectory, 'provider.sock')
+    this.socketToken = randomUUID()
+    this.socketTokenPath = join(this.socketDirectory, 'provider.token')
+    writeFileSync(this.socketTokenPath, this.socketToken, { encoding: 'utf8', mode: 0o600 })
+    // Why: macOS TCC attaches Accessibility/Screen Recording to the helper
+    // app bundle; direct child-process stdio cannot reliably read AX windows.
+    const opener = spawn(
+      '/usr/bin/open',
+      [
+        '-n',
+        helperAppPath,
+        '--args',
+        '--agent',
+        this.socketPath,
+        '--token-file',
+        this.socketTokenPath
+      ],
+      { detached: true, stdio: 'ignore' }
+    )
+    opener.unref()
+    try {
+      const socket = await connectMacOSProviderSocket(this.socketPath, HELPER_CONNECT_TIMEOUT_MS)
+      socket.setEncoding('utf8')
+      socket.on('data', (chunk: string) => this.handleSocketData(chunk))
+      socket.on('close', () => this.handleSocketClose())
+      socket.on('error', (error) => this.handleTransportError(error))
+      this.socket = socket
+      return socket
+    } catch (error) {
+      this.cleanupSocketDirectory()
+      this.socketPath = null
+      this.socketTokenPath = null
+      this.socketToken = null
+      throw error
+    }
+  }
+  private handleSocketData(chunk: string): void {
+    this.socketBuffer += chunk
+    this.socketBuffer = this.consumeLines(this.socketBuffer)
+  }
+  private consumeLines(buffer: string): string {
+    let remaining = buffer
+    while (true) {
+      const newline = remaining.indexOf('\n')
+      if (newline < 0) {
+        return remaining
+      }
+      const line = remaining.slice(0, newline)
+      remaining = remaining.slice(newline + 1)
+      if (line.trim()) {
+        this.handleLine(line)
+      }
+    }
+  }
+  private handleLine(line: string): void {
+    let response: NativeResponse
+    try {
+      response = JSON.parse(line) as NativeResponse
+    } catch {
+      return
+    }
+    const pending = this.pending.get(response.id)
+    if (!pending) {
+      return
+    }
+    clearTimeout(pending.timer)
+    this.pending.delete(response.id)
+    if (response.ok) {
+      pending.resolve(response.result)
+      return
+    }
+    pending.reject(new RuntimeClientError(response.error.code, response.error.message))
+  }
+  private handleSocketClose(): void {
+    this.socket = null
+    this.cleanupSocketDirectory()
+    this.rejectPending(
+      new RuntimeClientError('accessibility_error', 'native macOS helper app connection closed')
+    )
+  }
+  private handleTransportError(error: Error): void {
+    this.rejectPending(new RuntimeClientError('accessibility_error', error.message))
+  }
+  private cleanupSocketDirectory(): void {
+    if (!this.socketDirectory) {
+      return
+    }
+    rmSync(this.socketDirectory, { recursive: true, force: true })
+    this.socketDirectory = null
+    this.socketPath = null
+    this.socketTokenPath = null
+  }
+  private rejectPending(error: Error): void {
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timer)
+      pending.reject(error)
+      this.pending.delete(id)
+    }
+  }
+}
+
+function isMacOS14OrNewer(): boolean {
+  const darwinMajor = Number.parseInt(release().split('.')[0] ?? '', 10)
+  return Number.isFinite(darwinMajor) && darwinMajor >= 23
+}

src/main/computer/macos-native-provider-contract.ts

@@ -0,0 +1,56 @@
+import type net from 'net'
+import type { ComputerProviderCapabilities } from '../../shared/runtime-types'
+
+export type NativeMethod =
+  | 'handshake'
+  | 'listApps'
+  | 'listWindows'
+  | 'getAppState'
+  | 'click'
+  | 'performSecondaryAction'
+  | 'scroll'
+  | 'drag'
+  | 'typeText'
+  | 'pressKey'
+  | 'hotkey'
+  | 'pasteText'
+  | 'setValue'
+  | 'terminate'
+
+export type NativeActionMethod = Exclude<
+  NativeMethod,
+  'handshake' | 'listApps' | 'listWindows' | 'getAppState' | 'terminate'
+>
+
+export type NativeResponse =
+  | { id: number; ok: true; result: unknown }
+  | { id: number; ok: false; error: { code: string; message: string } }
+
+export type PendingNativeRequest = {
+  resolve: (value: unknown) => void
+  reject: (error: Error) => void
+  timer: NodeJS.Timeout
+}
+
+export const REQUIRED_MACOS_PROVIDER_PROTOCOL_VERSION = 1
+
+export function assertMacOSProviderCapability(
+  capabilities: ComputerProviderCapabilities | null,
+  group: keyof ComputerProviderCapabilities['supports'],
+  capability: string
+): boolean {
+  const groupCapabilities = capabilities?.supports[group] as Record<string, boolean> | undefined
+  return groupCapabilities?.[capability] === true
+}
+
+export function writeNativeProviderLine(transport: net.Socket, line: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    transport.write(line, (error) => {
+      if (error) {
+        reject(error)
+        return
+      }
+      resolve()
+    })
+  })
+}

src/main/computer/macos-native-provider-paths.ts

@@ -0,0 +1,44 @@
+import { existsSync } from 'fs'
+import { join, resolve } from 'path'
+
+export function resolveMacOSComputerUseAppPath(): string | null {
+  const override = process.env.ORCA_COMPUTER_MACOS_HELPER_APP_PATH
+  if (override && existsSync(override)) {
+    return override
+  }
+
+  const packaged = [join(process.resourcesPath ?? '', 'Orca Computer Use.app')]
+  const dev = [
+    join(process.cwd(), 'native/computer-use-macos/.build/release/Orca Computer Use.app'),
+    resolve(__dirname, '../../native/computer-use-macos/.build/release/Orca Computer Use.app')
+  ]
+  const candidates = process.resourcesPath ? [...packaged, ...dev] : dev
+
+  return candidates.find((candidate) => candidate && existsSync(candidate)) ?? null
+}
+
+export function resolveMacOSComputerUseExecutablePath(): string | null {
+  const appPath = resolveMacOSComputerUseAppPath()
+  if (!appPath) {
+    return null
+  }
+  const executablePath = join(appPath, 'Contents', 'MacOS', 'orca-computer-use-macos')
+  return existsSync(executablePath) ? executablePath : null
+}
+
+export function resolveMacOSNativeProviderPath(): string | null {
+  const override = process.env.ORCA_COMPUTER_MACOS_PROVIDER_PATH
+  if (override && existsSync(override)) {
+    return override
+  }
+
+  const packaged = [join(process.resourcesPath ?? '', 'computer-use-macos/orca-computer-use-macos')]
+  const dev = [
+    join(process.cwd(), 'native/computer-use-macos/.build/debug/orca-computer-use-macos'),
+    join(process.cwd(), 'native/computer-use-macos/.build/release/orca-computer-use-macos'),
+    resolve(__dirname, '../../native/computer-use-macos/.build/debug/orca-computer-use-macos')
+  ]
+  const candidates = process.resourcesPath ? [...packaged, ...dev] : dev
+
+  return candidates.find((candidate) => candidate && existsSync(candidate)) ?? null
+}

src/main/computer/macos-native-provider-socket.ts

@@ -0,0 +1,37 @@
+import net from 'net'
+import { RuntimeClientError } from './runtime-client-error'
+
+export async function connectMacOSProviderSocket(
+  socketPath: string,
+  timeoutMs: number
+): Promise<net.Socket> {
+  const deadline = Date.now() + timeoutMs
+  let lastError: Error | null = null
+  while (Date.now() < deadline) {
+    try {
+      return await connectSocket(socketPath)
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error))
+      await new Promise((resolve) => setTimeout(resolve, 100))
+    }
+  }
+  throw new RuntimeClientError(
+    'action_timeout',
+    `native macOS helper app did not open its socket: ${lastError?.message ?? 'timed out'}`
+  )
+}
+
+function connectSocket(socketPath: string): Promise<net.Socket> {
+  return new Promise((resolve, reject) => {
+    const socket = net.createConnection(socketPath)
+    const onError = (error: Error) => {
+      socket.destroy()
+      reject(error)
+    }
+    socket.once('error', onError)
+    socket.once('connect', () => {
+      socket.off('error', onError)
+      resolve(socket)
+    })
+  })
+}

src/main/computer/permissions.ts

@@ -0,0 +1,51 @@
+import { Notification, shell, systemPreferences } from 'electron'
+
+const ACCESSIBILITY_SETTINGS_URL =
+  'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility'
+const DEFAULT_ACCESSIBILITY_INSTRUCTIONS =
+  'System Settings -> Privacy & Security -> Accessibility -> enable Orca'
+
+const activePermissionNotifications = new Set<Notification>()
+
+/** Probe accessibility permissions; lazy -- invoked only on first failure path. */
+export async function checkAccessibilityPermission(): Promise<{
+  ok: boolean
+  instructions?: string
+}> {
+  if (process.platform !== 'darwin') {
+    return { ok: true }
+  }
+
+  try {
+    const ok = systemPreferences.isTrustedAccessibilityClient(false)
+    return ok ? { ok: true } : { ok: false, instructions: DEFAULT_ACCESSIBILITY_INSTRUCTIONS }
+  } catch {
+    return { ok: false, instructions: DEFAULT_ACCESSIBILITY_INSTRUCTIONS }
+  }
+}
+
+/** Surface a notification through Orca's existing notification system (do not duplicate UI). */
+export function notifyPermissionRequired(instructions: string): void {
+  if (!Notification.isSupported()) {
+    return
+  }
+
+  const notification = new Notification({
+    title: 'Accessibility permission required',
+    body: instructions
+  })
+  activePermissionNotifications.add(notification)
+
+  const release = (): void => {
+    activePermissionNotifications.delete(notification)
+  }
+  notification.on('close', release)
+  notification.on('click', () => {
+    release()
+    if (process.platform === 'darwin') {
+      void shell.openExternal(ACCESSIBILITY_SETTINGS_URL)
+    }
+  })
+  setTimeout(release, 5 * 60 * 1000)
+  notification.show()
+}

src/main/computer/runtime-client-error.ts

@@ -0,0 +1,9 @@
+export class RuntimeClientError extends Error {
+  readonly code: string
+
+  constructor(code: string, message: string) {
+    super(message)
+    this.name = 'RuntimeClientError'
+    this.code = code
+  }
+}

src/main/computer/sidecar-client.ts

@@ -0,0 +1,227 @@
+import { fork, type ChildProcess } from 'child_process'
+import { join } from 'path'
+import type {
+  ComputerActionResult,
+  ComputerListAppsResult,
+  ComputerListWindowsResult,
+  ComputerProviderCapabilities,
+  ComputerSnapshotResult
+} from '../../shared/runtime-types'
+import { RuntimeClientError } from './runtime-client-error'
+
+type ComputerSidecarMethod =
+  | 'capabilities'
+  | 'listApps'
+  | 'listWindows'
+  | 'getAppState'
+  | 'click'
+  | 'performSecondaryAction'
+  | 'scroll'
+  | 'drag'
+  | 'typeText'
+  | 'pressKey'
+  | 'hotkey'
+  | 'pasteText'
+  | 'setValue'
+
+type ComputerSidecarRequest = {
+  id: number
+  method: ComputerSidecarMethod
+  params: unknown
+}
+
+type ComputerSidecarResponse =
+  | { id: number; ok: true; result: unknown }
+  | { id: number; ok: false; error: { code: string; message: string } }
+
+type PendingRequest = {
+  resolve: (value: unknown) => void
+  reject: (error: Error) => void
+  timer: NodeJS.Timeout
+}
+
+const REQUEST_TIMEOUT_MS = 60_000
+let sidecar: ComputerSidecarProcess | null = null
+
+export function shouldUseComputerSidecar(): boolean {
+  return (
+    (process.platform === 'darwin' ||
+      process.platform === 'linux' ||
+      process.platform === 'win32') &&
+    typeof process.versions.electron === 'string' &&
+    process.env.ORCA_COMPUTER_SIDECAR !== '1'
+  )
+}
+
+export async function callComputerSidecarListApps(): Promise<ComputerListAppsResult> {
+  return (await getComputerSidecar().call('listApps', {})) as ComputerListAppsResult
+}
+
+export async function callComputerSidecarCapabilities(): Promise<ComputerProviderCapabilities> {
+  return (await getComputerSidecar().call('capabilities', {})) as ComputerProviderCapabilities
+}
+
+export async function callComputerSidecarListWindows(
+  params: unknown
+): Promise<ComputerListWindowsResult> {
+  return (await getComputerSidecar().call('listWindows', params)) as ComputerListWindowsResult
+}
+
+export async function callComputerSidecarSnapshot(
+  params: unknown
+): Promise<ComputerSnapshotResult> {
+  return (await getComputerSidecar().call('getAppState', params)) as ComputerSnapshotResult
+}
+
+export async function callComputerSidecarAction(
+  method: Exclude<
+    ComputerSidecarMethod,
+    'capabilities' | 'listApps' | 'listWindows' | 'getAppState'
+  >,
+  params: unknown
+): Promise<ComputerActionResult> {
+  return (await getComputerSidecar().call(method, params)) as ComputerActionResult
+}
+
+export function resetComputerSidecarForTest(): void {
+  sidecar?.shutdown()
+  sidecar = null
+}
+
+function getComputerSidecar(): ComputerSidecarProcess {
+  if (!sidecar) {
+    sidecar = new ComputerSidecarProcess(getComputerSidecarEntryPath())
+  }
+  return sidecar
+}
+
+function getComputerSidecarEntryPath(): string {
+  const app = loadElectronApp()
+  const appPath = app?.getAppPath() ?? process.cwd()
+  const isPackaged = app?.isPackaged ?? false
+  // Why: packaged sidecars must be forked from app.asar.unpacked because
+  // ELECTRON_RUN_AS_NODE bypasses Electron's asar require integration.
+  const basePath = isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath
+  return join(basePath, 'out', 'main', 'computer-sidecar.js')
+}
+
+function loadElectronApp(): { getAppPath(): string; isPackaged: boolean } | null {
+  try {
+    return require('electron').app
+  } catch {
+    return null
+  }
+}
+
+class ComputerSidecarProcess {
+  private child: ChildProcess | null = null
+  private nextId = 1
+  private pending = new Map<number, PendingRequest>()
+
+  constructor(private readonly entryPath: string) {}
+
+  call(method: ComputerSidecarMethod, params: unknown): Promise<unknown> {
+    const child = this.ensureStarted()
+    const id = this.nextId++
+    const request: ComputerSidecarRequest = { id, method, params }
+
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id)
+        this.shutdown()
+        reject(new RuntimeClientError('action_timeout', `computer sidecar ${method} timed out`))
+      }, REQUEST_TIMEOUT_MS)
+
+      this.pending.set(id, { resolve, reject, timer })
+      child.send?.(request, (error) => {
+        if (!error) {
+          return
+        }
+        clearTimeout(timer)
+        this.pending.delete(id)
+        reject(new RuntimeClientError('accessibility_error', error.message))
+      })
+    })
+  }
+
+  shutdown(): void {
+    const child = this.child
+    this.child = null
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timer)
+      pending.reject(new RuntimeClientError('accessibility_error', 'computer sidecar shut down'))
+      this.pending.delete(id)
+    }
+    child?.kill('SIGTERM')
+  }
+
+  private ensureStarted(): ChildProcess {
+    if (this.child && !this.child.killed) {
+      return this.child
+    }
+
+    const child = fork(this.entryPath, [], {
+      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      env: {
+        ...process.env,
+        ELECTRON_RUN_AS_NODE: '1',
+        ORCA_COMPUTER_SIDECAR: '1'
+      },
+      ...(process.platform === 'win32' ? { windowsHide: true } : {})
+    })
+
+    child.on('message', (message) => this.handleMessage(message))
+    child.on('exit', (code, signal) => this.handleExit(code, signal))
+    child.on('error', (error) => this.handleError(error))
+    this.child = child
+    return child
+  }
+
+  private handleMessage(message: unknown): void {
+    if (!isSidecarResponse(message)) {
+      return
+    }
+    const pending = this.pending.get(message.id)
+    if (!pending) {
+      return
+    }
+    clearTimeout(pending.timer)
+    this.pending.delete(message.id)
+    if (message.ok) {
+      pending.resolve(message.result)
+      return
+    }
+    pending.reject(new RuntimeClientError(message.error.code, message.error.message))
+  }
+
+  private handleExit(code: number | null, signal: NodeJS.Signals | null): void {
+    this.child = null
+    const detail = signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`
+    const error = new RuntimeClientError(
+      'accessibility_error',
+      `computer sidecar exited with ${detail}`
+    )
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timer)
+      pending.reject(error)
+      this.pending.delete(id)
+    }
+  }
+
+  private handleError(error: Error): void {
+    const wrapped = new RuntimeClientError('accessibility_error', error.message)
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timer)
+      pending.reject(wrapped)
+      this.pending.delete(id)
+    }
+  }
+}
+
+function isSidecarResponse(message: unknown): message is ComputerSidecarResponse {
+  if (!message || typeof message !== 'object') {
+    return false
+  }
+  const record = message as Record<string, unknown>
+  return typeof record.id === 'number' && typeof record.ok === 'boolean'
+}

src/main/computer/sidecar-entry.ts

@@ -0,0 +1,137 @@
+/* eslint-disable max-lines -- Why: sidecar dispatch keeps one method table shared across native providers. */
+import {
+  MacOSNativeProviderClient,
+  shouldUseMacOSNativeProvider
+} from './macos-native-provider-client'
+import {
+  DesktopScriptProviderClient,
+  shouldUseDesktopScriptProvider
+} from './desktop-script-provider-client'
+import { RuntimeClientError } from './runtime-client-error'
+
+type SidecarRequest = {
+  id: number
+  method: string
+  params?: Record<string, unknown>
+}
+
+const nativeMacOSProvider = shouldUseMacOSNativeProvider() ? new MacOSNativeProviderClient() : null
+const desktopScriptProvider = shouldUseDesktopScriptProvider()
+  ? new DesktopScriptProviderClient()
+  : null
+
+process.once('disconnect', shutdownProviders)
+process.once('SIGTERM', () => {
+  shutdownProviders()
+  process.exit(0)
+})
+process.once('SIGINT', () => {
+  shutdownProviders()
+  process.exit(130)
+})
+process.once('beforeExit', shutdownProviders)
+
+process.on('message', (message: unknown) => {
+  void handleMessage(message)
+})
+
+async function handleMessage(message: unknown): Promise<void> {
+  if (!isRequest(message)) {
+    return
+  }
+
+  try {
+    const result = await dispatch(message.method, message.params ?? {})
+    process.send?.({ id: message.id, ok: true, result })
+  } catch (error) {
+    const mapped = errorToResponse(error)
+    process.send?.({ id: message.id, ok: false, error: mapped })
+  }
+}
+
+async function dispatch(method: string, params: Record<string, unknown>): Promise<unknown> {
+  const provider = nativeMacOSProvider ?? desktopScriptProvider
+  if (!provider) {
+    throw new RuntimeClientError(
+      'unsupported_capability',
+      `computer-use has no native provider for ${process.platform}`
+    )
+  }
+
+  switch (method) {
+    case 'capabilities': {
+      return await provider.capabilities()
+    }
+    case 'listApps': {
+      return await provider.listApps()
+    }
+    case 'listWindows': {
+      return await provider.listWindows(params)
+    }
+    case 'getAppState': {
+      return await provider.snapshot(params)
+    }
+    case 'click': {
+      return await provider.action('click', params)
+    }
+    case 'performSecondaryAction': {
+      return await provider.action('performSecondaryAction', params)
+    }
+    case 'scroll': {
+      return await provider.action('scroll', params)
+    }
+    case 'drag': {
+      return await provider.action('drag', params)
+    }
+    case 'typeText': {
+      return await provider.action('typeText', params)
+    }
+    case 'pressKey': {
+      return await provider.action('pressKey', params)
+    }
+    case 'hotkey': {
+      return await provider.action('hotkey', params)
+    }
+    case 'pasteText': {
+      return await provider.action('pasteText', params)
+    }
+    case 'setValue': {
+      return await provider.action('setValue', params)
+    }
+    default:
+      throw new RuntimeClientError(
+        'invalid_argument',
+        `unknown computer sidecar method '${method}'`
+      )
+  }
+}
+
+function isRequest(message: unknown): message is SidecarRequest {
+  if (!message || typeof message !== 'object') {
+    return false
+  }
+  const record = message as Record<string, unknown>
+  return (
+    typeof record.id === 'number' &&
+    typeof record.method === 'string' &&
+    (record.params === undefined || (typeof record.params === 'object' && record.params !== null))
+  )
+}
+
+function errorToResponse(error: unknown): { code: string; message: string } {
+  if (
+    error instanceof Error &&
+    'code' in error &&
+    typeof (error as { code: unknown }).code === 'string'
+  ) {
+    return { code: (error as { code: string }).code, message: error.message }
+  }
+  return {
+    code: 'accessibility_error',
+    message: error instanceof Error ? error.message : String(error)
+  }
+}
+
+function shutdownProviders(): void {
+  nativeMacOSProvider?.shutdown()
+}

src/main/ipc/computer-use-permissions.test.ts

@@ -0,0 +1,71 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { handleMock, openComputerUsePermissionsMock, getComputerUsePermissionStatusMock } =
+  vi.hoisted(() => ({
+    handleMock: vi.fn(),
+    openComputerUsePermissionsMock: vi.fn(),
+    getComputerUsePermissionStatusMock: vi.fn()
+  }))
+
+vi.mock('electron', () => ({
+  ipcMain: {
+    handle: handleMock
+  }
+}))
+
+vi.mock('../computer/macos-computer-use-permissions', () => ({
+  getComputerUsePermissionStatus: getComputerUsePermissionStatusMock,
+  openComputerUsePermissions: openComputerUsePermissionsMock
+}))
+
+import { registerComputerUsePermissionHandlers } from './computer-use-permissions'
+
+describe('registerComputerUsePermissionHandlers', () => {
+  beforeEach(() => {
+    handleMock.mockReset()
+    getComputerUsePermissionStatusMock.mockReset()
+    openComputerUsePermissionsMock.mockReset()
+  })
+
+  it('launches the computer-use helper setup', async () => {
+    const result = {
+      platform: 'darwin',
+      helperAppPath: '/Applications/Orca Computer Use.app',
+      permissionId: 'accessibility',
+      openedSettings: false,
+      launchedHelper: true
+    }
+    openComputerUsePermissionsMock.mockReturnValue(result)
+
+    registerComputerUsePermissionHandlers()
+
+    const registration = handleMock.mock.calls.find(
+      ([channel]) => channel === 'computerUsePermissions:openSetup'
+    )
+    expect(registration).toBeTruthy()
+
+    await expect(registration![1](null, { id: 'accessibility' })).resolves.toBe(result)
+    expect(openComputerUsePermissionsMock).toHaveBeenCalledWith('accessibility')
+  })
+
+  it('returns computer-use permission status', async () => {
+    const result = {
+      platform: 'darwin',
+      permissions: [
+        { id: 'accessibility', status: 'granted' },
+        { id: 'screenshots', status: 'not-granted' }
+      ]
+    }
+    getComputerUsePermissionStatusMock.mockReturnValue(result)
+
+    registerComputerUsePermissionHandlers()
+
+    const registration = handleMock.mock.calls.find(
+      ([channel]) => channel === 'computerUsePermissions:getStatus'
+    )
+    expect(registration).toBeTruthy()
+
+    await expect(registration![1]()).resolves.toBe(result)
+    expect(getComputerUsePermissionStatusMock).toHaveBeenCalledWith()
+  })
+})

src/main/ipc/computer-use-permissions.ts

@@ -0,0 +1,28 @@
+import { ipcMain } from 'electron'
+import type {
+  ComputerUsePermissionId,
+  ComputerUsePermissionSetupResult,
+  ComputerUsePermissionStatusResult
+} from '../../shared/computer-use-permissions-types'
+
+export function registerComputerUsePermissionHandlers(): void {
+  ipcMain.handle(
+    'computerUsePermissions:openSetup',
+    async (
+      _event,
+      args?: { id?: ComputerUsePermissionId }
+    ): Promise<ComputerUsePermissionSetupResult> => {
+      const { openComputerUsePermissions } =
+        await import('../computer/macos-computer-use-permissions')
+      return openComputerUsePermissions(args?.id)
+    }
+  )
+  ipcMain.handle(
+    'computerUsePermissions:getStatus',
+    async (): Promise<ComputerUsePermissionStatusResult> => {
+      const { getComputerUsePermissionStatus } =
+        await import('../computer/macos-computer-use-permissions')
+      return getComputerUsePermissionStatus()
+    }
+  )
+}

src/main/ipc/register-core-handlers.test.ts

@@ -11,6 +11,7 @@ const {
   registerMemoryHandlersMock,
   registerNotificationHandlersMock,
   registerDeveloperPermissionHandlersMock,
+  registerComputerUsePermissionHandlersMock,
   registerSettingsHandlersMock,
   registerTelemetryHandlersMock,
   registerShellHandlersMock,
@@ -45,6 +46,7 @@ const {
   registerMemoryHandlersMock: vi.fn(),
   registerNotificationHandlersMock: vi.fn(),
   registerDeveloperPermissionHandlersMock: vi.fn(),
+  registerComputerUsePermissionHandlersMock: vi.fn(),
   registerSettingsHandlersMock: vi.fn(),
   registerTelemetryHandlersMock: vi.fn(),
   registerShellHandlersMock: vi.fn(),
@@ -118,6 +120,10 @@ vi.mock('./developer-permissions', () => ({
   registerDeveloperPermissionHandlers: registerDeveloperPermissionHandlersMock
 }))
 
+vi.mock('./computer-use-permissions', () => ({
+  registerComputerUsePermissionHandlers: registerComputerUsePermissionHandlersMock
+}))
+
 vi.mock('./settings', () => ({
   registerSettingsHandlers: registerSettingsHandlersMock
 }))
@@ -207,6 +213,7 @@ describe('registerCoreHandlers', () => {
     registerMemoryHandlersMock.mockReset()
     registerNotificationHandlersMock.mockReset()
     registerDeveloperPermissionHandlersMock.mockReset()
+    registerComputerUsePermissionHandlersMock.mockReset()
     registerSettingsHandlersMock.mockReset()
     registerTelemetryHandlersMock.mockReset()
     registerShellHandlersMock.mockReset()
@@ -266,6 +273,7 @@ describe('registerCoreHandlers', () => {
     expect(registerMemoryHandlersMock).toHaveBeenCalledWith(store)
     expect(registerNotificationHandlersMock).toHaveBeenCalledWith(store, runtime)
     expect(registerDeveloperPermissionHandlersMock).toHaveBeenCalled()
+    expect(registerComputerUsePermissionHandlersMock).toHaveBeenCalled()
     expect(registerSettingsHandlersMock).toHaveBeenCalledWith(store)
     expect(registerTelemetryHandlersMock).toHaveBeenCalledWith(store)
     expect(registerSessionHandlersMock).toHaveBeenCalledWith(store)

src/main/ipc/register-core-handlers.ts

@@ -19,6 +19,7 @@ import { registerRuntimeHandlers } from './runtime'
 import { registerNotificationHandlers } from './notifications'
 import { registerOnboardingHandlers } from './onboarding'
 import { registerDeveloperPermissionHandlers } from './developer-permissions'
+import { registerComputerUsePermissionHandlers } from './computer-use-permissions'
 import { setTrustedBrowserRendererWebContentsId, setAgentBrowserBridgeRef } from './browser'
 import { registerSessionHandlers } from './session'
 import { registerSettingsHandlers } from './settings'
@@ -86,6 +87,7 @@ export function registerCoreHandlers(
   registerNotificationHandlers(store, runtime)
   registerOnboardingHandlers(store)
   registerDeveloperPermissionHandlers()
+  registerComputerUsePermissionHandlers()
   registerSettingsHandlers(store)
   registerTelemetryHandlers(store)
   registerBrowserHandlers()

src/main/runtime/rpc/errors.ts

@@ -3,6 +3,7 @@
 // format human-facing messages. Centralizing this mapping keeps the allowlist
 // auditable in one place instead of spread across per-method branches.
 import type { RpcEnvelopeMeta, RpcFailure, RpcSuccess } from './core'
+import { COMPUTER_ERROR_CODES } from '../../../shared/runtime-types'
 
 export function successResponse(id: string, meta: RpcEnvelopeMeta, result: unknown): RpcSuccess {
   return {
@@ -46,8 +47,18 @@ const RUNTIME_PASSTHROUGH_CODES: ReadonlySet<string> = new Set([
   'invalid_limit'
 ])
 
+const COMPUTER_PASSTHROUGH_CODES: ReadonlySet<string> = new Set(Object.values(COMPUTER_ERROR_CODES))
+
 export function mapRuntimeError(id: string, meta: RpcEnvelopeMeta, error: unknown): RpcFailure {
   const message = error instanceof Error ? error.message : String(error)
+  if (
+    error instanceof Error &&
+    'code' in error &&
+    typeof (error as { code: unknown }).code === 'string' &&
+    COMPUTER_PASSTHROUGH_CODES.has((error as { code: string }).code)
+  ) {
+    return errorResponse(id, meta, (error as { code: string }).code, message)
+  }
   if (RUNTIME_PASSTHROUGH_CODES.has(message)) {
     return errorResponse(id, meta, message, message)
   }

src/main/runtime/rpc/methods/computer.test.ts

@@ -0,0 +1,341 @@
+/* eslint-disable max-lines -- Why: computer RPC coverage shares one mocked registry setup across all method contracts. */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { buildRegistry } from '../core'
+
+const computerMocks = vi.hoisted(() => ({
+  callComputerSidecarAction: vi.fn(),
+  callComputerSidecarCapabilities: vi.fn(),
+  callComputerSidecarListApps: vi.fn(),
+  callComputerSidecarListWindows: vi.fn(),
+  callComputerSidecarSnapshot: vi.fn(),
+  resetComputerSidecarForTest: vi.fn(),
+  openComputerUsePermissions: vi.fn()
+}))
+
+vi.mock('../../../computer/sidecar-client', () => ({
+  callComputerSidecarAction: computerMocks.callComputerSidecarAction,
+  callComputerSidecarCapabilities: computerMocks.callComputerSidecarCapabilities,
+  callComputerSidecarListApps: computerMocks.callComputerSidecarListApps,
+  callComputerSidecarListWindows: computerMocks.callComputerSidecarListWindows,
+  callComputerSidecarSnapshot: computerMocks.callComputerSidecarSnapshot,
+  resetComputerSidecarForTest: computerMocks.resetComputerSidecarForTest
+}))
+
+vi.mock('../../../computer/macos-computer-use-permissions', () => ({
+  openComputerUsePermissions: computerMocks.openComputerUsePermissions
+}))
+
+import { COMPUTER_METHODS, resetComputerSessionsForTest } from './computer'
+
+describe('computer RPC methods', () => {
+  beforeEach(() => {
+    computerMocks.callComputerSidecarAction.mockReset()
+    computerMocks.callComputerSidecarCapabilities.mockReset()
+    computerMocks.callComputerSidecarListApps.mockReset()
+    computerMocks.callComputerSidecarListWindows.mockReset()
+    computerMocks.callComputerSidecarSnapshot.mockReset()
+    computerMocks.resetComputerSidecarForTest.mockReset()
+    computerMocks.openComputerUsePermissions.mockReset()
+    resetComputerSessionsForTest()
+    computerMocks.resetComputerSidecarForTest.mockClear()
+  })
+
+  it('registers all computer methods', () => {
+    const registry = buildRegistry(COMPUTER_METHODS)
+
+    expect([...registry.keys()].sort()).toEqual([
+      'computer.capabilities',
+      'computer.click',
+      'computer.drag',
+      'computer.getAppState',
+      'computer.hotkey',
+      'computer.listApps',
+      'computer.listWindows',
+      'computer.pasteText',
+      'computer.performSecondaryAction',
+      'computer.permissions',
+      'computer.pressKey',
+      'computer.scroll',
+      'computer.setValue',
+      'computer.typeText'
+    ])
+  })
+
+  it('resets the sidecar test process', () => {
+    resetComputerSessionsForTest()
+
+    expect(computerMocks.resetComputerSidecarForTest).toHaveBeenCalledTimes(1)
+  })
+
+  it('lists running apps through the sidecar', async () => {
+    const result = {
+      apps: [{ name: 'Finder', bundleId: 'com.apple.finder', pid: 100 }]
+    }
+    computerMocks.callComputerSidecarListApps.mockResolvedValue(result)
+
+    await expect(call('computer.listApps', {})).resolves.toBe(result)
+    expect(computerMocks.callComputerSidecarListApps).toHaveBeenCalledWith()
+  })
+
+  it('returns provider capabilities through the sidecar', async () => {
+    const result = { platform: 'darwin', provider: 'orca-computer-use-macos', protocolVersion: 1 }
+    computerMocks.callComputerSidecarCapabilities.mockResolvedValue(result)
+
+    await expect(call('computer.capabilities', {})).resolves.toBe(result)
+    expect(computerMocks.callComputerSidecarCapabilities).toHaveBeenCalledWith()
+  })
+
+  it('opens computer-use permission setup', async () => {
+    const result = {
+      platform: 'darwin',
+      helperAppPath: '/Applications/Orca Computer Use.app',
+      openedSettings: false,
+      launchedHelper: true
+    }
+    computerMocks.openComputerUsePermissions.mockReturnValue(result)
+
+    await expect(call('computer.permissions', {})).resolves.toBe(result)
+    expect(computerMocks.openComputerUsePermissions).toHaveBeenCalledWith()
+  })
+
+  it('lists windows through the sidecar', async () => {
+    const result = {
+      app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+      windows: []
+    }
+    const params = { app: 'Finder', worktree: 'path:/tmp/repo' }
+    computerMocks.callComputerSidecarListWindows.mockResolvedValue(result)
+
+    await expect(call('computer.listWindows', params)).resolves.toBe(result)
+    expect(computerMocks.callComputerSidecarListWindows).toHaveBeenCalledWith(params)
+  })
+
+  it('gets app state through the sidecar', async () => {
+    const snapshot = {
+      snapshot: {
+        id: 'snap-test',
+        app: { name: 'Finder', bundleId: 'com.apple.finder', pid: 100 },
+        window: { title: 'Finder', width: 100, height: 100 },
+        treeText: 'tree',
+        elementCount: 1,
+        focusedElementId: null
+      },
+      screenshot: null,
+      screenshotStatus: { state: 'skipped', reason: 'no_screenshot_flag' }
+    }
+    const params = {
+      app: 'Finder',
+      worktree: 'path:/tmp/repo',
+      noScreenshot: true,
+      restoreWindow: true,
+      windowId: 123
+    }
+    computerMocks.callComputerSidecarSnapshot.mockResolvedValue(snapshot)
+
+    await expect(call('computer.getAppState', params)).resolves.toBe(snapshot)
+    expect(computerMocks.callComputerSidecarSnapshot).toHaveBeenCalledWith(params)
+  })
+
+  it('rejects missing app in getAppState schema', () => {
+    const method = findMethod('computer.getAppState')
+    expect(() => method.params!.parse({})).toThrow()
+  })
+
+  it('rejects ambiguous window targeting', () => {
+    expect(() =>
+      findMethod('computer.getAppState').params!.parse({
+        app: 'Finder',
+        windowId: 1,
+        windowIndex: 0
+      })
+    ).toThrow(/either --window-id or --window-index/)
+    expect(() =>
+      findMethod('computer.click').params!.parse({
+        app: 'Finder',
+        elementIndex: 0,
+        windowId: 1,
+        windowIndex: 0
+      })
+    ).toThrow(/either --window-id or --window-index/)
+  })
+
+  it('rejects incomplete pointer action coordinates', () => {
+    expect(() => findMethod('computer.click').params!.parse({ app: 'Finder' })).toThrow(
+      /Click requires/
+    )
+    expect(() => findMethod('computer.click').params!.parse({ app: 'Finder', x: 1 })).toThrow(
+      /both --x and --y/
+    )
+    expect(() =>
+      findMethod('computer.click').params!.parse({ app: 'Finder', elementIndex: 0, x: 1, y: 2 })
+    ).toThrow(/either --element-index or coordinate flags/)
+    expect(() =>
+      findMethod('computer.scroll').params!.parse({ app: 'Finder', direction: 'down' })
+    ).toThrow(/Scroll requires/)
+    expect(() =>
+      findMethod('computer.scroll').params!.parse({
+        app: 'Finder',
+        elementIndex: 0,
+        x: 1,
+        y: 2,
+        direction: 'down'
+      })
+    ).toThrow(/either --element-index or coordinate flags/)
+    expect(() =>
+      findMethod('computer.drag').params!.parse({ app: 'Finder', fromX: 1, fromY: 2 })
+    ).toThrow(/Drag coordinates/)
+    expect(() =>
+      findMethod('computer.drag').params!.parse({ app: 'Finder', fromElementIndex: 1 })
+    ).toThrow(/both --from-element-index and --to-element-index/)
+    expect(() =>
+      findMethod('computer.drag').params!.parse({
+        app: 'Finder',
+        fromElementIndex: 0,
+        toElementIndex: 1,
+        fromX: 1,
+        fromY: 2,
+        toX: 3,
+        toY: 4
+      })
+    ).toThrow(/either element indexes or coordinate flags/)
+  })
+
+  it('rejects unsupported scroll directions', () => {
+    expect(() =>
+      findMethod('computer.scroll').params!.parse({
+        app: 'Finder',
+        elementIndex: 0,
+        direction: 'diagonal'
+      })
+    ).toThrow()
+    expect(() =>
+      findMethod('computer.scroll').params!.parse({
+        app: 'Finder',
+        elementIndex: 0,
+        direction: 'down',
+        pages: 0
+      })
+    ).toThrow()
+  })
+
+  it('dispatches pointer and element actions through the sidecar', async () => {
+    computerMocks.callComputerSidecarAction.mockResolvedValue({ ok: true })
+
+    await call('computer.click', {
+      app: 'Finder',
+      worktree: 'path:/tmp/repo',
+      elementIndex: 0,
+      clickCount: 2,
+      mouseButton: 'left',
+      noScreenshot: true
+    })
+    await call('computer.performSecondaryAction', {
+      app: 'Finder',
+      elementIndex: 0,
+      action: 'Raise'
+    })
+    await call('computer.drag', {
+      app: 'Finder',
+      fromX: 1,
+      fromY: 2,
+      toX: 3,
+      toY: 4
+    })
+
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(1, 'click', {
+      app: 'Finder',
+      worktree: 'path:/tmp/repo',
+      elementIndex: 0,
+      clickCount: 2,
+      mouseButton: 'left',
+      noScreenshot: true
+    })
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(
+      2,
+      'performSecondaryAction',
+      {
+        app: 'Finder',
+        elementIndex: 0,
+        action: 'Raise'
+      }
+    )
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(3, 'drag', {
+      app: 'Finder',
+      fromX: 1,
+      fromY: 2,
+      toX: 3,
+      toY: 4
+    })
+  })
+
+  it('dispatches keyboard and text actions through the sidecar', async () => {
+    computerMocks.callComputerSidecarAction.mockResolvedValue({ ok: true })
+
+    await call('computer.typeText', { app: 'Finder', text: 'hello', noScreenshot: true })
+    await call('computer.pressKey', { app: 'Finder', key: 'Return' })
+    await call('computer.hotkey', { app: 'Finder', key: 'CmdOrCtrl+L' })
+    await call('computer.pasteText', { app: 'Finder', text: 'long text' })
+
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(1, 'typeText', {
+      app: 'Finder',
+      text: 'hello',
+      noScreenshot: true
+    })
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(2, 'pressKey', {
+      app: 'Finder',
+      key: 'Return'
+    })
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(3, 'hotkey', {
+      app: 'Finder',
+      key: 'CmdOrCtrl+L'
+    })
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(4, 'pasteText', {
+      app: 'Finder',
+      text: 'long text'
+    })
+  })
+
+  it('dispatches scroll and setValue actions through the sidecar', async () => {
+    computerMocks.callComputerSidecarAction.mockResolvedValue({ ok: true })
+
+    await call('computer.scroll', {
+      app: 'Finder',
+      elementIndex: 0,
+      direction: 'down',
+      pages: 2
+    })
+    await call('computer.setValue', {
+      app: 'Finder',
+      elementIndex: 1,
+      value: ''
+    })
+
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(1, 'scroll', {
+      app: 'Finder',
+      elementIndex: 0,
+      direction: 'down',
+      pages: 2
+    })
+    expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(2, 'setValue', {
+      app: 'Finder',
+      elementIndex: 1,
+      value: ''
+    })
+  })
+})
+
+function findMethod(name: string) {
+  const method = COMPUTER_METHODS.find((candidate) => candidate.name === name)
+  if (!method) {
+    throw new Error(`missing method ${name}`)
+  }
+  return method
+}
+
+async function call(name: string, params: Record<string, unknown>) {
+  const method = findMethod(name)
+  const parsed = method.params ? method.params.parse(params) : undefined
+  return await method.handler(parsed, {
+    runtime: { getRuntimeId: () => 'runtime-1' } as never
+  })
+}

src/main/runtime/rpc/methods/computer.ts

@@ -0,0 +1,301 @@
+/* eslint-disable max-lines -- Why: computer RPC schemas and sidecar dispatch stay together so provider behavior is audited in one place. */
+import { z } from 'zod'
+import {
+  callComputerSidecarAction,
+  callComputerSidecarCapabilities,
+  callComputerSidecarListApps,
+  callComputerSidecarListWindows,
+  callComputerSidecarSnapshot,
+  resetComputerSidecarForTest
+} from '../../../computer/sidecar-client'
+import { defineMethod, type RpcMethod } from '../core'
+import {
+  OptionalBoolean,
+  OptionalFiniteNumber,
+  OptionalPlainString,
+  OptionalString,
+  requiredStringAllowingEmpty,
+  requiredString
+} from '../schemas'
+
+const OptionalNonNegativeInt = z.number().int().nonnegative().optional()
+const OptionalPositiveInt = z.number().int().positive().optional()
+
+const ComputerTarget = z.object({
+  app: requiredString('Missing app'),
+  session: OptionalString,
+  worktree: OptionalPlainString
+})
+
+const ComputerObserveTargetBase = ComputerTarget.extend({
+  noScreenshot: OptionalBoolean,
+  restoreWindow: OptionalBoolean,
+  windowId: OptionalNonNegativeInt,
+  windowIndex: OptionalNonNegativeInt
+})
+
+function validateWindowTarget(
+  value: { windowId?: number; windowIndex?: number },
+  ctx: z.RefinementCtx
+): void {
+  if (value.windowId !== undefined && value.windowIndex !== undefined) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Window targeting accepts either --window-id or --window-index, not both'
+    })
+  }
+}
+
+const ComputerObserveTarget = ComputerObserveTargetBase.superRefine(validateWindowTarget)
+
+const ListApps = z.object({
+  worktree: OptionalPlainString
+})
+
+const ListWindows = ComputerTarget
+
+const Click = ComputerObserveTargetBase.extend({
+  elementIndex: OptionalNonNegativeInt,
+  x: OptionalFiniteNumber,
+  y: OptionalFiniteNumber,
+  clickCount: OptionalPositiveInt,
+  mouseButton: z.enum(['left', 'right', 'middle']).optional()
+}).superRefine((value, ctx) => {
+  validateWindowTarget(value, ctx)
+  const hasElement = value.elementIndex !== undefined
+  const hasX = value.x !== undefined
+  const hasY = value.y !== undefined
+  if (!hasElement && !(hasX && hasY)) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Click requires --element-index or both --x and --y'
+    })
+  }
+  if (hasX !== hasY) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Click coordinates require both --x and --y'
+    })
+  }
+  if (hasElement && (hasX || hasY)) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Click accepts either --element-index or coordinate flags, not both'
+    })
+  }
+})
+
+const PerformSecondaryAction = ComputerObserveTargetBase.extend({
+  elementIndex: OptionalNonNegativeInt,
+  action: requiredString('Missing action')
+}).superRefine((value, ctx) => {
+  validateWindowTarget(value, ctx)
+  if (value.elementIndex === undefined) {
+    ctx.addIssue({ code: 'custom', message: 'Missing element index' })
+  }
+})
+
+const Scroll = ComputerObserveTargetBase.extend({
+  elementIndex: OptionalNonNegativeInt,
+  x: OptionalFiniteNumber,
+  y: OptionalFiniteNumber,
+  direction: z.enum(['up', 'down', 'left', 'right']),
+  pages: z.number().positive().optional()
+}).superRefine((value, ctx) => {
+  validateWindowTarget(value, ctx)
+  const hasElement = value.elementIndex !== undefined
+  const hasX = value.x !== undefined
+  const hasY = value.y !== undefined
+  if (!hasElement && !(hasX && hasY)) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Scroll requires --element-index or both --x and --y'
+    })
+  }
+  if (hasX !== hasY) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Scroll coordinates require both --x and --y'
+    })
+  }
+  if (hasElement && (hasX || hasY)) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Scroll accepts either --element-index or coordinate flags, not both'
+    })
+  }
+})
+
+const Drag = ComputerObserveTargetBase.extend({
+  fromElementIndex: OptionalNonNegativeInt,
+  toElementIndex: OptionalNonNegativeInt,
+  fromX: OptionalFiniteNumber,
+  fromY: OptionalFiniteNumber,
+  toX: OptionalFiniteNumber,
+  toY: OptionalFiniteNumber
+}).superRefine((value, ctx) => {
+  validateWindowTarget(value, ctx)
+  const hasElementPair = value.fromElementIndex !== undefined && value.toElementIndex !== undefined
+  const hasPartialElementPair =
+    value.fromElementIndex !== undefined || value.toElementIndex !== undefined
+  const coordinateKeys = [value.fromX, value.fromY, value.toX, value.toY]
+  const hasCoordinatePair = coordinateKeys.every((coordinate) => coordinate !== undefined)
+  const hasPartialCoordinatePair = coordinateKeys.some((coordinate) => coordinate !== undefined)
+  if (hasElementPair && hasCoordinatePair) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Drag accepts either element indexes or coordinate flags, not both'
+    })
+  }
+  if (!hasElementPair && !hasCoordinatePair) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Drag requires --from-element-index and --to-element-index, or all coordinate flags'
+    })
+  }
+  if (hasPartialElementPair && !hasElementPair) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Drag element targeting requires both --from-element-index and --to-element-index'
+    })
+  }
+  if (hasPartialCoordinatePair && !hasCoordinatePair) {
+    ctx.addIssue({
+      code: 'custom',
+      message: 'Drag coordinates require --from-x, --from-y, --to-x, and --to-y'
+    })
+  }
+})
+
+const TypeText = ComputerObserveTargetBase.extend({
+  text: requiredString('Missing text')
+}).superRefine(validateWindowTarget)
+
+const PressKey = ComputerObserveTargetBase.extend({
+  key: requiredString('Missing key')
+}).superRefine(validateWindowTarget)
+
+const Hotkey = ComputerObserveTargetBase.extend({
+  key: requiredString('Missing key')
+}).superRefine(validateWindowTarget)
+
+const PasteText = ComputerObserveTargetBase.extend({
+  text: requiredString('Missing text')
+}).superRefine(validateWindowTarget)
+
+const SetValue = ComputerObserveTargetBase.extend({
+  elementIndex: OptionalNonNegativeInt,
+  value: requiredStringAllowingEmpty('Missing value')
+}).superRefine((value, ctx) => {
+  validateWindowTarget(value, ctx)
+  if (value.elementIndex === undefined) {
+    ctx.addIssue({ code: 'custom', message: 'Missing element index' })
+  }
+})
+
+export function resetComputerSessionsForTest(): void {
+  resetComputerSidecarForTest()
+}
+
+export const COMPUTER_METHODS: RpcMethod[] = [
+  defineMethod({
+    name: 'computer.capabilities',
+    params: z.object({}),
+    handler: async () => {
+      return await callComputerSidecarCapabilities()
+    }
+  }),
+  defineMethod({
+    name: 'computer.listApps',
+    params: ListApps,
+    handler: async () => {
+      return await callComputerSidecarListApps()
+    }
+  }),
+  defineMethod({
+    name: 'computer.permissions',
+    params: z.object({}),
+    handler: async () => {
+      const { openComputerUsePermissions } =
+        await import('../../../computer/macos-computer-use-permissions')
+      return openComputerUsePermissions()
+    }
+  }),
+  defineMethod({
+    name: 'computer.listWindows',
+    params: ListWindows,
+    handler: async (params) => {
+      return await callComputerSidecarListWindows(params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.getAppState',
+    params: ComputerObserveTarget,
+    handler: async (params) => {
+      return await callComputerSidecarSnapshot(params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.click',
+    params: Click,
+    handler: async (params) => {
+      return await callComputerSidecarAction('click', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.performSecondaryAction',
+    params: PerformSecondaryAction,
+    handler: async (params) => {
+      return await callComputerSidecarAction('performSecondaryAction', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.scroll',
+    params: Scroll,
+    handler: async (params) => {
+      return await callComputerSidecarAction('scroll', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.drag',
+    params: Drag,
+    handler: async (params) => {
+      return await callComputerSidecarAction('drag', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.typeText',
+    params: TypeText,
+    handler: async (params) => {
+      return await callComputerSidecarAction('typeText', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.pressKey',
+    params: PressKey,
+    handler: async (params) => {
+      return await callComputerSidecarAction('pressKey', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.hotkey',
+    params: Hotkey,
+    handler: async (params) => {
+      return await callComputerSidecarAction('hotkey', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.pasteText',
+    params: PasteText,
+    handler: async (params) => {
+      return await callComputerSidecarAction('pasteText', params)
+    }
+  }),
+  defineMethod({
+    name: 'computer.setValue',
+    params: SetValue,
+    handler: async (params) => {
+      return await callComputerSidecarAction('setValue', params)
+    }
+  })
+]

src/main/runtime/rpc/methods/index.ts

@@ -9,6 +9,7 @@ import { ORCHESTRATION_METHODS } from './orchestration'
 import { NOTIFICATION_METHODS } from './notifications'
 import { STATS_METHODS } from './stats'
 import { ACCOUNT_METHODS } from './accounts'
+import { COMPUTER_METHODS } from './computer'
 
 // Why: a flat manifest keeps registration order explicit and provides one
 // grep-point for "what methods does the RPC server expose?" — useful when
@@ -23,5 +24,6 @@ export const ALL_RPC_METHODS: readonly RpcAnyMethod[] = [
   ...ORCHESTRATION_METHODS,
   ...NOTIFICATION_METHODS,
   ...STATS_METHODS,
-  ...ACCOUNT_METHODS
+  ...ACCOUNT_METHODS,
+  ...COMPUTER_METHODS
 ]

src/main/runtime/rpc/schemas.ts

@@ -77,6 +77,10 @@ export function requiredString(message: string) {
     .pipe(z.string().min(1, message))
 }
 
+export function requiredStringAllowingEmpty(message: string) {
+  return z.unknown().refine((value): value is string => typeof value === 'string', { message })
+}
+
 export function requiredNumber(message: string) {
   return z
     .unknown()

src/preload/api-types.ts

@@ -129,6 +129,11 @@ import type {
   DeveloperPermissionRequestResult,
   DeveloperPermissionState
 } from '../shared/developer-permissions-types'
+import type {
+  ComputerUsePermissionId,
+  ComputerUsePermissionSetupResult,
+  ComputerUsePermissionStatusResult
+} from '../shared/computer-use-permissions-types'
 import type {
   ClaudeUsageBreakdownKind,
   ClaudeUsageBreakdownRow,
@@ -807,6 +812,12 @@ export type PreloadApi = {
     request: (args: { id: DeveloperPermissionId }) => Promise<DeveloperPermissionRequestResult>
     openSettings: (args: { id: DeveloperPermissionId }) => Promise<void>
   }
+  computerUsePermissions: {
+    getStatus: () => Promise<ComputerUsePermissionStatusResult>
+    openSetup: (args?: {
+      id?: ComputerUsePermissionId
+    }) => Promise<ComputerUsePermissionSetupResult>
+  }
   shell: {
     openPath: (path: string) => Promise<void>
     openUrl: (url: string) => Promise<void>

src/preload/index.ts

@@ -1037,6 +1037,12 @@ const api = {
       ipcRenderer.invoke('developerPermissions:openSettings', args)
   },
 
+  computerUsePermissions: {
+    getStatus: (): Promise<unknown> => ipcRenderer.invoke('computerUsePermissions:getStatus'),
+    openSetup: (args?: { id?: string }): Promise<unknown> =>
+      ipcRenderer.invoke('computerUsePermissions:openSetup', args)
+  },
+
   shell: {
     openPath: (path: string): Promise<void> => ipcRenderer.invoke('shell:openPath', path),
 

src/renderer/src/components/settings/ComputerUsePane.tsx

@@ -0,0 +1,238 @@
+import { useCallback, useEffect, useMemo, useState, type ReactNode } from 'react'
+import { Accessibility, Camera, Copy, ExternalLink, RefreshCw, ShieldCheck } from 'lucide-react'
+import { toast } from 'sonner'
+import type {
+  ComputerUsePermissionId,
+  ComputerUsePermissionState,
+  ComputerUsePermissionStatus
+} from '../../../../shared/computer-use-permissions-types'
+import { Button } from '../ui/button'
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '../ui/tooltip'
+import type { SettingsSearchEntry } from './settings-search'
+
+const COMPUTER_USE_SKILL_INSTALL_COMMAND =
+  'npx skills add https://github.com/stablyai/orca --skill computer-use'
+
+export const COMPUTER_USE_PANE_SEARCH_ENTRIES: SettingsSearchEntry[] = [
+  {
+    title: 'Computer Use',
+    description: 'Allow agents to inspect screenshots and operate local apps when you ask.',
+    keywords: [
+      'computer use',
+      'accessibility',
+      'screen recording',
+      'screenshot',
+      'automation',
+      'skill'
+    ]
+  }
+]
+
+type PermissionDefinition = {
+  id: ComputerUsePermissionId
+  label: string
+  description: string
+  icon: ReactNode
+}
+
+const PERMISSIONS: PermissionDefinition[] = [
+  {
+    id: 'accessibility',
+    label: 'Accessibility',
+    description: 'Read app interface trees and perform requested actions.',
+    icon: <Accessibility className="size-4" />
+  },
+  {
+    id: 'screenshots',
+    label: 'Screenshots',
+    description: 'Capture app windows so agents can inspect visual state.',
+    icon: <Camera className="size-4" />
+  }
+]
+
+function statusLabel(status: ComputerUsePermissionStatus | undefined): string {
+  switch (status) {
+    case 'granted':
+      return 'Granted'
+    case 'unsupported':
+      return 'macOS only'
+    case 'not-granted':
+    default:
+      return 'Not enabled'
+  }
+}
+
+function statusClass(status: ComputerUsePermissionStatus | undefined): string {
+  if (status === 'granted') {
+    return 'border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300'
+  }
+  return 'border-border bg-muted text-muted-foreground'
+}
+
+export function ComputerUsePane(): React.JSX.Element {
+  const [platform, setPlatform] = useState<NodeJS.Platform | null>(null)
+  const [states, setStates] = useState<ComputerUsePermissionState[]>([])
+  const [loading, setLoading] = useState(true)
+  const [pendingId, setPendingId] = useState<ComputerUsePermissionId | null>(null)
+
+  const stateById = useMemo(
+    () => new Map(states.map((state) => [state.id, state.status] as const)),
+    [states]
+  )
+
+  const refresh = useCallback(async (): Promise<void> => {
+    setLoading(true)
+    try {
+      const result = await window.api.computerUsePermissions.getStatus()
+      setPlatform(result.platform)
+      setStates(result.permissions)
+    } catch (error) {
+      toast.error(
+        error instanceof Error ? error.message : 'Could not load Computer Use permissions'
+      )
+    } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  useEffect(() => {
+    void refresh()
+  }, [refresh])
+
+  // Why: users grant these in System Settings, so refresh when focus returns
+  // instead of polling while the settings pane is open.
+  useEffect(() => {
+    const onFocus = (): void => {
+      void refresh()
+    }
+    window.addEventListener('focus', onFocus)
+    return () => window.removeEventListener('focus', onFocus)
+  }, [refresh])
+
+  const openPermission = async (id: ComputerUsePermissionId): Promise<void> => {
+    setPendingId(id)
+    try {
+      const result = await window.api.computerUsePermissions.openSetup({ id })
+      if (result.launchedHelper) {
+        toast.message('Opened macOS Privacy & Security')
+      } else {
+        toast.message('Computer Use permissions are only required on macOS')
+      }
+    } catch (error) {
+      toast.error(
+        error instanceof Error ? error.message : 'Could not open Computer Use permissions'
+      )
+    } finally {
+      setPendingId(null)
+    }
+  }
+
+  const copySkillInstallCommand = async (): Promise<void> => {
+    try {
+      await window.api.ui.writeClipboardText(COMPUTER_USE_SKILL_INSTALL_COMMAND)
+      toast.success('Copied skill install command.')
+    } catch (error) {
+      toast.error(error instanceof Error ? error.message : 'Failed to copy install command.')
+    }
+  }
+
+  const isMac = platform === null || platform === 'darwin'
+
+  return (
+    <div className="space-y-5">
+      {isMac ? (
+        <>
+          <div className="flex items-start justify-between gap-4 rounded-lg border border-border/60 bg-muted/25 px-4 py-3">
+            <div className="space-y-1">
+              <div className="flex items-center gap-2 text-sm font-medium">
+                <ShieldCheck className="size-4" />
+                Allow Orca to use local apps when you ask.
+              </div>
+              <p className="text-xs text-muted-foreground">
+                Computer Use needs macOS privacy permissions before agents can inspect and operate
+                app windows.
+              </p>
+            </div>
+            <Button variant="outline" size="sm" className="gap-1.5" onClick={() => void refresh()}>
+              <RefreshCw className={`size-3.5 ${loading ? 'animate-spin' : ''}`} />
+              Refresh
+            </Button>
+          </div>
+
+          <div className="divide-y divide-border/60 rounded-lg border border-border/60">
+            {PERMISSIONS.map((permission) => {
+              const status = stateById.get(permission.id)
+              const pending = pendingId === permission.id
+
+              return (
+                <div
+                  key={permission.id}
+                  className="flex items-center justify-between gap-4 px-4 py-3"
+                >
+                  <div className="flex min-w-0 items-start gap-3">
+                    <div className="mt-0.5 text-muted-foreground">{permission.icon}</div>
+                    <div className="min-w-0 space-y-1">
+                      <div className="flex flex-wrap items-center gap-2">
+                        <span className="text-sm font-medium">{permission.label}</span>
+                        <span
+                          className={`rounded-full border px-2 py-0.5 text-[10px] font-medium uppercase tracking-wider ${statusClass(
+                            status
+                          )}`}
+                        >
+                          {statusLabel(status)}
+                        </span>
+                      </div>
+                      <p className="text-xs text-muted-foreground">{permission.description}</p>
+                    </div>
+                  </div>
+                  <Button
+                    variant="outline"
+                    size="sm"
+                    disabled={pending || status === 'unsupported'}
+                    onClick={() => void openPermission(permission.id)}
+                    className="shrink-0 gap-1.5"
+                  >
+                    <ExternalLink className="size-3.5" />
+                    {pending ? 'Opening...' : 'Open'}
+                  </Button>
+                </div>
+              )
+            })}
+          </div>
+        </>
+      ) : null}
+
+      <div className="space-y-2 rounded-lg border border-border/60 px-4 py-3">
+        <div className="space-y-1">
+          <p className="text-sm font-medium">Install Computer Use Skill</p>
+          <p className="text-xs text-muted-foreground">
+            Run this once in an agent project so agents know how to use Orca&apos;s computer
+            controls.
+          </p>
+        </div>
+        <div className="flex max-w-full items-center gap-2 rounded-lg border border-border/60 bg-background/60 px-3 py-2">
+          <code className="flex-1 overflow-x-auto whitespace-nowrap text-[11px] text-muted-foreground">
+            {COMPUTER_USE_SKILL_INSTALL_COMMAND}
+          </code>
+          <TooltipProvider delayDuration={250}>
+            <Tooltip>
+              <TooltipTrigger asChild>
+                <Button
+                  variant="ghost"
+                  size="icon-xs"
+                  onClick={() => void copySkillInstallCommand()}
+                  aria-label="Copy Computer Use skill install command"
+                >
+                  <Copy className="size-3.5" />
+                </Button>
+              </TooltipTrigger>
+              <TooltipContent side="bottom" sideOffset={6}>
+                Copy
+              </TooltipContent>
+            </Tooltip>
+          </TooltipProvider>
+        </div>
+      </div>
+    </div>
+  )
+}

src/renderer/src/components/settings/Settings.tsx

@@ -7,8 +7,10 @@ import {
   FlaskConical,
   GitBranch,
   Globe,
+  Info,
   Keyboard,
   Lock,
+  MousePointerClick,
   ShieldCheck,
   Palette,
   Server,
@@ -32,6 +34,7 @@ import { ShortcutsPane, SHORTCUTS_PANE_SEARCH_ENTRIES } from './ShortcutsPane'
 import { TerminalPane } from './TerminalPane'
 import { useGhosttyImport } from './useGhosttyImport'
 import { Button } from '../ui/button'
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '../ui/tooltip'
 import ghosttyIcon from '../../../../../resources/ghostty.svg'
 import { RepositoryPane, getRepositoryPaneSearchEntries } from './RepositoryPane'
 import { getTerminalPaneSearchEntries } from './terminal-search'
@@ -47,6 +50,7 @@ import {
   DeveloperPermissionsPane,
   DEVELOPER_PERMISSIONS_PANE_SEARCH_ENTRIES
 } from './DeveloperPermissionsPane'
+import { ComputerUsePane, COMPUTER_USE_PANE_SEARCH_ENTRIES } from './ComputerUsePane'
 import { PrivacyPane } from './PrivacyPane'
 import { PRIVACY_PANE_SEARCH_ENTRIES } from './privacy-search'
 import { SettingsSidebar } from './SettingsSidebar'
@@ -62,6 +66,7 @@ type SettingsNavTarget =
   | 'appearance'
   | 'terminal'
   | 'notifications'
+  | 'computer-use'
   | 'developer-permissions'
   | 'privacy'
   | 'shortcuts'
@@ -92,6 +97,16 @@ function getFallbackVisibleSection(sections: SettingsNavSection[]): SettingsNavS
   return sections.at(0)
 }
 
+function computerUsePlatformLabel(args: { isWindows: boolean; isMac: boolean }): string {
+  if (args.isWindows) {
+    return 'Windows'
+  }
+  if (!args.isMac) {
+    return 'Linux'
+  }
+  return 'This platform'
+}
+
 // Why: after a sidebar jump the target section is now in the viewport center
 // rather than the top, which can make it less obvious which section just
 // scrolled into view. Pulsing the border for a moment reassures the user that
@@ -157,6 +172,8 @@ function Settings(): React.JSX.Element {
   const systemPrefersDark = useSystemPrefersDark()
   const isWindows = isWindowsUserAgent()
   const isMac = isMacUserAgent()
+  const showComputerUsePreviewTooltip = !isMac
+  const computerUsePlatform = computerUsePlatformLabel({ isWindows, isMac })
   // Why: the Terminal settings section shares one search index with the
   // sidebar. We trim platform-only entries on other platforms so search never
   // reveals controls that the renderer will intentionally hide.
@@ -414,6 +431,14 @@ function Settings(): React.JSX.Element {
         icon: Bell,
         searchEntries: NOTIFICATIONS_PANE_SEARCH_ENTRIES
       },
+      {
+        id: 'computer-use',
+        title: 'Computer Use',
+        description: 'Enable agents to control any app on your computer.',
+        icon: MousePointerClick,
+        searchEntries: COMPUTER_USE_PANE_SEARCH_ENTRIES,
+        badge: 'Beta'
+      },
       ...(isMac
         ? [
             {
@@ -756,6 +781,39 @@ function Settings(): React.JSX.Element {
                   <NotificationsPane settings={settings} updateSettings={updateSettings} />
                 </SettingsSection>
 
+                <SettingsSection
+                  id="computer-use"
+                  title="Computer Use"
+                  badge="Beta"
+                  badgeAccessory={
+                    showComputerUsePreviewTooltip ? (
+                      <TooltipProvider delayDuration={250}>
+                        <Tooltip>
+                          <TooltipTrigger asChild>
+                            <button
+                              type="button"
+                              className="text-muted-foreground transition-colors hover:text-foreground"
+                              aria-label={`${computerUsePlatform} Computer Use preview details`}
+                            >
+                              <Info className="size-3.5" />
+                            </button>
+                          </TooltipTrigger>
+                          <TooltipContent side="top" sideOffset={6} className="max-w-72">
+                            <span>
+                              {computerUsePlatform} Computer Use is an early preview. Some apps and
+                              desktop environments may behave inconsistently.
+                            </span>
+                          </TooltipContent>
+                        </Tooltip>
+                      </TooltipProvider>
+                    ) : null
+                  }
+                  description="Enable agents to control any app on your computer."
+                  searchEntries={COMPUTER_USE_PANE_SEARCH_ENTRIES}
+                >
+                  <ComputerUsePane />
+                </SettingsSection>
+
                 {isMac ? (
                   <SettingsSection
                     id="developer-permissions"

src/renderer/src/components/settings/SettingsSection.tsx

@@ -10,6 +10,7 @@ type SettingsSectionProps = {
   children: React.ReactNode
   className?: string
   badge?: string
+  badgeAccessory?: React.ReactNode
   /** Rendered in the section header's upper-right corner — intended for
    *  section-scoped actions (e.g. "Import from Ghostty") that would otherwise
    *  crowd the settings list as their own row. */
@@ -24,6 +25,7 @@ export function SettingsSection({
   children,
   className,
   badge,
+  badgeAccessory,
   headerAction
 }: SettingsSectionProps): React.JSX.Element | null {
   const query = useAppStore((state) => state.settingsSearchQuery)
@@ -52,6 +54,7 @@ export function SettingsSection({
                 {badge}
               </span>
             ) : null}
+            {badgeAccessory}
           </h2>
           <p className="text-sm text-muted-foreground">{description}</p>
         </div>

src/renderer/src/store/slices/ui.ts

@@ -241,6 +241,7 @@ export type UISlice = {
       | 'browser'
       | 'appearance'
       | 'terminal'
+      | 'computer-use'
       | 'developer-permissions'
       | 'shortcuts'
       | 'repo'

src/shared/computer-use-permissions-types.ts

@@ -0,0 +1,23 @@
+export type ComputerUsePermissionId = 'accessibility' | 'screenshots'
+
+export type ComputerUsePermissionStatus = 'granted' | 'not-granted' | 'unsupported'
+
+export type ComputerUsePermissionState = {
+  id: ComputerUsePermissionId
+  status: ComputerUsePermissionStatus
+}
+
+export type ComputerUsePermissionStatusResult = {
+  platform: NodeJS.Platform
+  permissions: ComputerUsePermissionState[]
+}
+
+export type ComputerUsePermissionSetupResult = {
+  platform: NodeJS.Platform
+  helperAppPath: string | null
+  permissionId?: ComputerUsePermissionId
+  openedSettings: boolean
+  launchedHelper: boolean
+  permissions?: ComputerUsePermissionState[]
+  nextStep?: string | null
+}

src/shared/runtime-types.ts

@@ -482,3 +482,191 @@ export type BrowserErrorCode =
   | 'browser_debugger_detached'
   | 'browser_timeout'
   | 'browser_error'
+
+// Computer-use types (see docs/computer-use/plan.md §4 and §12.6).
+
+export const COMPUTER_ERROR_CODES = {
+  app_not_found: 'app_not_found',
+  app_blocked: 'app_blocked',
+  window_not_found: 'window_not_found',
+  window_stale: 'window_stale',
+  provider_incompatible: 'provider_incompatible',
+  unsupported_capability: 'unsupported_capability',
+  permission_denied: 'permission_denied',
+  element_not_found: 'element_not_found',
+  element_not_clickable: 'element_not_clickable',
+  action_not_supported: 'action_not_supported',
+  value_not_settable: 'value_not_settable',
+  invalid_argument: 'invalid_argument',
+  action_timeout: 'action_timeout',
+  screenshot_failed: 'screenshot_failed',
+  accessibility_error: 'accessibility_error'
+} as const
+
+export type ComputerErrorCode = keyof typeof COMPUTER_ERROR_CODES
+
+export type ComputerAppQuery = string
+
+export type ComputerSessionTarget = {
+  session?: string
+  worktree?: string
+  app?: ComputerAppQuery
+}
+
+export type ComputerListAppsArgs = {
+  worktree?: string
+}
+
+export type ComputerAppInfo = {
+  name: string
+  bundleId: string | null
+  pid: number
+}
+
+export type ComputerWindowInfo = {
+  id?: number | null
+  title: string
+  x?: number | null
+  y?: number | null
+  width: number
+  height: number
+  isMinimized?: boolean | null
+  isOffscreen?: boolean | null
+  screenIndex?: number | null
+  platform?: Record<string, unknown>
+}
+
+export type ComputerSnapshotData = {
+  id: string
+  app: ComputerAppInfo
+  window: ComputerWindowInfo
+  coordinateSpace: 'window'
+  treeText: string
+  elementCount: number
+  focusedElementId: number | null
+  truncation?: {
+    truncated: boolean
+    maxNodes?: number
+    maxDepth?: number
+    maxDepthReached?: boolean
+  }
+}
+
+export type ComputerScreenshotData = {
+  data?: string
+  format: 'png'
+  width: number
+  height: number
+  scale: number
+  path?: string
+  dataOmitted?: boolean
+  expiresAt?: string
+}
+
+export type ComputerScreenshotMetadata = {
+  engine?: 'screenCaptureKit' | 'cgWindowList' | 'unknown'
+  windowId?: number | null
+}
+
+export type ComputerScreenshotStatus =
+  | { state: 'captured'; metadata?: ComputerScreenshotMetadata }
+  | { state: 'skipped'; reason: 'no_screenshot_flag' }
+  | {
+      state: 'failed'
+      code: ComputerErrorCode
+      message: string
+      metadata?: ComputerScreenshotMetadata
+    }
+
+export type ComputerActionMetadata = {
+  path: 'accessibility' | 'synthetic' | 'clipboard'
+  actionName?: string | null
+  fallbackReason?: string | null
+  targetWindowId?: number | null
+  verification?: ComputerActionVerification
+}
+
+export type ComputerActionVerification =
+  | {
+      state: 'verified'
+      property: 'focusedText' | 'selection'
+      expected?: string | null
+      actualPreview?: string | null
+    }
+  | {
+      state: 'unverified'
+      reason: 'synthetic_input' | 'clipboard_paste' | 'provider_unavailable' | 'window_changed'
+    }
+
+export type ComputerSnapshotResult = {
+  snapshot: ComputerSnapshotData
+  screenshot: ComputerScreenshotData | null
+  screenshotStatus: ComputerScreenshotStatus
+}
+
+export type ComputerActionResult = ComputerSnapshotResult & {
+  action?: ComputerActionMetadata
+}
+
+export type ComputerProviderCapabilities = {
+  platform: NodeJS.Platform
+  provider: string
+  providerVersion: string
+  protocolVersion: number
+  supports: {
+    apps: {
+      list: boolean
+      bundleIds: boolean
+      pids: boolean
+    }
+    windows: {
+      list: boolean
+      targetById: boolean
+      targetByIndex: boolean
+      focus: boolean
+      moveResize: boolean
+    }
+    observation: {
+      screenshot: boolean
+      annotatedScreenshot: boolean
+      elementFrames: boolean
+      ocr: boolean
+    }
+    actions: {
+      click: boolean
+      typeText: boolean
+      pressKey: boolean
+      hotkey: boolean
+      pasteText: boolean
+      scroll: boolean
+      drag: boolean
+      setValue: boolean
+      performAction: boolean
+    }
+    surfaces: {
+      menus: boolean
+      dialogs: boolean
+      dock: boolean
+      menubar: boolean
+    }
+  }
+}
+
+export type ComputerWindowListWindow = ComputerWindowInfo & {
+  app: ComputerAppInfo
+  index: number
+  isMain?: boolean | null
+}
+
+export type ComputerListWindowsResult = {
+  app: ComputerAppInfo
+  windows: ComputerWindowListWindow[]
+}
+
+export type ComputerListAppsResult = {
+  apps: (ComputerAppInfo & {
+    isRunning: boolean
+    lastUsedAt: string | null
+    useCount: number | null
+  })[]
+}

tests/e2e/computer-linux.e2e.ts

@@ -0,0 +1,66 @@
+import { afterAll, beforeAll, describe, expect, test } from 'vitest'
+import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
+import {
+  ensureGeditLaunched,
+  killGedit,
+  parseJsonOutput,
+  runOrcaCli
+} from './helpers/computer-driver'
+
+const isLinux = process.platform === 'linux'
+const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
+
+describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
+  beforeAll(async () => {
+    await ensureGeditLaunched()
+  })
+
+  afterAll(async () => {
+    await killGedit()
+  })
+
+  test('gedit exposes a basic accessibility tree', async () => {
+    const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'gedit', '--json'])
+    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)
+
+    expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0)
+    expect(envelope.result.snapshot.coordinateSpace).toBe('window')
+    expect(envelope.result.snapshot.truncation?.truncated).toBe(false)
+    expect(envelope.result.screenshot?.data).toBeUndefined()
+    expect(envelope.result.screenshot?.path).toContain('orca-computer-use-')
+  })
+
+  test('paste-text mutates the test-owned document', async () => {
+    const marker = `orca-linux-paste-${Date.now()}`
+    const action = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'paste-text',
+          '--app',
+          'gedit',
+          '--text',
+          marker,
+          '--restore-window',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(action.result.action?.path).toBe('clipboard')
+
+    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'gedit',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(after.result.snapshot.treeText).toContain(marker)
+  })
+})

tests/e2e/computer-mac.e2e.ts

@@ -0,0 +1,154 @@
+import { afterAll, beforeAll, describe, expect, test } from 'vitest'
+import type {
+  ComputerActionResult,
+  ComputerListAppsResult,
+  ComputerSnapshotResult
+} from '../../src/shared/runtime-types'
+import {
+  ensureTextEditLaunched,
+  findRoleIndex,
+  killTextEdit,
+  parseJsonOutput,
+  runOrcaCli
+} from './helpers/computer-driver'
+
+const isMac = process.platform === 'darwin'
+const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
+
+describe.skipIf(!isMac || !e2eOptIn)('computer-use macOS e2e (TextEdit)', () => {
+  beforeAll(async () => {
+    await ensureTextEditLaunched()
+  })
+
+  afterAll(async () => {
+    await killTextEdit()
+  })
+
+  test('list-apps includes TextEdit', async () => {
+    const result = await runOrcaCli(['computer', 'list-apps', '--json'])
+    const envelope = parseJsonOutput<{ result: ComputerListAppsResult }>(result.stdout)
+
+    expect(envelope.result.apps.some((app) => app.name === 'TextEdit')).toBe(true)
+  })
+
+  test('get-app-state returns TextEdit state', async () => {
+    const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'TextEdit', '--json'])
+    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)
+
+    expect(envelope.result.snapshot.app.name).toBe('TextEdit')
+    expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0)
+    expect(envelope.result.snapshot.treeText).toContain('Window:')
+  })
+
+  test('click, type-text, and re-observe show inserted text', async () => {
+    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (await runOrcaCli(['computer', 'get-app-state', '--app', 'TextEdit', '--json'])).stdout
+    )
+    const textTarget = findRoleIndex(
+      before.result.snapshot.treeText,
+      /^\s*(\d+)\s+(text entry area|text field|HTML content)(?:\s|$)/m
+    )
+    expect(textTarget).toBeGreaterThanOrEqual(0)
+
+    await runOrcaCli([
+      'computer',
+      'click',
+      '--app',
+      'TextEdit',
+      '--element-index',
+      String(textTarget)
+    ])
+
+    const marker = `orca computer e2e ${Date.now()}`
+    await runOrcaCli(['computer', 'type-text', '--app', 'TextEdit', '--text', marker])
+
+    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (await runOrcaCli(['computer', 'get-app-state', '--app', 'TextEdit', '--json'])).stdout
+    )
+    expect(after.result.snapshot.treeText).toContain(marker)
+  })
+
+  test('paste-text and hotkey verify TextEdit text replacement', async () => {
+    const first = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'paste-text',
+          '--app',
+          'TextEdit',
+          '--text',
+          'orca paste first',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(first.result.action?.path).toBe('accessibility')
+    expect(first.result.action?.verification?.state).toBe('verified')
+
+    const selectAll = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'hotkey',
+          '--app',
+          'TextEdit',
+          '--key',
+          'CmdOrCtrl+A',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(selectAll.result.action?.actionName).toBe('AXSelectAll')
+    expect(selectAll.result.action?.verification?.state).toBe('verified')
+
+    const marker = `orca paste final ${Date.now()}`
+    const second = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'paste-text',
+          '--app',
+          'TextEdit',
+          '--text',
+          marker,
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(second.result.action?.actionName).toBe('AXReplaceSelection')
+    expect(second.result.action?.verification).toMatchObject({
+      state: 'verified',
+      property: 'focusedText',
+      expected: marker
+    })
+
+    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'TextEdit',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(after.result.snapshot.treeText).toContain(marker)
+    expect(after.result.snapshot.treeText).not.toContain('orca paste first')
+  })
+
+  test('screenshot capture returns image metadata', async () => {
+    const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'TextEdit', '--json'])
+    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)
+
+    expect(envelope.result.screenshotStatus.state).toBe('captured')
+    expect(envelope.result.screenshot?.format).toBe('png')
+    expect(envelope.result.screenshot?.data).toBeUndefined()
+    expect(envelope.result.screenshot?.dataOmitted).toBe(true)
+    expect(envelope.result.screenshot?.path).toContain('orca-computer-use-')
+  })
+})

tests/e2e/computer-windows.e2e.ts

@@ -0,0 +1,66 @@
+import { afterAll, beforeAll, describe, expect, test } from 'vitest'
+import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
+import {
+  ensureNotepadLaunched,
+  killNotepad,
+  parseJsonOutput,
+  runOrcaCli
+} from './helpers/computer-driver'
+
+const isWindows = process.platform === 'win32'
+const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
+
+describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', () => {
+  beforeAll(async () => {
+    await ensureNotepadLaunched()
+  })
+
+  afterAll(async () => {
+    await killNotepad()
+  })
+
+  test('Notepad exposes a basic accessibility tree', async () => {
+    const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'Notepad', '--json'])
+    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)
+
+    expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0)
+    expect(envelope.result.snapshot.coordinateSpace).toBe('window')
+    expect(envelope.result.snapshot.truncation?.truncated).toBe(false)
+    expect(envelope.result.screenshot?.data).toBeUndefined()
+    expect(envelope.result.screenshot?.path).toContain('orca-computer-use-')
+  })
+
+  test('paste-text mutates the test-owned document', async () => {
+    const marker = `orca-windows-paste-${Date.now()}`
+    const action = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'paste-text',
+          '--app',
+          'Notepad',
+          '--text',
+          marker,
+          '--restore-window',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(action.result.action?.path).toBe('clipboard')
+
+    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'Notepad',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(after.result.snapshot.treeText).toContain(marker)
+  })
+})

tests/e2e/helpers/computer-driver.ts

@@ -0,0 +1,127 @@
+import { execFile, spawn, type ChildProcess } from 'child_process'
+import { mkdtemp, rm, writeFile } from 'fs/promises'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import { promisify } from 'util'
+
+const execFileAsync = promisify(execFile)
+let textEditTempDir: string | null = null
+let linuxTempDir: string | null = null
+let windowsTempDir: string | null = null
+let geditProcess: ChildProcess | null = null
+let notepadProcess: ChildProcess | null = null
+
+export type CliResult = {
+  stdout: string
+  stderr: string
+}
+
+export async function runOrcaCli(args: string[]): Promise<CliResult> {
+  const devCli = join(process.cwd(), 'config/scripts/orca-dev')
+  const command = process.env.ORCA_COMPUTER_CLI ?? devCli
+  const cliArgs = process.env.ORCA_COMPUTER_CLI ? args : args
+  try {
+    const result = await execFileAsync(command, cliArgs, {
+      maxBuffer: 20 * 1024 * 1024
+    })
+    return { stdout: result.stdout, stderr: result.stderr }
+  } catch (error) {
+    if (error && typeof error === 'object' && 'stdout' in error && 'stderr' in error) {
+      const output = error as { message: string; stdout: string; stderr: string }
+      throw new Error(`${output.message}\nstdout:\n${output.stdout}\nstderr:\n${output.stderr}`)
+    }
+    throw error
+  }
+}
+
+export async function ensureTextEditLaunched(): Promise<void> {
+  await killTextEdit()
+  textEditTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-e2e-'))
+  const filePath = join(textEditTempDir, 'textedit-target.txt')
+  await writeFile(filePath, 'seed', 'utf8')
+  await execFileAsync('open', ['-a', 'TextEdit', '-n', filePath])
+  await delay(5500)
+}
+
+export async function killTextEdit(): Promise<void> {
+  try {
+    await execFileAsync('killall', ['TextEdit'])
+  } catch {
+    // TextEdit may already be closed by the user or the OS.
+  }
+  if (textEditTempDir) {
+    await rm(textEditTempDir, { force: true, recursive: true })
+    textEditTempDir = null
+  }
+}
+
+export async function ensureGeditLaunched(): Promise<void> {
+  await killGedit()
+  linuxTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-linux-e2e-'))
+  const filePath = join(linuxTempDir, 'gedit-target.txt')
+  await writeFile(filePath, 'seed', 'utf8')
+  geditProcess = spawn('gedit', [filePath], { detached: true, stdio: 'ignore' })
+  geditProcess.unref()
+  await delay(3500)
+}
+
+export async function killGedit(): Promise<void> {
+  if (geditProcess?.pid) {
+    try {
+      process.kill(-geditProcess.pid, 'SIGTERM')
+    } catch {
+      // The test-owned gedit process may already be closed.
+    }
+    geditProcess = null
+  }
+  if (linuxTempDir) {
+    await rm(linuxTempDir, { force: true, recursive: true })
+    linuxTempDir = null
+  }
+}
+
+export async function ensureNotepadLaunched(): Promise<void> {
+  await killNotepad()
+  windowsTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-windows-e2e-'))
+  const filePath = join(windowsTempDir, 'notepad-target.txt')
+  await writeFile(filePath, 'seed', 'utf8')
+  notepadProcess = spawn('notepad.exe', [filePath], { detached: true, stdio: 'ignore' })
+  notepadProcess.unref()
+  await delay(2500)
+}
+
+export async function killNotepad(): Promise<void> {
+  if (notepadProcess?.pid) {
+    try {
+      await execFileAsync('taskkill.exe', ['/PID', String(notepadProcess.pid), '/T', '/F'])
+    } catch {
+      // The test-owned Notepad process may already be closed.
+    }
+    notepadProcess = null
+  }
+  if (windowsTempDir) {
+    await rm(windowsTempDir, { force: true, recursive: true })
+    windowsTempDir = null
+  }
+}
+
+export function findRoleIndex(treeText: string, role: string | RegExp): number {
+  const matcher =
+    typeof role === 'string'
+      ? new RegExp(`^\\s*(\\d+)\\s+${escapeRegExp(role)}(?:\\s|$)`, 'm')
+      : role
+  const match = treeText.match(matcher)
+  return match?.[1] ? Number.parseInt(match[1], 10) : -1
+}
+
+export function parseJsonOutput<T>(stdout: string): T {
+  return JSON.parse(stdout) as T
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function escapeRegExp(input: string): string {
+  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}

tests/e2e/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['tests/e2e/computer-*.e2e.ts'],
+    testTimeout: 60_000,
+    hookTimeout: 60_000
+  }
+})

tools/spikes/.gitignore

@@ -0,0 +1 @@
+*.png