Merge main to validate environment store integration

This commit is contained in:
Brennan Benson
2026-11-13 18:53:33 -08:00
744 changed files with 33633 additions and 10769 deletions
-1
View File
@@ -101,7 +101,6 @@ jobs:
src/cli/output/command-result-output.test.ts
src/cli/computer/computer-state-formatting.test.ts
src/cli/index.test.ts
src/cli/runtime/envelope-schema.test.ts
src/shared/remote-runtime/remote-runtime-client.test.ts
- run: pnpm verify:computer-native
- run: pnpm build:cli
+8
View File
@@ -227,7 +227,15 @@ jobs:
with:
arches: ${{ (inputs.build_template && matrix.os == 'windows-2022') && 'x64 arm64' || (runner.arch == 'ARM64' && 'arm64' || 'x64') }}
- uses: actions/download-artifact@v8
id: mobile-web-artifact
if: needs.changes.outputs.qualification != 'false'
continue-on-error: true
with:
name: orcad-mobile-web-bundle
path: out/mobile-web
- name: Retry mobile web artifact download
if: steps.mobile-web-artifact.outcome == 'failure'
uses: actions/download-artifact@v8
with:
name: orcad-mobile-web-bundle
path: out/mobile-web
+3 -1
View File
@@ -1121,7 +1121,9 @@ jobs:
eb_cache_path: |
~\AppData\Local\electron\Cache
~\AppData\Local\electron-builder\Cache
- os: ubuntu-latest
# Why: ubuntu-latest moved to the amd64v3 Ubuntu 26.04 image, whose GCC
# defaults to -march=x86-64-v3 and ships AVX in node-pty (#41637).
- os: ubuntu-24.04
platform: linux-x64
release_command: node "$(if [ -f config/scripts/native-build/node-addons/ensure-native-runtime.mjs ]; then printf %s config/scripts/native-build/node-addons/ensure-native-runtime.mjs; else printf %s config/scripts/ensure-native-runtime.mjs; fi)" --runtime=electron && pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish always -c.publish.releaseType=draft
unpacked_dir: dist/linux-unpacked
+1 -1
View File
@@ -1,2 +1,2 @@
#!/usr/bin/env sh
pnpm exec lint-staged
lint-staged
+1 -2
View File
@@ -64,8 +64,7 @@ click. Interference probes remain separate from task latency comparisons.
Explicit-target typing is a native capability probe. Orca's public CLI/RPC accepts
an element target for text (`type-text --element-index`) on providers that report
`actions.typeTextElement` (macOS and Linux); Windows types only at the focused
field. Click-then-type tasks send no element with the text, like `type-text`
`actions.typeTextElement` (macOS, Linux and Windows). Click-then-type tasks send no element with the text, like `type-text`
without the index, although this benchmark still bypasses public transports.
The Linux Electron job also records restoration stages in `focus-probe/results.json`.
@@ -18,8 +18,8 @@ const GUEST_PRELOADS = [
{ entry: 'src/preload/browser-extension-preload.ts', fileName: 'browser-extension-preload.js' },
{ entry: 'src/preload/browser-guest.ts', fileName: 'browser-guest-preload.js' },
{
entry: 'src/preload/browser-passkey-request.ts',
fileName: 'browser-passkey-request-preload.js'
entry: 'src/preload/browser-system-dialogs.ts',
fileName: 'browser-system-dialogs-preload.js'
},
{ entry: 'src/preload/doc-preview-link.ts', fileName: 'doc-preview-link-preload.js' },
{ entry: 'src/preload/plugin-frame-guest.ts', fileName: 'plugin-frame-guest-preload.js' }
+1 -1
View File
@@ -4,7 +4,7 @@
"src/main/index.ts",
"src/preload/index.ts",
"src/preload/browser-guest.ts",
"src/preload/browser-passkey-request.ts",
"src/preload/browser-system-dialogs.ts",
"src/main/daemon/daemon-entry.ts",
"src/main/plugins/workers/plugin-host-entry.ts",
"src/main/computer/computer-sidecar.ts",
+78 -3
View File
@@ -1,5 +1,5 @@
diff --git a/lib/gitWorkflow.js b/lib/gitWorkflow.js
index 3a32d6a9cd65c7ecb6b3cbccfce5a15c96530435..363513128bf7302eab73ea775102bb8a633a3b8b 100644
index 3a32d6a9cd65c7ecb6b3cbccfce5a15c96530435..f91b1931224ce59df5cf9880248d4fad93e74f03 100644
--- a/lib/gitWorkflow.js
+++ b/lib/gitWorkflow.js
@@ -50,6 +50,9 @@ export const STASH = 'lint-staged automatic backup'
@@ -39,7 +39,66 @@ index 3a32d6a9cd65c7ecb6b3cbccfce5a15c96530435..363513128bf7302eab73ea775102bb8a
/** Print stash list with short hash and subject */
const stashes = await this.execGit(['stash', 'list', '--format="%h %s"', '-z'])
.then(parseGitZOutput)
@@ -270,11 +286,14 @@ export class GitWorkflow {
@@ -142,11 +158,44 @@ export class GitWorkflow {
*/
async getDeletedFiles() {
debugLog('Getting deleted files...')
- const lsFiles = await this.execGit(['ls-files', '--deleted'])
- const deletedFiles = lsFiles
- .split('\n')
- .filter(Boolean)
- .map((file) => path.resolve(this.topLevelDir, file))
+ // Orca patch: `ls-files --deleted` lstat()s every tracked file one at a time, which takes tens
+ // of seconds when the stat cache is cold. `diff-files` reads the index with parallel lstat
+ // (preload) instead. Like `ls-files --deleted`, it must not trust an fsmonitor (a stale one
+ // would hide a deletion), so the monitor is turned off for this read-only query. It does
+ // trust the assume-unchanged bit, so those entries (lowercase `ls-files -v` tag; `s` is also
+ // skip-worktree, which `--deleted` skips) are added as candidates. A candidate counts only
+ // if lstat() fails, which is the `--deleted` rule.
+ const [diffOutput, taggedOutput] = await Promise.all([
+ this.execGit([
+ '-c',
+ 'core.fsmonitor=false',
+ 'diff-files',
+ '--name-only',
+ '--diff-filter=D',
+ '--no-renames',
+ '--no-relative',
+ '--ignore-submodules=none',
+ '-z',
+ ]),
+ this.execGit(['ls-files', '-v', '-z']),
+ ])
+ const assumeUnchanged = parseGitZOutput(taggedOutput).flatMap((line) =>
+ /^[a-rt-z] /.test(line) ? [line.slice(2)] : []
+ )
+ const candidates = new Set(
+ [...parseGitZOutput(diffOutput), ...assumeUnchanged].map((file) =>
+ path.resolve(this.topLevelDir, file)
+ )
+ )
+ const isGone = await Promise.all(
+ [...candidates].map((file) =>
+ fs.lstat(file).then(
+ () => false,
+ () => true
+ )
+ )
+ )
+ const deletedFiles = [...candidates].filter((_, i) => isGone[i])
debugLog('Found deleted files:', deletedFiles)
return deletedFiles
}
@@ -194,7 +243,7 @@ export class GitWorkflow {
*/
async getUnstagedFiles({ onlyPartial = false } = {}) {
debugLog('Getting partially staged files...')
- const status = await this.execGit(['status', '-z'])
+ const status = await this.execGit(['status', '-z', '--untracked-files=no'])
/**
* See https://git-scm.com/docs/git-status#_short_format
* Entries returned in machine format are separated by a NUL character.
@@ -270,11 +319,14 @@ export class GitWorkflow {
} else {
/** Save stash of all changes, keeping all files as-is */
const stashHash = await this.execGit(['stash', 'create'])
@@ -56,7 +115,7 @@ index 3a32d6a9cd65c7ecb6b3cbccfce5a15c96530435..363513128bf7302eab73ea775102bb8a
debugLog(task.title)
}
} catch (error) {
@@ -425,7 +444,11 @@ export class GitWorkflow {
@@ -425,7 +477,11 @@ export class GitWorkflow {
async cleanup(ctx) {
try {
debugLog('Dropping backup stash...')
@@ -111,6 +170,22 @@ index 993f8d81cac48132b5ae2ebdc8ca452c7217caad..2dc2291bc82451eacc5babfc8ce40805
> git stash list --format="%h %s"
${hash} On main: lint-staged automatic backup
diff --git a/lib/searchConfigs.js b/lib/searchConfigs.js
index 77ffe8f2423bb24ed80e2e23b57847a69c137195..822ff82c189cf712029e5e5870e5b3d9266f74a2 100644
--- a/lib/searchConfigs.js
+++ b/lib/searchConfigs.js
@@ -36,8 +36,9 @@ const listConfigFilesFromGit = async ({ cwd, topLevelDir }) =>
[
...EXEC_GIT,
'--cached', // show all tracked files
- '--others', // show untracked files
- '--exclude-standard', // apply standard git exclusions (.gitignore, etc.)
+ // Orca patch: no `--others`. Listing untracked files walks the whole worktree, which takes
+ // seconds on a cold stat cache. Untracked configs in `cwd` and above are still found by
+ // `listConfigFilesFromFs`; a nested config must be tracked or staged to apply.
'--',
...CONFIG_PATHSPEC,
],
diff --git a/lib/state.js b/lib/state.js
index da30e6f639d31307ebf503691aa991bb93948f8a..4a38799fa7773a8e8629c7c3a6b72287e6b22f4b 100644
--- a/lib/state.js
+1 -4
View File
@@ -65,7 +65,6 @@
2 src/renderer/src/components/browser-pane/passwords/BrowserPasswordFillDropdown.tsx
1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-input.ts
4 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-lifecycle.ts
1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-navigation.ts
2 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-wheel.ts
3 src/renderer/src/components/browser-pane/workspace-doc/doc-preview-directory-access.tsx
1 src/renderer/src/components/browser-pane/workspace-doc/doc-preview-overflow-menu.tsx
@@ -402,7 +401,6 @@
1 src/renderer/src/components/settings/browser/extensions/BrowserExtensionsSection.tsx
1 src/renderer/src/components/settings/browser/extensions/BrowserExtensionsWebStoreInstall.tsx
1 src/renderer/src/components/settings/browser/profiles/BrowserNewProfileDialog.tsx
1 src/renderer/src/components/settings/chat/ChatNamingSetting.tsx
1 src/renderer/src/components/settings/cli/install/CliSection.tsx
3 src/renderer/src/components/settings/cli/install/WslCliRegistration.tsx
2 src/renderer/src/components/settings/cli/install/use-cli-registration-actions.ts
@@ -510,8 +508,6 @@
1 src/renderer/src/components/sidebar/workspace-board/viewport/WorkspaceKanbanLaneCardList.tsx
1 src/renderer/src/components/sidebar/workspace-board/viewport/WorkspaceKanbanLaneGrid.tsx
3 src/renderer/src/components/sidebar/workspace-board/viewport/use-workspace-kanban-column-resize.ts
1 src/renderer/src/components/sidebar/workspace-card/activity/WorktreeCardAgents.tsx
1 src/renderer/src/components/sidebar/workspace-card/activity/useWorktreeAgentRows.ts
3 src/renderer/src/components/sidebar/workspace-card/activity/worktree-card-compact-agent-row.tsx
2 src/renderer/src/components/sidebar/workspace-card/activity/worktree-card-compact-agents.tsx
1 src/renderer/src/components/sidebar/workspace-card/identity/AutoRenameFailedDialog.tsx
@@ -704,6 +700,7 @@
6 src/renderer/src/hooks/useVirtualizedScrollAnchor.ts
1 src/renderer/src/lib/agent-session/launch/inventory/host-chat-program-acquisition.ts
1 src/renderer/src/lib/files/search/quick-open-file-history.ts
1 src/renderer/src/lib/list-interaction/use-windowed-command-virtualizer.ts
2 src/renderer/src/runtime/agent-orchestration/worker-auto-close.ts
1 src/renderer/src/runtime/native-chat/naming/structured-chat-name-observation.ts
1 src/renderer/src/store/plugins/appearance/plugin-icon-themes.ts
@@ -33,6 +33,8 @@ src/renderer/src/components/native-chat/tools/runs/use-native-chat-tool-run-wind
src/renderer/src/components/right-sidebar/file-explorer/panel/FileExplorerVirtualRows.tsx
src/renderer/src/components/right-sidebar/file-explorer/tree/FileExplorerFilesTreePane.tsx
src/renderer/src/components/sidebar/worktree-list/viewport/VirtualizedWorktreeViewport.tsx
src/renderer/src/components/automations/authoring/WorkspaceCombobox.tsx
src/renderer/src/components/skills/installation/SkillInstallWorkspaceCombobox.tsx
# The workbench subtree relies on the shell re-rendering on every app render.
src/renderer/src/app-shell/AppWorkspaceShell.tsx
File diff suppressed because one or more lines are too long
@@ -1742,12 +1742,6 @@
"auto.components.settings.ExperimentalPane.agentHibernation.toggleLabel": {
"ko": "에이전트 최대 절전 전환"
},
"auto.components.settings.ExperimentalPane.newWorktreeCardStyle.copy": {
"ko": "업데이트된 워크트리 카드 레이아웃, 메타데이터 배치, 카드 표시 메뉴 옵션 및 상태 표시를 미리 봅니다."
},
"auto.components.settings.ExperimentalPane.newWorktreeCardStyle.description": {
"ko": "업데이트된 워크트리 카드 레이아웃, 메타데이터 배치, 카드 표시 메뉴 옵션 및 상태 표시를 미리 봅니다."
},
"auto.components.settings.EphemeralVmsPane.recipes": {
"ko": "레시피"
},
@@ -2585,9 +2579,6 @@
"auto.components.settings.experimental.search.agentHibernation.title": {
"ko": "에이전트 최대 절전"
},
"auto.components.settings.experimental.search.newWorktreeCardStyle.description": {
"ko": "업데이트된 워크트리 카드 레이아웃, 메타데이터 배치, 카드 표시 메뉴 옵션 및 상태 표시를 미리 봅니다."
},
"auto.components.settings.general.search.12ecc640a8": {
"ko": "mru"
},
@@ -75,6 +75,32 @@ const checks = [
],
enabled: process.platform === 'win32'
},
{
name: 'Windows text entry tests',
command: 'powershell.exe',
args: [
'-NoLogo',
'-NoProfile',
'-ExecutionPolicy',
'Bypass',
'-File',
'native/computer-use-windows/runtime-text-entry.test.ps1'
],
enabled: process.platform === 'win32'
},
{
name: 'Windows native control tests',
command: 'powershell.exe',
args: [
'-NoLogo',
'-NoProfile',
'-ExecutionPolicy',
'Bypass',
'-File',
'native/computer-use-windows/runtime-native-controls.test.ps1'
],
enabled: process.platform === 'win32'
},
{
name: 'macOS helper app bundle and signature',
run: verifyMacOSHelperApp,
+1 -1
View File
@@ -56,7 +56,7 @@
5 src/renderer/src/store/repos/repo-catalog-merge.ts # store writer
5 src/renderer/src/store/repos/repo-removal.ts # store writer
6 src/renderer/src/store/slices/diff-comments/diff-comment-persistence.ts # store writer
12 src/renderer/src/store/slices/workspace-attention/workspace-unread-flag.ts # store writer
10 src/renderer/src/store/slices/workspace-attention/workspace-unread-flag.ts # store writer
2 src/renderer/src/store/slices/worktrees/create/created-worktree-state-merge.ts # store writer
1 src/renderer/src/store/slices/worktrees/listing/detected-worktree-meta.ts # P5b-2 must clear: feeds an existing row-type cast
3 src/renderer/src/store/slices/worktrees/listing/fetch-detected-worktrees.ts # store writer
+13 -3
View File
@@ -71,6 +71,16 @@ orca computer get-app-state --app com.microsoft.edgemac --json
App names work when unambiguous (`--app Spotify`). Use `--app pid:<number>` only when bundle ID and name both collide.
## Starting an app
`list-apps` shows only running apps. To start a closed one without pulling you away from what you're doing, use `launch-app`:
```
orca computer launch-app TextEdit --json
```
It asks macOS and Windows to start the app in the background and leaves an app that's already running alone. Some apps still come forward on their own, so the result's `delivery` says whether the app ended up in front. `--activate` brings it forward instead. Linux doesn't support it yet.
## Available actions
```
@@ -90,9 +100,9 @@ Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over
`hotkey` and `press-key` refuse shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session, such as `Cmd+Q`, `Shift+Cmd+Q`, `Ctrl+Cmd+Q` and `Option+Cmd+Esc` on macOS, `Alt+F4` and `Win+L` on Windows, and `Ctrl+Q`, `Alt+F4`, `Super+L` and `Ctrl+Alt+Delete` on Linux. The error code is `system_shortcut_blocked`. Pass `--allow-system-shortcut` to send one anyway.
`type-text --element-index <i>` inserts the text at that field's caret without focusing it or bringing its window forward. It works on macOS and Linux; on Windows, click the field, confirm it is the focused text receiver, and run `type-text` without the index.
`type-text --element-index <i>` inserts the text at that field's caret without focusing it or bringing its window forward. It works on macOS, Linux and Windows. On macOS, and for a Windows field written through its UI Automation value, a field that doesn't report its caret gets the text at the end. An app can still bring its own window forward in response, so check `action.delivery`.
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. The bundled skill tells agents never to add it on their own: they ask you first, and error messages say the same. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux, element clicks through accessibility actions, `set-value`, `type-text --element-index` and secondary actions run in the background; actions that need the real pointer or the focused window are refused unless you pass `--restore-window`. On Wayland they are refused even with it, because synthetic input never reaches native windows. On Windows, element clicks, `set-value` (including sliders), scrolls and secondary actions (`Invoke`, `Toggle`, `Select`, `Expand`, `Collapse`, `ScrollToVisible`) go through UI Automation when the control supports them, without moving the pointer or sending keys. Some apps still bring their own window forward when driven this way (WPF apps do for most of these), so `action.delivery` reports `foreground` whenever the target window came forward. Actions that need the real mouse, such as coordinate clicks, right-clicks and drags, are refused unless you pass `--restore-window`; then only the target window comes forward and the pointer goes back where it was.
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. The bundled skill tells agents never to add it on their own: they ask you first, and error messages say the same. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux, element clicks through accessibility actions, `set-value`, `type-text --element-index` and secondary actions run in the background; actions that need the real pointer or the focused window are refused unless you pass `--restore-window`. On Wayland they are refused even with it, because synthetic input never reaches native windows. On Windows, element clicks, `set-value` (including sliders), scrolls and secondary actions (`Invoke`, `Toggle`, `Select`, `Expand`, `Collapse`, `ScrollToVisible`) go through UI Automation when the control supports them, without moving the pointer or sending keys. `type-text --element-index` also works there for text boxes. Some apps still bring their own window forward when driven this way (WPF apps do for most of these), so `action.delivery` reports `foreground` whenever the target window came forward. Actions that need the real mouse, such as coordinate clicks, right-clicks and drags, are refused unless you pass `--restore-window`; then only the target window comes forward and the pointer goes back where it was. Keyboard input (`type-text` without an index, `press-key`, `hotkey`, `paste-text`) is refused the same way while the window isn't in front.
When accessibility targeting fails, fall back carefully to coordinates:
@@ -116,7 +126,7 @@ On Linux and Windows, action payloads also pass briefly through a local operatio
## Screenshots
`get-app-state` returns an accessibility tree and, by default, a screenshot. With `--json`, the image bytes are written to disk and the path is returned in `screenshot.path` rather than embedded in the response. Pass `--no-screenshot` when pixels aren't needed (faster, smaller payload). On macOS, a window that is minimized, hidden or on another Space can't be read; bring it on screen, or pass `--restore-window` to have Orca bring it forward.
`get-app-state` returns an accessibility tree and, by default, a screenshot. With `--json`, the image bytes are written to disk and the path is returned in `screenshot.path` rather than embedded in the response. Pass `--no-screenshot` when pixels aren't needed (faster, smaller payload). On macOS, a window that is minimized or hidden has no pixels, so the screenshot fails while the accessibility tree still comes back; a window on another Space is captured.
On Wayland, screenshots work on KDE Plasma, Hyprland and Sway. On Hyprland and Sway they need `grim`, GdkPixbuf, and `hyprctl` or `swaymsg`; Orca finds the target window by process ID and title and captures its region, so overlapping windows can show in the image.
+3
View File
@@ -39,6 +39,8 @@ orca open --json
orca status --json
```
`orca open` starts Orca without taking keyboard focus from the app you're using. If an Orca window is already open, it does nothing.
Use `--json` when another tool will parse the result. Human-readable output is for quick terminal checks.
## Selectors
@@ -306,6 +308,7 @@ Use `orca computer` for native desktop apps outside the built-in browser:
```bash
orca computer permissions --json
orca computer list-apps --json
orca computer launch-app Safari --json
orca computer get-app-state --app com.apple.Safari --json
orca computer click --app com.apple.Safari --element-index 12 --json
orca computer paste-text --app com.apple.Safari --text "hello" --json
@@ -6,7 +6,6 @@ import org.junit.Assert.assertEquals
import org.junit.Test
class BoundedGunzipTest {
// Shared with the Swift checks and the JS tests, which pin fflate's answer to the same inputs.
private val vectors = JSONArray(File("../tests/gunzip-vectors.json").readText())
@Test
@@ -26,11 +26,10 @@ import Foundation
}
static func main() throws {
// Shared with the Kotlin tests and the JS tests, which pin fflate's answer to the same inputs.
let url = URL(fileURLWithPath: "tests/gunzip-vectors.json")
let vectors = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) as! [[String: Any]]
for vector in vectors where [
"one member", "reaches the limit", "bomb", "bad CRC", "trailing data", "truncated body"
"one member", "reaches the limit", "bad CRC", "trailing data", "truncated body"
].contains(vector["name"] as? String ?? "") {
let name = vector["name"] as! String
let expected = (vector["native"] as? String).map { bytes(hex: $0) }
@@ -6,13 +6,6 @@
"native": "77696e646f7720303a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720313a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720323a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720333a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720343a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720353a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a",
"js": "77696e646f7720303a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720313a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720323a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720333a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720343a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720353a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a"
},
{
"name": "empty member",
"gzipHex": "1f8b080000000000001303000000000000000000",
"outputLimit": 1,
"native": "",
"js": ""
},
{
"name": "reaches the limit",
"gzipHex": "1f8b08000000000000132bcfcc4bc92f5730b052a8562a482cc950b2524a2c2e4e2d29d64fd4cb2a56d251cacf56b22a292a4dd5513ab70728f968cee40ff3673629d5729543741a92add3886c9dc664eb34215ba729e93a01177932565c010000",
@@ -20,13 +13,6 @@
"native": "77696e646f7720303a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720313a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720323a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720333a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720343a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f772035",
"js": "77696e646f7720303a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720313a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720323a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720333a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720343a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f772035"
},
{
"name": "bomb",
"gzipHex": "1f8b0800000000000013edc13101000000c2a0f54f6d085fa000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e031cea38a700001000",
"outputLimit": 1025,
"native": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
"js": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
},
{
"name": "name, comment and header CRC",
"gzipHex": "1f8b081a000000000003612e6a73006e6f74650071bc2bcfcc4bc92f5730b052a8562a482cc950b2524a2c2e4e2d29d64fd4cb2a56d251cacf56b22a292a4dd5513ab70728f968cee40ff3673629d5729543741a92add3886c9dc664eb34215ba729e93a01177932565c010000",
@@ -69,13 +55,6 @@
"native": null,
"js": "77696e646f7720303a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720313a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720323a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720333a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720343a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720353a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a"
},
{
"name": "second member",
"gzipHex": "1f8b08000000000000132bcfcc4bc92f5730b052a8562a482cc950b2524a2c2e4e2d29d64fd4cb2a56d251cacf56b22a292a4dd5513ab70728f968cee40ff3673629d5729543741a92add3886c9dc664eb34215ba729e93a01177932565c0100001f8b08000000000000132bcfcc4bc92f5730b052a8562a482cc950b2524a2c2e4e2d29d64fd4cb2a56d251cacf56b22a292a4dd5513ab70728f968cee40ff3673629d5729543741a92add3886c9dc664eb34215ba729e93a01177932565c010000",
"outputLimit": 697,
"native": null,
"js": "77696e646f7720303a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720313a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720323a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720333a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720343a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a77696e646f7720353a207b2270617468223a226173736574732f612e6a73222c226f6b223a747275652c22cebc223a22e29c93f09f9982227d0a"
},
{
"name": "truncated body",
"gzipHex": "1f8b08000000000000132bcfcc4bc92f5730b052a8562a482cc950b2524a",
@@ -97,20 +76,6 @@
"native": null,
"js": null
},
{
"name": "unknown method",
"gzipHex": "1f8b09000000000000132bcfcc4bc92f5730b052a8562a482cc950b2524a2c2e4e2d29d64fd4cb2a56d251cacf56b22a292a4dd5513ab70728f968cee40ff3673629d5729543741a92add3886c9dc664eb34215ba729e93a01177932565c010000",
"outputLimit": 349,
"native": null,
"js": null
},
{
"name": "header only",
"gzipHex": "1f8b0800000000000013",
"outputLimit": 349,
"native": null,
"js": null
},
{
"name": "empty input",
"gzipHex": "",
+3 -200
View File
@@ -1,21 +1,9 @@
import { describe, expect, it } from 'vitest'
import { harness, ID, OTHER, subscribeFrame, type Harness } from './bridge-host-test-harness'
import { BRIDGE_MAX_UNACKED_BYTES, BRIDGE_MAX_UNACKED_FRAMES } from './bridge-host-subscriptions'
import {
bridgeId,
clientFrame,
createFakeRpcClient,
flushBridge,
rpcSuccess
} from './bridge-host-test-fakes'
import {
BRIDGE_MAX_MESSAGE_BYTES,
BRIDGE_MAX_PENDING_DESKTOP_REQUESTS,
BRIDGE_MAX_REPLY_BYTES
} from './bridge/bridge-caps'
import type { BridgeHostMessage } from './bridge/bridge-envelope'
import { BridgeReplyAssembler } from './bridge/bridge-reply-chunking'
import { BRIDGE_MAX_UNACKED_FRAMES } from './bridge-host-subscriptions'
import { clientFrame, createFakeRpcClient, flushBridge, rpcSuccess } from './bridge-host-test-fakes'
import { BRIDGE_MAX_MESSAGE_BYTES } from './bridge/bridge-caps'
describe('requests', () => {
it('answers a synchronous throw from the client and frees the slot', () => {
@@ -54,46 +42,6 @@ describe('requests', () => {
})
})
it('refuses a subscription id as a request id, because one ledger answers for both', () => {
const bridge = harness({ ready: true })
bridge.host.receive(subscribeFrame(ID))
expect(bridge.client.streams).toHaveLength(1)
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'status.get' }))
expect(bridge.client.requests).toHaveLength(0)
expect(bridge.last()).toMatchObject({
type: 'error',
error: { message: expect.stringContaining('already in flight') }
})
})
it('holds the cap against a page that closes between batches', async () => {
const bridge = harness({ ready: true })
const fill = (offset: number): void => {
for (let index = 0; index < BRIDGE_MAX_PENDING_DESKTOP_REQUESTS; index += 1) {
bridge.host.receive(
clientFrame({ type: 'request', id: bridgeId(offset + index), method: 'status.get' })
)
}
}
fill(0)
expect(bridge.client.requests).toHaveLength(BRIDGE_MAX_PENDING_DESKTOP_REQUESTS)
expect(bridge.frames().filter((frame) => frame.type !== 'init')).toEqual([])
// Desktop calls survive page close because sendRequest has no cancel; keep their slots occupied instead of resetting the cap.
bridge.host.receive(clientFrame({ type: 'close' }))
bridge.host.receive(clientFrame({ type: 'ready' }))
fill(100)
expect(bridge.client.requests).toHaveLength(BRIDGE_MAX_PENDING_DESKTOP_REQUESTS)
expect(bridge.frames().filter((frame) => frame.type === 'error')).toHaveLength(
BRIDGE_MAX_PENDING_DESKTOP_REQUESTS
)
for (const request of bridge.client.requests) {
request.resolve(rpcSuccess('wire-1', 'ok'))
}
await flushBridge()
fill(200)
expect(bridge.client.requests).toHaveLength(BRIDGE_MAX_PENDING_DESKTOP_REQUESTS * 2)
})
it('stops answering a cancelled request without pretending the desktop stopped running it', async () => {
const bridge = harness({ ready: true })
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'status.get' }))
@@ -104,35 +52,6 @@ describe('requests', () => {
})
})
describe('replies too big for one frame', () => {
it('chunks and reassembles to the same payload', async () => {
const bridge = harness({ ready: true })
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'worktree.list' }))
const payload = rpcSuccess('wire-1', 'y'.repeat(BRIDGE_MAX_MESSAGE_BYTES * 2))
bridge.client.requests[0]?.resolve(payload)
await flushBridge()
const replies = bridge.frames()
expect(replies.length).toBeGreaterThan(1)
const assembler = new BridgeReplyAssembler()
const assembled = replies.map((frame) =>
frame.type === 'reply' ? assembler.accept(frame) : { status: 'pending' as const }
)
expect(assembled.at(-1)).toEqual({ status: 'complete', payload })
})
it('aborts the request over the reply ceiling rather than truncating an answer', async () => {
const bridge = harness({ ready: true })
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'worktree.list' }))
bridge.client.requests[0]?.resolve(rpcSuccess('wire-1', 'y'.repeat(BRIDGE_MAX_REPLY_BYTES + 1)))
await flushBridge()
const frame = bridge.last()
expect(frame.type === 'error' && frame.error).toMatchObject({
category: 'BridgeReplyUndeliverableError',
isRpcDeliveryUnknown: false
})
})
})
describe('subscriptions', () => {
it('unsubscribes on cancel, says so, and delivers nothing after', () => {
const bridge = harness({ ready: true })
@@ -179,18 +98,6 @@ describe('backpressure', () => {
}
}
it('sends exactly the unacked frame window and then ends with overflow', () => {
const bridge = harness({ ready: true })
bridge.host.receive(subscribeFrame(ID, WINDOWED))
fill(bridge, BRIDGE_MAX_UNACKED_FRAMES)
expect(bridge.frames().filter((frame) => frame.type === 'event')).toHaveLength(
BRIDGE_MAX_UNACKED_FRAMES
)
fill(bridge, 1)
expect(bridge.last()).toEqual({ v: 1, type: 'end', id: ID, reason: 'overflow' })
expect(bridge.client.streams[0]?.unsubscribes).toBe(1)
})
it('acks only up to the seq it was given', () => {
const bridge = harness({ ready: true })
bridge.host.receive(subscribeFrame(ID, WINDOWED))
@@ -204,58 +111,6 @@ describe('backpressure', () => {
expect(bridge.last()).toEqual({ v: 1, type: 'end', id: ID, reason: 'overflow' })
})
it('ends on the unacked byte window well before the frame window is reached', () => {
const bridge = harness({ ready: true })
bridge.host.receive(subscribeFrame(ID, WINDOWED))
const chunk = 'z'.repeat(BRIDGE_MAX_MESSAGE_BYTES - 1024)
const ended = (): boolean => (bridge.posted.at(-1) ?? '').includes('"type":"end"')
for (let index = 0; index < BRIDGE_MAX_UNACKED_FRAMES && !ended(); index += 1) {
bridge.client.streams[0]?.emit(chunk)
}
const events = bridge.posted.filter((json) => JSON.parse(json).type === 'event')
expect(events.length).toBeGreaterThan(0)
expect(events.length).toBeLessThan(BRIDGE_MAX_UNACKED_FRAMES)
const eventBytes = events.reduce(
(total, json) => total + new TextEncoder().encode(json).byteLength,
0
)
expect(eventBytes).toBeLessThanOrEqual(BRIDGE_MAX_UNACKED_BYTES)
const nextBytes = new TextEncoder().encode(
JSON.stringify({ v: 1, type: 'event', id: ID, seq: events.length + 1, payload: chunk })
).byteLength
expect(eventBytes + nextBytes).toBeGreaterThan(BRIDGE_MAX_UNACKED_BYTES)
expect(bridge.last()).toEqual({ v: 1, type: 'end', id: ID, reason: 'overflow' })
})
it('reopens the byte window on ack, not just the frame window', () => {
const bridge = harness({ ready: true })
bridge.host.receive(subscribeFrame(ID, WINDOWED))
const chunk = 'z'.repeat(BRIDGE_MAX_MESSAGE_BYTES - 1024)
bridge.client.streams[0]?.emit(chunk)
const first = bridge.posted.find((json) => JSON.parse(json).type === 'event')
expect(first).toBeDefined()
if (first === undefined) {
throw new Error('no event posted')
}
const frameBytes = new TextEncoder().encode(first).byteLength
const fits = Math.floor(BRIDGE_MAX_UNACKED_BYTES / frameBytes)
const events = (): BridgeHostMessage[] => bridge.frames().filter((f) => f.type === 'event')
const emit = (times: number): void => {
for (let index = 0; index < times; index += 1) {
bridge.client.streams[0]?.emit(chunk)
}
}
emit(fits - 1)
expect(events()).toHaveLength(fits)
bridge.host.receive(clientFrame({ type: 'ack', id: ID, seq: fits }))
emit(fits)
expect(frameBytes * fits * 2).toBeGreaterThan(BRIDGE_MAX_UNACKED_BYTES)
// Keep the frame-count window open so only released acknowledgement bytes can unblock the batch.
expect(fits * 2).toBeLessThan(BRIDGE_MAX_UNACKED_FRAMES)
expect(events()).toHaveLength(fits * 2)
expect(bridge.frames().some((frame) => frame.type === 'end')).toBe(false)
})
it('keeps each stream on its own window', () => {
const bridge = harness({ ready: true })
bridge.host.receive(subscribeFrame(ID, WINDOWED))
@@ -269,42 +124,6 @@ describe('backpressure', () => {
})
describe('teardown', () => {
it('rejects every pending as delivery-unknown, ends every stream, and refuses later frames', async () => {
const bridge = harness({ ready: true })
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'status.get' }))
bridge.host.receive(subscribeFrame(OTHER))
bridge.host.dispose()
expect(bridge.frames().filter((frame) => frame.type !== 'init')).toEqual([
{
v: 1,
type: 'error',
id: ID,
error: {
category: 'BridgeHostDisposedError',
code: 'bridge_host_disposed',
message: 'the page bridge was torn down before this request answered',
isRpcDeliveryUnknown: true
}
},
{ v: 1, type: 'end', id: OTHER, reason: 'closed' }
])
expect(bridge.client.streams[0]?.unsubscribes).toBe(1)
bridge.client.requests[0]?.resolve(rpcSuccess('wire-1', 'ok'))
bridge.client.streams[0]?.emit({ chunk: 'a' })
bridge.host.receive(clientFrame({ type: 'request', id: bridgeId(9), method: 'status.get' }))
bridge.host.receive(subscribeFrame(bridgeId(10)))
bridge.host.receive(clientFrame({ type: 'notify', name: 'foreground' }))
bridge.host.receive(clientFrame({ type: 'ready' }))
await flushBridge()
expect(bridge.frames()).toHaveLength(3)
expect(bridge.client.requests).toHaveLength(1)
expect(bridge.client.streams).toHaveLength(1)
expect(bridge.client.foregroundCalls).toEqual([])
expect(bridge.diagnostics).toEqual(
Array.from({ length: 4 }, () => ({ kind: 'frame-after-dispose' }))
)
})
it('settles what the page owned on close without answering a page that said goodbye', async () => {
const bridge = harness({ ready: true })
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'status.get' }))
@@ -332,22 +151,6 @@ describe('teardown', () => {
expect(bridge.diagnostics).toEqual([])
})
it('forwards no straggler from the document that said goodbye', () => {
const bridge = harness()
bridge.host.receive(clientFrame({ type: 'close' }))
// Refuse stragglers from the closed document rather than answering into its replacement.
bridge.host.receive(clientFrame({ type: 'request', id: ID, method: 'status.get' }))
bridge.host.receive(subscribeFrame(OTHER))
bridge.host.receive(clientFrame({ type: 'notify', name: 'foreground' }))
expect(bridge.client.requests).toHaveLength(0)
expect(bridge.client.streams).toHaveLength(0)
expect(bridge.client.foregroundCalls).toEqual([])
expect(bridge.posted).toHaveLength(0)
expect(bridge.diagnostics).toEqual(
Array.from({ length: 3 }, () => ({ kind: 'frame-after-close' }))
)
})
it('posts nothing into a view that belongs to no document yet', () => {
const bridge = harness()
bridge.host.receive(clientFrame({ type: 'close' }))
@@ -139,4 +139,19 @@ describe('useMobileNewTabAgentOptions', () => {
expect(drawer.options()).toEqual([])
drawer.unmount()
})
it('keeps a stale-sheet automatic refresh distinct from an explicit user refresh', async () => {
const client = createClient({ agents: () => claude })
const first = mountDrawer(queryClient)
await first.render({ client, open: true })
first.unmount()
vi.setSystemTime(1_000_001)
const reopened = mountDrawer(queryClient)
await reopened.render({ client, open: true })
expect(
client.sendRequest.mock.calls
.filter(([method]) => method === 'preflight.refreshAgents')
.map(([, params]) => params)
).toEqual([expect.not.objectContaining({ userInitiated: true })])
reopened.unmount()
})
})
@@ -0,0 +1,73 @@
import { describe, expect, it, vi } from 'vitest'
import type { RpcResponse } from '../../transport/rpc/rpc-wire-types'
import { selectOpenedSessionDiff } from './select-opened-session-diff'
function ok(result: unknown): RpcResponse {
return { id: 'rpc', ok: true, result, _meta: { runtimeId: 'runtime-1' } }
}
describe('selectOpenedSessionDiff', () => {
it('selects the opened diff for this phone once the desktop lists it', async () => {
vi.useFakeTimers()
try {
let lists = 0
const sendRequest = vi.fn(async (method: string, _params?: unknown) => {
if (method === 'session.tabs.list') {
lists += 1
return ok({
tabs:
lists === 1
? []
: [
{
id: 'staged',
type: 'file',
mode: 'diff',
relativePath: 'a.ts',
diffSource: 'staged'
},
{ id: 'edit', type: 'file', mode: 'edit', relativePath: 'a.ts' },
{ id: 'branch', type: 'file', mode: 'diff', relativePath: 'a.ts' },
{
id: 'unstaged',
type: 'file',
mode: 'diff',
relativePath: 'a.ts',
diffSource: 'unstaged'
}
]
})
}
return ok({ activeTabId: 'unstaged' })
})
const selecting = selectOpenedSessionDiff(
{ sendRequest },
{
worktreeId: 'wt-1',
relativePath: 'a.ts',
staged: false
},
() => true
)
await vi.runAllTimersAsync()
await selecting
expect(
sendRequest.mock.calls.filter(([method]) => method === 'session.tabs.activate')
).toEqual([
[
'session.tabs.activate',
{
worktree: 'id:wt-1',
tabId: 'unstaged',
notifyClients: false,
navigation: 'caller',
intent: 'user'
}
]
])
} finally {
vi.useRealTimers()
}
})
})
@@ -0,0 +1,45 @@
import { sessionFileTabListRead } from '../../files/session-documents/mobile-source-file-open-operations'
import { activateMobileSessionTab } from '../../session/tabs/mobile-session-tab-activation'
// The desktop publishes the tab after files.openDiff replies; a few short polls cover that hop.
const TAB_POLL_DELAYS_MS = [0, 300, 600, 900] as const
/** Selects the diff tab "Open in session" opened for this phone only; the desktop keeps its own view (#33052). */
export async function selectOpenedSessionDiff(
client: Parameters<typeof activateMobileSessionTab>[0],
target: { worktreeId: string; relativePath: string; staged: boolean },
isCurrent: () => boolean
): Promise<void> {
const worktree = `id:${target.worktreeId}`
const source = target.staged ? 'staged' : 'unstaged'
for (const delayMs of TAB_POLL_DELAYS_MS) {
if (delayMs > 0) {
await new Promise<void>((resolve) => setTimeout(resolve, delayMs))
}
if (!isCurrent()) {
return
}
const listed = sessionFileTabListRead.interpret(
await sessionFileTabListRead.request(client, { worktree })
)
const tab = listed.accepted
? listed.value?.tabs.find(
(candidate) =>
candidate.mode === 'diff' &&
candidate.relativePath === target.relativePath &&
// Branch and commit diffs list no source; they are never this working-tree diff.
candidate.diffSource === source
)
: undefined
if (tab) {
await activateMobileSessionTab(client, {
worktree,
tabId: tab.id,
notifyClients: false,
navigation: 'caller',
intent: 'user'
})
return
}
}
}
@@ -1,4 +1,4 @@
import { useRef, type Dispatch, type SetStateAction } from 'react'
import { useEffect, useRef, type Dispatch, type SetStateAction } from 'react'
import { translate } from '../../i18n/mobile-i18n'
import type { ConnectionState } from '../../transport/connection/connection-state'
import type { RpcClient } from '../../transport/rpc/rpc-client-contract'
@@ -8,6 +8,7 @@ import type { ReviewSheet, ReviewSheetIntents } from './mobile-diff-review-sheet
import type { MobileReviewTerminalTab } from './review-terminal-reply-schema'
import type { MobileDiffReviewNoteDraft } from './use-mobile-diff-review-note-draft'
import { sourceFileDiffOpenRun } from '../../files/session-documents/mobile-source-file-open-operations'
import { selectOpenedSessionDiff } from './select-opened-session-diff'
import { refusedRpcMessageOrFallback } from '../../transport/rpc/rpc-refusal-message'
import { useMobileDiffReviewCommentActions } from './use-mobile-diff-review-comment-actions'
import { useMobileDiffReviewGitActions } from './use-mobile-diff-review-git-actions'
@@ -73,6 +74,14 @@ export function useMobileDiffReviewInteractions(input: InteractionInput) {
// Bumped by every file the reader picks, so a slow save or stage never overrides a newer pick.
const navigationRef = useRef(0)
// Open in session waits for the desktop's tab; leaving first must not replace the route the user went to.
const mountedRef = useRef(true)
useEffect(() => {
mountedRef.current = true
return () => {
mountedRef.current = false
}
}, [])
const {
clearSavedNote,
@@ -169,23 +178,38 @@ export function useMobileDiffReviewInteractions(input: InteractionInput) {
) {
return
}
const response = await sourceFileDiffOpenRun.request(client, {
worktree: `id:${worktreeId}`,
const target = {
worktreeId,
relativePath: currentItem.filePath,
staged: currentItem.scope === 'staged'
})
try {
sourceFileDiffOpenRun.interpret(response)
} catch (error) {
setActionError(
refusedRpcMessageOrFallback(
error,
translate('session.useDiffReviewInteractions.openFailed', 'Unable to open in session')
)
)
return
}
onOpenSession()
setBusyAction('open-in-session')
try {
const response = await sourceFileDiffOpenRun.request(client, {
worktree: `id:${worktreeId}`,
relativePath: target.relativePath,
staged: target.staged
})
try {
sourceFileDiffOpenRun.interpret(response)
} catch (error) {
setActionError(
refusedRpcMessageOrFallback(
error,
translate('session.useDiffReviewInteractions.openFailed', 'Unable to open in session')
)
)
return
}
const isCurrent = () => mountedRef.current
// Best effort: the session still opens, on this phone's previous tab, if selecting the diff fails.
await selectOpenedSessionDiff(client, target, isCurrent).catch(() => {})
if (isCurrent()) {
onOpenSession()
}
} finally {
setBusyAction(null)
}
},
openSendSheet,
retryAction: connectionRetryAction({
+432 -52
View File
@@ -156,20 +156,93 @@ class GeometryReader:
finally:
Atspi.set_timeout(ATSPI_METHOD_TIMEOUT_MS, ATSPI_STARTUP_TIMEOUT_MS)
def rect(self, node):
return screen_rect(node, self.read)
def screen_rect(node, read=attempt):
return extents_rect(node, Atspi.CoordType.SCREEN, read)
def extents_rect(node, coord_type, read=attempt):
component = read(node.get_component_iface)
if component is None:
return None
rect = read(lambda: Atspi.Component.get_extents(component, Atspi.CoordType.SCREEN))
rect = read(lambda: Atspi.Component.get_extents(component, coord_type))
if rect is None or rect.width <= 0 or rect.height <= 0:
return None
return Rect(float(rect.x), float(rect.y), float(rect.width), float(rect.height))
UNREAD = object()
class Placement:
"""Screen and window-relative rectangles for one window's nodes.
GTK 4 reports SCREEN extents of [0, 0, w, h] for nearly every node, so its rectangles come from WINDOW
extents plus the X11 client origin (agent-desktop probes/linux/FINDINGS.md L23-37).
"""
def __init__(self, app, window):
self.app, self.window = app, window
self.base = self._origin = UNREAD
self.window_coords = toolkit_kind(app) == "gtk4"
@property
def origin(self):
if self._origin is UNREAD:
self._origin = (gtk4_client_origin(self.app, self.window)
if self.window_coords and not wayland_session() else None)
return self._origin
def screen(self, node, read=attempt):
if not self.window_coords:
return screen_rect(node, read)
rect = extents_rect(node, Atspi.CoordType.WINDOW, read) if self.origin is not None else None
return None if rect is None else Rect(rect.x + self.origin[0], rect.y + self.origin[1], rect.width, rect.height)
def bounds(self):
if self.window_coords and wayland_session():
# Why: Wayland capture finds the window through the compositor and only checks its size.
return screen_rect(self.window)
return self.screen(self.window)
def frame(self, node, read=attempt):
coord_type = Atspi.CoordType.WINDOW if self.window_coords else Atspi.CoordType.SCREEN
if self.base is UNREAD:
self.base = extents_rect(self.window, coord_type, read)
rect = self.base if node is self.window else extents_rect(node, coord_type, read)
if rect is None or self.base is None:
return None
return Rect(rect.x - self.base.x, rect.y - self.base.y, rect.width, rect.height)
# Why per operation: list-windows places every window of an app, and each lookup is a subprocess.
_x11_windows_by_pid = {}
def gtk4_client_origin(app, window):
xdotool = shutil.which("xdotool")
if not xdotool:
return None
pid = pid_of(app)
if pid not in _x11_windows_by_pid:
ids = x11_window_ids(app, xdotool) or []
_x11_windows_by_pid[pid] = [(value, x11_window_name(xdotool, value) if len(ids) > 1 else None) for value in ids]
windows = _x11_windows_by_pid[pid]
if len(windows) > 1:
title = name_of(window)
windows = [item for item in windows if item[1] == title]
return x11_client_origin(int(windows[0][0])) if len(windows) == 1 else None
def x11_window_name(xdotool, window_id):
try:
result = subprocess.run([xdotool, "getwindowname", window_id], check=False, capture_output=True,
text=True, timeout=2)
except (OSError, subprocess.TimeoutExpired):
return None
return result.stdout.rstrip("\n") if result.returncode == 0 else None
def desktop_apps():
for _, app in children(desktop_root()):
if name_of(app):
@@ -323,36 +396,91 @@ def restore_pointer_window(app, window):
restore_window(app, window)
if not wait_window_focused(window):
raise RuntimeError("window_not_focused: window restoration did not focus the target window; no pointer input was sent")
return screen_rect(window)
X11_WINDOW = ctypes.c_ulong
X11_ERROR_HANDLER = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)
# Why kept: Xlib's default handler exits the process on a BadWindow from a window that closed mid-read.
_ignore_x11_errors = X11_ERROR_HANDLER(lambda _display, _event: 0)
@contextmanager
def x11_display():
"""libX11 and an open display, or (None, None) when no X11 display answers."""
try:
x = ctypes.CDLL("libX11.so.6")
except OSError:
yield None, None
return
pointer, window, integer = ctypes.c_void_p, X11_WINDOW, ctypes.c_int
for name, result, args in (
("XOpenDisplay", pointer, [ctypes.c_char_p]),
("XCloseDisplay", integer, [pointer]),
("XSetErrorHandler", X11_ERROR_HANDLER, [X11_ERROR_HANDLER]),
("XDefaultRootWindow", window, [pointer]),
("XQueryPointer", integer, [pointer, window, ctypes.POINTER(window), ctypes.POINTER(window),
*[ctypes.POINTER(integer)] * 4, ctypes.POINTER(ctypes.c_uint)]),
("XTranslateCoordinates", integer, [pointer, window, window, integer, integer, ctypes.POINTER(integer),
ctypes.POINTER(integer), ctypes.POINTER(window)]),
("XInternAtom", ctypes.c_ulong, [pointer, ctypes.c_char_p, integer]),
("XGetWindowProperty", integer, [pointer, window, ctypes.c_ulong, ctypes.c_long, ctypes.c_long, integer,
ctypes.c_ulong, ctypes.POINTER(ctypes.c_ulong), ctypes.POINTER(integer),
ctypes.POINTER(ctypes.c_ulong), ctypes.POINTER(ctypes.c_ulong),
ctypes.POINTER(ctypes.POINTER(ctypes.c_long))]),
("XFree", integer, [pointer]),
):
fn = getattr(x, name)
fn.restype, fn.argtypes = result, args
display = x.XOpenDisplay(None)
if not display:
yield None, None
return
previous = x.XSetErrorHandler(_ignore_x11_errors)
try:
yield x, display
finally:
x.XCloseDisplay(display)
x.XSetErrorHandler(previous)
def x11_pointer_state():
"""The X11 pointer's root position (None off the default screen) and modifier mask, or None with no X11 display."""
try:
x = ctypes.CDLL("libX11.so.6")
except OSError:
return None
pointer, window, integer = ctypes.c_void_p, ctypes.c_ulong, ctypes.c_int
x.XOpenDisplay.restype, x.XOpenDisplay.argtypes = pointer, [ctypes.c_char_p]
x.XDefaultRootWindow.restype, x.XDefaultRootWindow.argtypes = window, [pointer]
x.XQueryPointer.restype = integer
x.XQueryPointer.argtypes = [pointer, window, ctypes.POINTER(window), ctypes.POINTER(window),
*[ctypes.POINTER(integer)] * 4, ctypes.POINTER(ctypes.c_uint)]
x.XCloseDisplay.argtypes = [pointer]
display = x.XOpenDisplay(None)
if not display:
return None
try:
root, child = window(), window()
root_x, root_y, window_x, window_y, mask = integer(), integer(), integer(), integer(), ctypes.c_uint()
with x11_display() as (x, display):
if display is None:
return None
root, child = X11_WINDOW(), X11_WINDOW()
root_x, root_y, window_x, window_y = (ctypes.c_int() for _ in range(4))
mask = ctypes.c_uint()
# Why keep the mask on False: that only means the pointer is on another screen; the modifiers are still valid.
if not x.XQueryPointer(display, x.XDefaultRootWindow(display), ctypes.byref(root), ctypes.byref(child),
ctypes.byref(root_x), ctypes.byref(root_y), ctypes.byref(window_x),
ctypes.byref(window_y), ctypes.byref(mask)):
return None, None, mask.value
return root_x.value, root_y.value, mask.value
finally:
x.XCloseDisplay(display)
def x11_client_origin(window_id):
"""Where the visible area of an X11 client window starts on the root: its origin plus its client-side shadow."""
with x11_display() as (x, display):
if display is None:
return None
root_x, root_y, child = ctypes.c_int(), ctypes.c_int(), X11_WINDOW()
if not x.XTranslateCoordinates(display, window_id, x.XDefaultRootWindow(display), 0, 0,
ctypes.byref(root_x), ctypes.byref(root_y), ctypes.byref(child)):
return None
shadow = (0, 0)
atom = x.XInternAtom(display, b"_GTK_FRAME_EXTENTS", 1)
actual_type, actual_format = ctypes.c_ulong(), ctypes.c_int()
count, remaining, data = ctypes.c_ulong(), ctypes.c_ulong(), ctypes.POINTER(ctypes.c_long)()
if atom and x.XGetWindowProperty(display, window_id, atom, 0, 4, 0, 6, ctypes.byref(actual_type),
ctypes.byref(actual_format), ctypes.byref(count), ctypes.byref(remaining),
ctypes.byref(data)) == 0 and data:
try:
if actual_format.value == 32 and count.value == 4:
shadow = (int(data[0]), int(data[2])) # left, right, top, bottom
finally:
x.XFree(data)
return root_x.value + shadow[0], root_y.value + shadow[1]
@contextmanager
@@ -1293,7 +1421,9 @@ def app_json(app):
def window_json(app, index, window):
bounds = screen_rect(window)
bounds = Placement(app, window).bounds()
# Why the SCREEN fallback: a GTK 4 window with no known origin still has a true size, just no position.
size = bounds or screen_rect(window)
showing = has_state(window, Atspi.StateType.SHOWING)
return {
"index": index,
@@ -1302,8 +1432,8 @@ def window_json(app, index, window):
"title": name_of(window),
"x": round(bounds.x) if bounds else None,
"y": round(bounds.y) if bounds else None,
"width": max(0, round(bounds.width if bounds else 0)),
"height": max(0, round(bounds.height if bounds else 0)),
"width": max(0, round(size.width if size else 0)),
"height": max(0, round(size.height if size else 0)),
"isMinimized": not showing,
"isOffscreen": not showing,
"screenIndex": None,
@@ -1317,7 +1447,8 @@ def make_snapshot(query, include_screenshot, window_id=None, window_index=None,
if restore:
restore_window(app, window)
window_index, window = choose_window(app, window_id, window_index)
bounds = screen_rect(window)
placement = Placement(app, window)
bounds = placement.bounds()
tree_root, root_path = window, [window_index]
if root is not None:
tree_root = find_element(app, root)
@@ -1336,12 +1467,13 @@ def make_snapshot(query, include_screenshot, window_id=None, window_index=None,
app_info = app_json(app)
window_title = name_of(window)
screenshot = capture_png(bounds, app, window) if include_screenshot else None
if include_screenshot and bounds is None and placement.window_coords:
screenshot = screenshot_failure("GTK 4 screenshot skipped: Orca can't tell where this window sits on screen; "
"use --no-screenshot and trust the tree")
geometry = GeometryReader()
for item in records:
node = geometry_nodes[id(item)]
rect = bounds if node is window else geometry.rect(node)
if rect is not None and bounds is not None:
rect = Rect(rect.x - bounds.x, rect.y - bounds.y, rect.width, rect.height)
rect = placement.frame(node, geometry.read)
item["frame"] = rect.to_json() if rect else None
return {
"snapshotId": str(uuid.uuid4()),
@@ -1401,7 +1533,7 @@ def handshake_response():
"pressKey": True,
"hotkey": has_hotkey,
"pasteText": has_clipboard,
"scroll": not is_wayland,
"scroll": True,
"drag": not is_wayland,
"setValue": True,
"performAction": True,
@@ -1424,10 +1556,11 @@ def all_nodes(root):
result = []
def walk(node):
if len(result) >= MAX_NODES:
return
result.append(node)
for _, child in children(node):
# Why check before each child: every lookup is an AT-SPI round trip, and a list can hold thousands.
if len(result) >= MAX_NODES:
return
walk(child)
walk(root)
@@ -1556,8 +1689,12 @@ def activate_focused_entry(app, window):
"verification": {"state": "unverified", "reason": "accessibility_action_unasserted"}}
def screen_point(window_rect, saved_element=None, x=None, y=None, node=None):
rect = screen_rect(node) if node is not None else None
def screen_point(placed, saved_element=None, x=None, y=None, node=None):
if placed.window_coords and placed.origin is None:
raise RuntimeError("unsupported capability: Orca can't tell where this GTK 4 window sits on screen, so "
"pointer input would miss; nothing was sent. Use an element's accessibility action or set-value")
window_rect = placed.screen(placed.window)
rect = placed.screen(node) if node is not None else None
if rect is not None:
return rect.x + rect.width / 2, rect.y + rect.height / 2
if saved_element is not None:
@@ -1567,8 +1704,9 @@ def screen_point(window_rect, saved_element=None, x=None, y=None, node=None):
return window_rect.x + float(x), window_rect.y + float(y)
def window_point(window_rect, saved_element=None, x=None, y=None, node=None):
point = screen_point(window_rect, saved_element, x, y, node)
def window_point(placed, saved_element=None, x=None, y=None, node=None):
point = screen_point(placed, saved_element, x, y, node)
window_rect = placed.screen(placed.window)
if (window_rect is None or not window_rect.x <= point[0] < window_rect.x + window_rect.width
or not window_rect.y <= point[1] < window_rect.y + window_rect.height):
raise RuntimeError("coordinate action target lies outside the target window, so the pointer would land on "
@@ -1696,6 +1834,241 @@ def drag_between(start, end):
Atspi.generate_mouse_event(round(x), round(y), "b1r")
SCROLL_DIRECTIONS = {"up": ("vertical", -1), "down": ("vertical", 1), "left": ("horizontal", -1), "right": ("horizontal", 1)}
DOCUMENT_ROLES = {"document web", "document frame", "scroll pane"}
MAX_SCROLL_BAR_SIBLINGS = 8
MAX_SCROLL_TO_PAGES = 10
SCROLL_SETTLE_SECONDS = 1
def scroll_direction(raw):
direction = str(raw or "").strip().lower()
if not direction:
raise RuntimeError("direction is required")
if direction not in SCROLL_DIRECTIONS:
raise RuntimeError(f"unsupported scroll direction: {direction}")
return SCROLL_DIRECTIONS[direction]
def scroll_without_pointer(window, node, placed, x, y, direction, pages):
"""Scroll through scroll bar values or Component.ScrollTo, which move no pointer and raise no window
(agent-desktop actions/scroll/scroll_bus.rs; FINDINGS L4-22)."""
axis, sign = scroll_direction(direction)
pages = require_positive_number(1 if pages is None else pages, "pages")
if node is None and x is not None and y is not None:
node = node_at(window, placed, float(x), float(y), [MAX_NODES])
at_limit, budget = None, [MAX_NODES]
# Why innermost first: like a wheel, the nearest view that can move takes the scroll, and one at its end hands it out.
for container in ancestors_to(node, window):
bar = scroll_bar(container, axis)
if bar is not None:
action = write_scroll_bar(container, bar, axis, sign, pages)
elif float(pages).is_integer():
action = scroll_to_pages(container, placed, axis, sign, int(pages), budget)
else:
continue
if action is not None and action["verification"].get("reason") != "suspected_noop":
return action
at_limit = at_limit or action
return at_limit
def ancestors_to(node, window):
found, current = [], node
for _ in range(MAX_DEPTH):
if current is None:
break
found.append(current)
if current is window or current == window:
break
current = attempt(lambda current=current: current.get_parent())
return found
def node_at(window, placed, x, y, budget):
"""The deepest node whose window-relative frame holds the point, preferring later (topmost) siblings."""
node = window
for _ in range(MAX_DEPTH):
hit = None
count = int(attempt(lambda node=node: node.get_child_count(), 0) or 0)
for index in range(count - 1, -1, -1):
budget[0] -= 1
if budget[0] < 0:
return node
child = attempt(lambda node=node, index=index: node.get_child_at_index(index))
frame = placed.frame(child) if child is not None else None
if frame is not None and frame.x <= x < frame.x + frame.width and frame.y <= y < frame.y + frame.height:
hit = child
break
if hit is None:
return node
node = hit
return node
def scroll_bar(container, axis):
vertical, horizontal = (getattr(Atspi.StateType, name, None) for name in ("VERTICAL", "HORIZONTAL"))
# Why capped: scroll bars sit beside one viewport child, and reading every row of a big list costs the app memory.
if int(attempt(lambda: container.get_child_count(), 0) or 0) > MAX_SCROLL_BAR_SIBLINGS:
return None
for _, child in children(container):
if role_of(child).lower() != "scroll bar":
continue
if vertical is not None and has_state(child, vertical):
matches = axis == "vertical"
elif horizontal is not None and has_state(child, horizontal):
matches = axis == "horizontal"
else:
rect = screen_rect(child)
matches = rect is not None and (rect.height > rect.width) == (axis == "vertical")
value = attempt(lambda child=child: child.get_value_iface()) if matches else None
if value is None:
continue
low, high, current = (attempt(lambda read=read: read(value)) for read in (
Atspi.Value.get_minimum_value, Atspi.Value.get_maximum_value, Atspi.Value.get_current_value))
if None not in (low, high, current) and high > low:
return value, float(low), float(high), float(current)
return None
def write_scroll_bar(container, bar, axis, sign, pages):
value, low, high, current = bar
viewport = screen_rect(container)
if viewport is None:
return None
# Why the viewport length: toolkit adjustments count pixels, so one page is one viewport (scroll_plan.rs).
page = viewport.height if axis == "vertical" else viewport.width
wanted = min(high, max(low, current + sign * pages * page))
action = {"path": "accessibility", "actionName": "scrollBarValue", "fallbackReason": None}
if abs(wanted - current) < 0.5:
action["verification"] = {"state": "unverified", "reason": "suspected_noop"}
return action
if not attempt(lambda: Atspi.Value.set_current_value(value, wanted), False):
return None
readback = attempt(lambda: Atspi.Value.get_current_value(value))
action["verification"] = ({"state": "verified", "property": "value"}
if readback is not None and abs(float(readback) - wanted) < 0.5
else {"state": "unverified", "reason": "value_mismatch"})
return action
def scroll_to_pages(container, placed, axis, sign, pages, budget):
edges = {("vertical", 1): "TOP_EDGE", ("vertical", -1): "BOTTOM_EDGE",
("horizontal", 1): "LEFT_EDGE", ("horizontal", -1): "RIGHT_EDGE"}
scroll_type = getattr(getattr(Atspi, "ScrollType", None), edges[(axis, sign)], None)
if scroll_type is None:
return None
moved = accepted = 0
at_end = False
for _ in range(min(pages, MAX_SCROLL_TO_PAGES)):
viewport = placed.frame(container)
if viewport is None:
break
start, length = (viewport.y, viewport.height) if axis == "vertical" else (viewport.x, viewport.width)
item = item_past_edge(container, placed, axis, sign, start, start + length, MAX_DEPTH, budget)
at_end = item is None and budget[0] >= 0
component = attempt(lambda: item.get_component_iface()) if item is not None else None
before = placed.frame(item) if component is not None else None
if before is None or not attempt(lambda: Atspi.Component.scroll_to(component, scroll_type), False):
break
accepted += 1
if not wait_item_moved(item, placed, axis, before):
break
moved += 1
if not accepted:
# Why only documents: a document with nothing past its edge is at its end, while other nodes may not scroll.
if at_end and role_of(container).lower() in DOCUMENT_ROLES:
return {"path": "accessibility", "actionName": "scrollTo", "fallbackReason": None,
"verification": {"state": "unverified", "reason": "suspected_noop"}}
return None
# Why an action after any page: once the document moved, falling back to the pointer would scroll it twice.
verification = ({"state": "unverified", "reason": "accessibility_action_unasserted"} if moved
else {"state": "unverified", "reason": "suspected_noop"})
return {"path": "accessibility", "actionName": "scrollTo", "fallbackReason": None, "verification": verification}
def item_past_edge(node, placed, axis, sign, viewport_start, viewport_end, depth, budget):
"""The item that should lead the next page: the first one cut by or beyond the viewport edge we scroll toward."""
if depth <= 0:
return None
edge = viewport_end if sign > 0 else viewport_start
count = int(attempt(lambda: node.get_child_count(), 0) or 0)
for index in (range(count) if sign > 0 else range(count - 1, -1, -1)):
# Why a shared budget: each child is an AT-SPI round trip, and a document can hold thousands.
budget[0] -= 1
if budget[0] < 0:
return None
child = attempt(lambda index=index: node.get_child_at_index(index))
frame = placed.frame(child) if child is not None else None
if frame is None:
continue
start, end = (frame.y, frame.y + frame.height) if axis == "vertical" else (frame.x, frame.x + frame.width)
if (start >= edge - 1) if sign > 0 else (end <= edge + 1):
return child
if start < edge < end:
inner = item_past_edge(child, placed, axis, sign, viewport_start, viewport_end, depth - 1, budget)
# Why the cut item itself: its hidden part leads the next page, unless it already fills the viewport.
leads = start > viewport_start + 1 if sign > 0 else end < viewport_end - 1
if inner is not None or leads:
return inner if inner is not None else child
return None
def wait_item_moved(item, placed, axis, before):
"""Wait until ScrollTo moved the item, so the next page is planned from settled geometry."""
deadline = time.monotonic() + SCROLL_SETTLE_SECONDS
was = before.y if axis == "vertical" else before.x
while True:
frame = placed.frame(item)
now = None if frame is None else frame.y if axis == "vertical" else frame.x
if now is not None and abs(now - was) > 1:
return True
if time.monotonic() >= deadline:
return False
time.sleep(0.05)
def select_in_parent(app, node):
"""Select a row through its container's Selection, proving the index names this row first (select_index.rs)."""
parent = attempt(lambda: node.get_parent()) if node is not None else None
selection = attempt(lambda: parent.get_selection_iface()) if parent is not None else None
index = attempt(lambda: node.get_index_in_parent()) if selection is not None else None
if index is None or index < 0 or not selection_indices_proven(app, parent, selection):
return None
if attempt(lambda: parent.get_child_at_index(index)) != node:
raise RuntimeError("stale element: the selection index no longer names this element; run get-app-state again and use a fresh element index")
multiselectable = getattr(Atspi.StateType, "MULTISELECTABLE", None)
# Why clear first: a plain click replaces the selection, while SelectChild adds to a multi-select list's.
if (multiselectable is not None and has_state(parent, multiselectable)
and not attempt(lambda: Atspi.Selection.clear_selection(selection), False)):
return None
if not attempt(lambda: Atspi.Selection.select_child(selection, index), False):
return None
attempt(lambda: node.clear_cache())
verified = has_state(node, Atspi.StateType.SELECTED)
return {"path": "accessibility", "actionName": "selectChild", "fallbackReason": None,
"verification": {"state": "verified", "property": "selection"} if verified
else {"state": "unverified", "reason": "accessibility_action_unasserted"}}
def selection_indices_proven(app, parent, selection):
# Why: GTK 4 Selection counts item widgets, which can differ from its accessible children (select_index.rs).
if toolkit_kind(app) in {"gtk3", "qt", "chromium", "gecko"}:
return True
count = int(attempt(lambda: parent.get_child_count(), 0) or 0)
def answer(index):
try:
Atspi.Selection.is_child_selected(selection, index)
return "answered"
except Exception as error:
# Why only InvalidArgs: a timeout or lost connection proves nothing about the index space.
return "refused" if "InvalidArgs" in str(error) else "unknown"
# Why counted: the index spaces match exactly when the last child answers and the one past it is refused.
return count > 0 and answer(count - 1) == "answered" and answer(count) == "refused"
def is_enter_key(raw):
return key_name(raw) in {"Return", "KP_Enter"}
@@ -2125,6 +2498,7 @@ def set_value(node, value):
def run_operation(operation):
_x11_windows_by_pid.clear()
tool = operation.get("tool")
include_screenshot = not bool(operation.get("noScreenshot"))
if tool == "handshake":
@@ -2154,11 +2528,11 @@ def run_operation(operation):
action = None
def pointer_input(what, hint, points, send):
points(screen_rect(window)) # Why first: a stale or off-window target is not a reason to ask for a raise.
points(Placement(app, window)) # Why first: a stale or off-window target is not a reason to ask for a raise.
require_pointer_route(operation, what, hint)
restored = restore_pointer_window(app, window)
restore_pointer_window(app, window)
with pointer_put_back():
send(*points(restored))
send(*points(Placement(app, window)))
if tool == "click":
preferred = preferred_action(node)
@@ -2168,7 +2542,8 @@ def run_operation(operation):
else 1
)
has_modifiers = bool(str(operation.get("modifiers") or "").strip())
semantic = not has_modifiers and operation.get("mouse_button", "left") == "left" and click_count <= 1 and preferred is not None
simple = not has_modifiers and operation.get("mouse_button", "left") == "left" and click_count <= 1
semantic = simple and preferred is not None
if semantic:
names = action_names(node)
name = (names[preferred] if preferred < len(names) else "") or "action"
@@ -2176,9 +2551,11 @@ def run_operation(operation):
if semantic and perform_action(node, preferred):
action = {"path": "accessibility", "actionName": name, "fallbackReason": None}
else:
action = select_in_parent(app, node) if simple else None
if action is None:
pointer_input(
"this click", "Click an element by --element-index that has an accessibility action, or use set-value.",
lambda bounds: [window_point(bounds, saved, operation.get("x"), operation.get("y"), node)],
lambda placed: [window_point(placed, saved, operation.get("x"), operation.get("y"), node)],
lambda point: click_at(*point, operation.get("mouse_button", "left"), click_count, operation.get("modifiers")),
)
action = {"path": "synthetic", "actionName": None, "fallbackReason": "actionUnsupported"}
@@ -2196,20 +2573,23 @@ def run_operation(operation):
raise RuntimeError(f'{operation.get("action", "")} is not a valid secondary action')
action = {"path": "accessibility", "actionName": label, "fallbackReason": None}
elif tool == "scroll":
pointer_input(
"scrolling", "",
lambda bounds: [window_point(bounds, saved, operation.get("x"), operation.get("y"), node)],
lambda point: scroll_at(*point, operation.get("direction"), operation.get("pages")),
)
action = {"path": "synthetic", "actionName": "scroll", "fallbackReason": None}
action = scroll_without_pointer(window, node, Placement(app, window), operation.get("x"), operation.get("y"),
operation.get("direction"), operation.get("pages"))
if action is None:
pointer_input(
"scrolling", "This view exposes no scroll bar or scrollable document to scroll in the background.",
lambda placed: [window_point(placed, saved, operation.get("x"), operation.get("y"), node)],
lambda point: scroll_at(*point, operation.get("direction"), operation.get("pages")),
)
action = {"path": "synthetic", "actionName": "scroll", "fallbackReason": None}
elif tool == "drag":
from_node = find_element(app, operation.get("fromElement"))
to_node = find_element(app, operation.get("toElement"))
pointer_input(
"dragging", "",
lambda bounds: [
window_point(bounds, operation.get("fromElement"), operation.get("from_x"), operation.get("from_y"), from_node),
window_point(bounds, operation.get("toElement"), operation.get("to_x"), operation.get("to_y"), to_node),
lambda placed: [
window_point(placed, operation.get("fromElement"), operation.get("from_x"), operation.get("from_y"), from_node),
window_point(placed, operation.get("toElement"), operation.get("to_x"), operation.get("to_y"), to_node),
],
drag_between,
)
@@ -22,6 +22,9 @@ class FakeStateType:
SENSITIVE = 8
HAS_POPUP = 9
MULTI_LINE = 10
VERTICAL = 11
HORIZONTAL = 12
MULTISELECTABLE = 13
class FakeStateSet:
@@ -101,6 +104,15 @@ class FakeAccessible:
self.name = name
self.value = value
self.children = list(children)
self.parent = None
for child in self.children:
child.parent = self
def get_parent(self):
return self.parent
def get_index_in_parent(self):
return self.parent.children.index(self)
def get_child_count(self):
return len(self.children)
@@ -161,7 +173,7 @@ class SnapshotFixture(ExitStack):
(runtime.time, "monotonic", lambda: self.clock),
(runtime.Atspi, "set_timeout", self.set_timeout),
(runtime.Atspi, "Component", types.SimpleNamespace(get_extents=self.extents)),
(runtime.Atspi, "CoordType", types.SimpleNamespace(SCREEN=0)),
(runtime.Atspi, "CoordType", types.SimpleNamespace(SCREEN=0, WINDOW=1)),
(runtime, "capture_png", self.capture),
(runtime, "restore_window", lambda *_: self.restore()),
(runtime, "x11_pointer_state", lambda: self.pointer),
@@ -673,6 +685,160 @@ class RuntimeRenderTest(unittest.TestCase):
self.assertEqual(effects, [])
keys.assert_not_called()
def test_gtk4_rectangles_come_from_window_extents_and_the_x11_client_origin(self):
button = FakeAccessible("push button", "Open")
window = FakeAccessible("frame", "Files", children=[button])
app = FakeAccessible("application", "Files", children=[window])
app.get_toolkit_name, app.get_toolkit_version = (lambda: "GTK"), (lambda: "4.14.5")
# Why these numbers: GTK 4 answers SCREEN with [0, 0, w, h] (agent-desktop FINDINGS L23-37).
extents = {(window, 0): (0, 0, 800, 600), (window, 1): (0, 0, 800, 600),
(button, 0): (0, 0, 64, 44), (button, 1): (16, 299, 64, 44)}
for node in (window, button):
node.get_component_iface = lambda node=node: node
def get_extents(node, coord_type):
x, y, width, height = extents[(node, coord_type)]
return types.SimpleNamespace(x=x, y=y, width=width, height=height)
for origin, wayland in (((1000, 500), False), (None, False), (None, True)):
with self.subTest(origin=origin, wayland=wayland), \
patch.object(runtime.Atspi, "Component", types.SimpleNamespace(get_extents=get_extents), create=True), \
patch.object(runtime.Atspi, "CoordType", types.SimpleNamespace(SCREEN=0, WINDOW=1), create=True), \
patch.object(runtime, "gtk4_client_origin", lambda *_: origin), \
patch.object(runtime, "wayland_session", lambda: wayland):
placed = runtime.Placement(app, window)
self.assertEqual(placed.frame(button).to_json(), {"x": 16, "y": 299, "width": 64, "height": 44})
if origin:
self.assertEqual(runtime.window_point(placed, {"index": 1}, node=button), (1048, 821))
else:
with self.assertRaisesRegex(RuntimeError, "GTK 4 window"):
runtime.window_point(placed, {"index": 1}, node=button)
def test_background_scroll_pages_a_scroll_bar_and_hands_the_end_to_the_outer_view(self):
def view(name, value, high, children=()):
bar = FakeAccessible("scroll bar", name + " bar")
bar.position = value
bar.high = high
bar.get_state_set = lambda: FakeStateSet(FakeStateType.VERTICAL)
bar.get_value_iface = lambda: bar
pane = FakeAccessible("scroll pane", name, children=[bar, *children])
return pane, bar
row = FakeAccessible("list item", "Row")
inner, inner_bar = view("inner", 300, 300, [row])
outer, outer_bar = view("outer", 0, 2000, [inner])
writes = []
def set_value(bar, value):
writes.append((bar.name, value))
bar.position = value
return True
value_api = types.SimpleNamespace(get_minimum_value=lambda bar: 0, get_maximum_value=lambda bar: bar.high,
get_current_value=lambda bar: bar.position, set_current_value=set_value)
with SnapshotFixture(FakeAccessible("frame", "Editor", children=[outer])) as fixture, \
patch.object(runtime.Atspi, "Value", value_api, create=True):
fixture.bounds[outer] = runtime.Rect(100, 200, 300, 250)
element = next(item for item in runtime.make_snapshot("Fixture", False)["elements"] if item["name"] == "Row")
result = runtime.run_operation({"tool": "scroll", "app": "Fixture", "element": element, "direction": "down",
"pages": 2, "noScreenshot": True})
self.assertEqual(writes, [("outer bar", 500)])
self.assertEqual(result["action"]["verification"], {"state": "verified", "property": "value"})
self.assertEqual(result["action"]["delivery"], "background")
runtime.run_operation({"tool": "scroll", "app": "Fixture", "element": element, "direction": "up",
"pages": 0.5, "noScreenshot": True})
self.assertEqual(writes[-1], ("inner bar", 300 - 0.5 * fixture.bounds[inner].height))
def test_background_scroll_pages_a_document_with_scroll_to(self):
items = [FakeAccessible("paragraph", f"Item {index}") for index in range(6)]
document = FakeAccessible("document web", "Page", children=items)
scrolled = []
offset = [0]
with SnapshotFixture(FakeAccessible("frame", "Browser", children=[document])) as fixture:
fixture.bounds[document] = runtime.Rect(0, 100, 400, 300)
def place():
for index, item in enumerate(items):
fixture.bounds[item] = runtime.Rect(0, 100 + index * 120 - offset[0], 400, 120)
place()
def scroll_to(item, edge):
scrolled.append((item.name, edge))
offset[0] += fixture.bounds[item].y - 100
place()
return True
with patch.object(runtime.Atspi.Component, "scroll_to", scroll_to, create=True), \
patch.object(runtime.Atspi, "ScrollType", types.SimpleNamespace(TOP_EDGE="top", BOTTOM_EDGE="bottom"), create=True):
result = runtime.run_operation({"tool": "scroll", "app": "Fixture", "x": 50, "y": 50,
"direction": "down", "pages": 3, "noScreenshot": True})
# Why Item 2: it spans 340-460 and the viewport ends at 400, so its hidden half leads the next page.
# The third page finds nothing past the document's end, and the two pages already moved still count.
self.assertEqual(scrolled, [("Item 2", "top"), ("Item 4", "top")])
self.assertEqual(result["action"]["actionName"], "scrollTo")
def test_background_scroll_moves_the_inner_scroller_before_the_page(self):
inner_items = [FakeAccessible("list item", f"Message {index}") for index in range(4)]
log = FakeAccessible("panel", "Chat log", children=inner_items)
page_items = [FakeAccessible("paragraph", f"Section {index}") for index in range(4)]
document = FakeAccessible("document web", "Page", children=[log, *page_items])
scrolled = []
with SnapshotFixture(FakeAccessible("frame", "Browser", children=[document])) as fixture:
fixture.bounds[fixture.root] = runtime.Rect(0, 0, 400, 400)
fixture.bounds[document] = runtime.Rect(0, 0, 400, 400)
fixture.bounds[log] = runtime.Rect(0, 0, 400, 100)
for index, item in enumerate(inner_items):
fixture.bounds[item] = runtime.Rect(0, index * 60, 400, 60)
for index, item in enumerate(page_items):
fixture.bounds[item] = runtime.Rect(0, 100 + index * 150, 400, 150)
element = next(item for item in runtime.make_snapshot("Fixture", False)["elements"] if item["name"] == "Message 0")
def scroll_to(item, _):
scrolled.append(item.name)
fixture.bounds[item] = runtime.Rect(0, 0, 400, 60)
return True
with patch.object(runtime.Atspi.Component, "scroll_to", scroll_to, create=True), \
patch.object(runtime.Atspi, "ScrollType", types.SimpleNamespace(TOP_EDGE="top"), create=True):
runtime.run_operation({"tool": "scroll", "app": "Fixture", "element": element, "direction": "down",
"pages": 1, "noScreenshot": True})
# Why Message 1: it spans 60-120 and the chat log's own box ends at 100, so the log scrolls, not the page.
self.assertEqual(scrolled, ["Message 1"])
def test_click_without_an_action_selects_the_row_only_when_its_index_is_proven(self):
cases = (
("Qt", "6.6", None, "selected"),
("GTK", "4.14", (True, False), "selected"),
("GTK", "4.14", (True, True), "refused"),
("GTK", "4.14", (True, "timeout"), "refused"),
("Qt", "6.6", "replaced", "stale"),
("GTK", "3.24", "multiple", "selected"),
)
for toolkit, version, counted, expected in cases:
rows = [FakeAccessible("list item", name) for name in ("Mouse", "Keyboard")]
listing = FakeAccessible("list", "Devices", children=rows)
selected = ["Mouse"] if counted == "multiple" else []
listing.get_state_set = lambda: FakeStateSet(*([FakeStateType.MULTISELECTABLE] if counted == "multiple" else []))
listing.get_selection_iface = lambda: listing
if counted == "replaced":
rows[1].get_index_in_parent = lambda: 0
for row in rows:
row.get_state_set = lambda row=row: FakeStateSet(*([FakeStateType.SELECTED] if row.name in selected else []))
def is_child_selected(_, index):
answer = (counted if isinstance(counted, tuple) else (True, False))[index - len(rows) + 1]
if answer is not True:
raise RuntimeError("Timeout was reached" if answer == "timeout" else "GDBus.Error:org.freedesktop.DBus.Error.InvalidArgs")
return False
selection = types.SimpleNamespace(
select_child=lambda _, index: selected.append(rows[index].name) or True,
clear_selection=lambda _: selected.clear() or True,
is_child_selected=is_child_selected)
with self.subTest(toolkit=toolkit, counted=counted), \
SnapshotFixture(FakeAccessible("frame", "Settings", children=[listing])) as fixture, \
patch.object(runtime.Atspi, "Selection", selection, create=True), \
patch.object(runtime, "gtk4_client_origin", lambda *_: (0, 0)):
fixture.app.get_toolkit_name, fixture.app.get_toolkit_version = (lambda: toolkit), (lambda: version)
element = next(item for item in runtime.make_snapshot("Fixture", False)["elements"] if item["name"] == "Keyboard")
request = {"tool": "click", "app": "Fixture", "element": element, "noScreenshot": True}
if expected == "selected":
result = runtime.run_operation(request)
self.assertEqual(selected, ["Keyboard"])
self.assertEqual(result["action"]["verification"], {"state": "verified", "property": "selection"})
else:
with self.assertRaisesRegex(RuntimeError, "background_unavailable" if expected == "refused" else "stale element"):
runtime.run_operation(request)
self.assertEqual(selected, [])
def test_text_entry_inserts_into_the_focused_field_of_a_background_window(self):
cases = (
("type_text", "🚀 café", (6, 11), "Hello 🚀 café", 12),
@@ -156,6 +156,7 @@ struct Snapshot {
let windowBounds: CGRect
let windowId: CGWindowID
let windowLayer: Int
let windowVisibility: WindowVisibility
let treeText: String
let focusedElementId: Int?
let screenshot: ScreenshotPayload?
@@ -173,6 +174,7 @@ struct Snapshot {
func showsSameState(as other: Snapshot) -> Bool {
treeText == other.treeText && windowBounds == other.windowBounds && appWindowIds == other.appWindowIds
&& windowVisibility == other.windowVisibility
}
func withoutScreenshotPayload() -> Snapshot {
@@ -183,6 +185,7 @@ struct Snapshot {
windowBounds: windowBounds,
windowId: windowId,
windowLayer: windowLayer,
windowVisibility: windowVisibility,
treeText: treeText,
focusedElementId: focusedElementId,
screenshot: nil,
@@ -212,6 +215,31 @@ struct CapturedImage {
let engine: String
}
/// Where an observed window is; only an on-screen one can take background pointer or key input.
enum WindowVisibility {
case onScreen
case minimized
case appHidden
case otherSpace
var reason: String {
switch self {
case .onScreen: "is on screen"
case .minimized: "is minimized"
case .appHidden: "belongs to a hidden app"
case .otherSpace: "is on another Space"
}
}
}
/// A window an agent can observe: on screen, or off screen with an accessibility element it can still read.
private struct ObservableWindow {
let candidate: WindowCandidate
/// Off-screen windows carry the element found for them; on-screen ones are matched later.
let element: AXUIElement?
let visibility: WindowVisibility
}
enum ScreenshotStatus {
case captured
case skipped
@@ -224,9 +252,25 @@ private struct CachedSnapshotEntry {
let createdAt: Date
}
private struct OtherSpaceLookups {
var elements: [CGWindowID: AXUIElement] = [:]
var misses: [CGWindowID: Date] = [:]
}
/// The app's observable windows, plus other-Space windows that exist but accessibility can't read yet.
private struct WindowInventory {
let windows: [ObservableWindow]
let unreadableOtherSpace: Set<CGWindowID>
/// Every window the app has, closed-but-kept ones included.
let windowIds: Set<CGWindowID>
}
final class Provider {
private var snapshots: [String: Snapshot] = [:]
private var snapshotEntries: [CachedSnapshotEntry] = []
/// Per app, window elements already read (valid while the window lives) for when a window moves to another Space,
/// and when a remote-token lookup last missed, since a window becomes findable only once some client reads it.
private var otherSpaceLookups: [pid_t: OtherSpaceLookups] = [:]
init() {
// Why: the system-wide element sets the wait for every AX call in this process, so a hung app costs one bounded timeout whatever the macOS default.
@@ -239,53 +283,65 @@ final class Provider {
return providerHandshake()
case "listApps":
return ["apps": listApps().map(renderListedApp)]
case "launchApp":
return try launchApp(params: params)
case "listWindows":
return try listWindows(params: params)
case "getAppState":
let params = isForegroundRequest(params) ? try recoverRequestedWindow(params: params) : params
return try renderSnapshot(observe(params: params))
case "click":
return try actionResult(params: params) { try click(params: params) }
return try actionResult(params: params) { try click(params: $0) }
case "performSecondaryAction":
return try actionResult(params: params) { try performSecondaryAction(params: params) }
return try actionResult(params: params) { try performSecondaryAction(params: $0) }
case "setValue":
return try actionResult(params: params) { try setValue(params: params) }
return try actionResult(params: params) { try setValue(params: $0) }
case "typeText":
return try actionResult(params: params) { try typeText(params: params) }
return try actionResult(params: params) { try typeText(params: $0) }
case "pressKey":
return try actionResult(params: params) { try pressKey(params: params) }
return try actionResult(params: params) { try pressKey(params: $0) }
case "hotkey":
return try actionResult(params: params) { try hotkey(params: params) }
return try actionResult(params: params) { try hotkey(params: $0) }
case "pasteText":
return try actionResult(params: params) { try pasteText(params: params) }
return try actionResult(params: params) { try pasteText(params: $0) }
case "scroll":
return try actionResult(params: params) { try scroll(params: params) }
return try actionResult(params: params) { try scroll(params: $0) }
case "drag":
return try actionResult(params: params) { try drag(params: params) }
return try actionResult(params: params) { try drag(params: $0) }
default:
throw ProviderError.coded("invalid_argument", "unknown method '\(method)'")
}
}
private func actionResult(params: [String: JSONValue], action runAction: () throws -> [String: Any]) throws -> [String: Any] {
var action = try runAction()
// Why: only --restore-window activates the app or posts to the HID tap; every other route leaves the user's desktop alone.
action["delivery"] = isForegroundRequest(params) ? "foreground" : "background"
private func actionResult(params requestParams: [String: JSONValue], action runAction: ([String: JSONValue]) throws -> [String: Any]) throws -> [String: Any] {
let app = try resolveApp(try requiredString(requestParams, "app"))
// Why: the main run loop keeps NSRunningApplication current while requests run on connection threads.
let frontBefore = ForegroundDelivery.Observation(
frontmost: app.app.isActive,
focusedWindowID: app.app.isActive ? focusedWindowId(of: app) : nil
)
// Why: raise once, before the action's reads, so the pre and post observations never raise again.
let params = isForegroundRequest(requestParams) ? try recoverRequestedWindow(params: requestParams) : requestParams
var action = try runAction(params)
// Why: the action's currentSnapshot just cached this target's tree from before the action;
// a subtree read can't be compared with the whole-window read that follows.
let before = action["path"] as? String == "accessibility" && action["verification"] == nil
? (try? cachedSnapshot(params: params)).flatMap { $0.scope == nil ? $0 : nil }
: nil
let snapshot: Snapshot
do {
let after = try observe(params: params)
guard let before, after.showsSameState(as: before) else {
return renderActionResult(action: action, snapshot: after)
if let before, after.showsSameState(as: before) {
if let changed = try observeLateChange(before: before, params: params) {
snapshot = changed
} else {
// Why: an AX call can succeed and do nothing, as AXPress does on many web controls (#31758).
action["verification"] = unverifiedAction(reason: "suspected_noop")
snapshot = after
}
} else {
snapshot = after
}
if let changed = try observeLateChange(before: before, params: params) {
return renderActionResult(action: action, snapshot: changed)
}
// Why: an AX call can succeed and do nothing, as AXPress does on many web controls (#31758).
action["verification"] = unverifiedAction(reason: "suspected_noop")
return renderActionResult(action: action, snapshot: after)
} catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) {
var fallbackParams = params
fallbackParams.removeValue(forKey: "windowId")
@@ -293,35 +349,130 @@ final class Provider {
if action["verification"] == nil {
action["verification"] = ["state": "unverified", "reason": "window_changed"]
}
return try renderActionResult(action: action, snapshot: observe(params: fallbackParams))
snapshot = try observe(params: fallbackParams)
}
// Why: measured after the settle reads, since apps that activate themselves on AXPress (Electron, sheet openers) do it a turn later.
let frontAfter = frontmostObservation(of: app)
let tookForeground = ForegroundDelivery.tookForeground(
before: frontBefore,
after: frontAfter,
targetWindowID: snapshot.windowId,
restoreRequested: isForegroundRequest(params)
)
action["delivery"] = tookForeground ? "foreground" : "background"
return renderActionResult(action: action, snapshot: snapshot)
}
/// Re-reads the target for about a second, since apps often apply an action on a later run-loop turn.
/// Returns the observation to report once something changed, or nil if nothing did.
private func observeLateChange(before: Snapshot, params: [String: JSONValue]) throws -> Snapshot? {
let settleParams = params.merging(["restoreWindow": .bool(false)]) { _, replacement in replacement }
let quietParams = settleParams.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }
let quietParams = params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }
let deadline = Date().addingTimeInterval(1)
while Date() < deadline {
Thread.sleep(forTimeInterval: 0.2)
let polled = try observe(params: quietParams)
if !polled.showsSameState(as: before) {
if params["noScreenshot"]?.bool == true { return polled }
return try observe(params: settleParams)
return try observe(params: params)
}
}
return nil
}
/// Brings the requested window forward for --restore-window, once per request, and returns the params pinned
/// to that window: raising reorders the window list, so a --window-index could name another window afterwards.
private func recoverRequestedWindow(params: [String: JSONValue]) throws -> [String: JSONValue] {
let app = try resolveApp(try requiredString(params, "app"))
// Why: the window-index cache keys are not per app, so another app's snapshot can sit under the same index.
let cached = try cachedSnapshot(params: params).flatMap { $0.app.pid == app.pid ? $0 : nil }
// Why: a stale index fails after the raise, which would take over the user's screen for nothing.
try requireCachedElements(cached: cached, requestedIndexes: try requestedElementIndexes(params))
let windows = windowInventory(app: app, appElement: AXUIElementCreateApplication(app.pid)).windows
let byIndex = try requestedWindowIndex(params).flatMap { index in
windows.filter { $0.candidate.layer == 0 }.dropFirst(index).first?.candidate.windowId
}
// Why: the cached snapshot is the window the agent's element indexes were read from, even if the index now names another.
let windowId = try requestedWindowId(params) ?? cached?.windowId ?? byIndex
let target = windowId.flatMap { id in windows.first { $0.candidate.windowId == id } }
if let requested = try requestedWindowId(params), target == nil {
throw ProviderError.coded("window_not_found", "window \(Int(requested)) of \(app.name) is closed or cannot be read, so nothing was raised; run list-windows and pick a listed window")
}
recoverWindow(app, windowId: windowId, windowBounds: target?.candidate.bounds ?? cached?.windowBounds, fallback: target?.element)
guard let windowId else { return params }
// Why: the raise reorders windows and focus, so later reads must not pick the target again; the id wins over
// the index when both are set, and the index still keys the snapshot cache.
var pinned = params
pinned["windowId"] = .number(Double(windowId))
return pinned
}
/// On-screen windows, plus minimized, hidden and other-Space windows that accessibility can still read.
private func windowInventory(app: AppDescriptor, appElement: AXUIElement) -> WindowInventory {
let all = WindowCapture.candidates(pid: app.pid, includeOffScreen: true)
let offScreen = all.filter { !$0.isOnScreen && $0.layer == 0 }
var elements: [CGWindowID: AXUIElement] = [:]
var unreadable = Set<CGWindowID>()
if !offScreen.isEmpty {
var listed: CFTypeRef?
let status = AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &listed)
for window in (listed as? [AXUIElement]) ?? [] {
if let id = windowNumber(window) { elements[id] = window }
}
let unlisted = offScreen.map(\.windowId).filter { elements[$0] == nil }
// Why: closed windows the app keeps belong to no Space; without SkyLight, try them all.
let inSpaces = WindowSpaces.windowsInAnySpace(unlisted) ?? Set(unlisted)
// Why: kAXWindowsAttribute leaves out windows on other Spaces; skip the lookup when the app stopped answering.
if status != .cannotComplete {
let live = Set(all.map(\.windowId))
elements.merge(otherSpaceWindows(pid: app.pid, windowIds: inSpaces, live: live)) { listed, _ in listed }
}
unreadable = inSpaces.subtracting(elements.keys)
}
var appHidden: Bool?
let windows = all.compactMap { candidate -> ObservableWindow? in
if candidate.isOnScreen {
return ObservableWindow(candidate: candidate, element: nil, visibility: .onScreen)
}
guard candidate.layer == 0, let element = elements[candidate.windowId] else { return nil }
let visibility: WindowVisibility
if boolAttribute(element, kAXMinimizedAttribute as String) == true {
visibility = .minimized
} else {
if appHidden == nil { appHidden = boolAttribute(appElement, kAXHiddenAttribute as String) == true }
visibility = appHidden == true ? .appHidden : .otherSpace
}
return ObservableWindow(candidate: candidate, element: element, visibility: visibility)
}
return WindowInventory(windows: windows, unreadableOtherSpace: unreadable, windowIds: Set(all.map(\.windowId)))
}
private func otherSpaceWindows(pid: pid_t, windowIds: Set<CGWindowID>, live: Set<CGWindowID>) -> [CGWindowID: AXUIElement] {
var lookups = otherSpaceLookups[pid] ?? OtherSpaceLookups()
lookups.elements = lookups.elements.filter { live.contains($0.key) && windowNumber($0.value) == $0.key }
lookups.misses = lookups.misses.filter { live.contains($0.key) }
let wanted = windowIds.filter { id in
lookups.elements[id] == nil && lookups.misses[id].map { Date().timeIntervalSince($0) >= 10 } ?? true
}
if !wanted.isEmpty {
let found = RemoteAccessibilityWindows.windows(pid: pid, wanted: wanted)
for id in wanted {
if let element = found[id] {
lookups.elements[id] = element
} else {
lookups.misses[id] = Date()
}
}
}
otherSpaceLookups = otherSpaceLookups.filter { pidIsLive($0.key) }
otherSpaceLookups[pid] = lookups
return lookups.elements.filter { windowIds.contains($0.key) }
}
private func observe(params: [String: JSONValue]) throws -> Snapshot {
let query = try requiredString(params, "app")
let windowId = try requestedWindowId(params)
let windowIndex = try requestedWindowIndex(params)
let app = try resolveApp(query)
if params["restoreWindow"]?.bool == true {
recoverWindow(app)
}
let namespace = snapshotNamespace(params)
pruneSnapshotCache()
let subtree = try requestedSubtree(params: params, app: app)
@@ -330,7 +481,6 @@ final class Provider {
includeScreenshot: params["noScreenshot"]?.bool != true,
windowId: subtree.map(\.base.windowId) ?? windowId,
windowIndex: subtree == nil ? windowIndex : nil,
restoreWindow: params["restoreWindow"]?.bool == true,
subtree: subtree,
previousSnapshot: { [snapshots] resolvedWindowId in
// Why: indexes carry over from this caller's own last view of exactly this window.
@@ -511,11 +661,19 @@ final class Provider {
}
}
private func validateRequestedElements(cached: Snapshot?, current: Snapshot, requestedIndexes: [Int]) throws {
private func requireCachedElements(cached: Snapshot?, requestedIndexes: [Int]) throws {
guard !requestedIndexes.isEmpty else { return }
guard let cached else {
throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window")
}
for index in requestedIndexes where cached.elements[index] == nil {
throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index")
}
}
private func validateRequestedElements(cached: Snapshot?, current: Snapshot, requestedIndexes: [Int]) throws {
try requireCachedElements(cached: cached, requestedIndexes: requestedIndexes)
guard let cached else { return }
for index in requestedIndexes {
// Why: a subtree read also carries the window's earlier records, which prove nothing about the element now.
guard let expected = cached.elements[index],
@@ -578,6 +736,7 @@ final class Provider {
"list": true,
"bundleIds": true,
"pids": true,
"launch": true,
],
"windows": [
"list": true,
@@ -646,13 +805,72 @@ final class Provider {
}
}
private func launchApp(params: [String: JSONValue]) throws -> [String: Any] {
let query = try requiredString(params, "app").trimmingCharacters(in: .whitespacesAndNewlines)
let activate = params["activate"]?.bool == true
// Why: Launch Services sends a running app a reopen event, and many apps answer it by opening a window.
if let running = listApps().first(where: { matches($0, query: query) }) {
try rejectBlockedApp(running)
return renderLaunch(running, launched: false)
}
guard let url = installedApplicationURL(query) else {
throw ProviderError.coded("app_not_found", "app '\(query)' is not running and no installed app has that name, bundle ID or path")
}
if let bundleId = Bundle(url: url)?.bundleIdentifier {
if blockedBundleIds.contains(bundleId) {
throw ProviderError.coded("app_blocked", "app '\(bundleId)' is blocked for safety")
}
// Why: a path or display-name query misses the running check above, and menu bar apps aren't listed.
if let running = NSRunningApplication.runningApplications(withBundleIdentifier: bundleId)
.first(where: { !$0.isTerminated }),
let app = appByPid(running.processIdentifier) {
return renderLaunch(app, launched: false)
}
}
let pid: pid_t
do {
pid = try BlockingAsync.run(timeout: 30, timeoutMessage: "launching '\(query)' timed out") {
let configuration = NSWorkspace.OpenConfiguration()
configuration.activates = activate
return try await NSWorkspace.shared.openApplication(at: url, configuration: configuration).processIdentifier
}
} catch let error as ProviderError {
throw error
} catch {
throw ProviderError.coded("app_launch_failed", "could not launch '\(query)': \(error.localizedDescription)")
}
guard let app = appByPid(pid) else {
throw ProviderError.coded("app_launch_failed", "'\(query)' exited right after it launched")
}
// Why: Launch Services reports the app once it checks in; activation, asked for or not, lands after it finishes launching.
let deadline = Date().addingTimeInterval(5)
while !app.app.isFinishedLaunching, !app.app.isTerminated, Date() < deadline {
Thread.sleep(forTimeInterval: 0.05)
}
if app.app.isTerminated {
throw ProviderError.coded("app_launch_failed", "'\(query)' quit while it was starting")
}
return renderLaunch(app, launched: true)
}
private func renderLaunch(_ app: AppDescriptor, launched: Bool) -> [String: Any] {
// Why: measured, not taken from the request; an app may activate itself however it was started.
let frontmost = NSWorkspace.shared.frontmostApplication?.processIdentifier == app.pid
return [
"app": ["name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid)],
"launched": launched,
"delivery": frontmost ? "foreground" : "background",
]
}
private func listWindows(params: [String: JSONValue]) throws -> [String: Any] {
let app = try resolveApp(try requiredString(params, "app"))
let windows = WindowCapture.candidates(pid: app.pid)
.filter { $0.layer == 0 }
let windows = windowInventory(app: app, appElement: AXUIElementCreateApplication(app.pid)).windows
.filter { $0.candidate.layer == 0 }
.enumerated()
.map { index, candidate -> [String: Any] in
[
.map { index, window -> [String: Any] in
let candidate = window.candidate
return [
"index": index,
"app": [
"name": app.name,
@@ -665,8 +883,8 @@ final class Provider {
"y": Int(candidate.bounds.origin.y.rounded()),
"width": Int(candidate.bounds.width.rounded()),
"height": Int(candidate.bounds.height.rounded()),
"isMinimized": false,
"isOffscreen": !candidate.isOnScreen,
"isMinimized": window.visibility == .minimized,
"isOffscreen": window.visibility != .onScreen,
"screenIndex": jsonNullable(screenIndex(for: candidate.bounds)),
"isMain": NSNull(),
"platform": [
@@ -696,7 +914,6 @@ final class Provider {
includeScreenshot: Bool,
windowId: CGWindowID?,
windowIndex: Int?,
restoreWindow: Bool,
subtree: (root: ElementRecord, base: Snapshot)?,
previousSnapshot: (CGWindowID) -> Snapshot?
) throws -> Snapshot {
@@ -716,27 +933,36 @@ final class Provider {
throw AccessibilityMessaging.appNotResponding(app.name)
}
enableManualAccessibilityIfNeeded(appElement, app: app)
let windowCandidates = WindowCapture.candidates(pid: app.pid)
let focused = try focusedWindow(
appElement: appElement,
app: app,
visibleWindowCount: windowCandidates.count,
allowRecovery: restoreWindow
)
let inventory = windowInventory(app: app, appElement: appElement)
let observable = inventory.windows
let focused = try focusedWindow(appElement: appElement, app: app, inventory: inventory)
let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name
let canCaptureScreenshot = includeScreenshot && screenCaptureTrustedSettled()
guard let capture = WindowCapture.resolve(
candidates: windowCandidates,
candidates: observable.map(\.candidate),
titleHint: focusedTitle,
windowId: windowId,
windowIndex: windowIndex,
captureImage: canCaptureScreenshot
) else {
throw ProviderError.coded("window_not_found", "app '\(app.name)' has no on-screen window")
if let windowId, inventory.unreadableOtherSpace.contains(windowId) {
throw unreadableOtherSpaceError(app)
}
let reason = windowId != nil || windowIndex != nil ? "no window matching the requested selector; run list-windows and pick a listed one" : "no window to observe"
throw ProviderError.coded("window_not_found", "app '\(app.name)' has \(reason)")
}
guard let window = matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else {
let target = observable.first { $0.candidate.windowId == capture.windowId }
let visibility = target?.visibility ?? .onScreen
guard let window = target?.element ?? matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else {
throw ProviderError.coded("window_not_found", "could not match accessibility window to requested window; run get-app-state again or retry without a window selector")
}
// Why: a window first read after a large tree walk gets a remote-token id past the lookup's range, so keep the element
// for when the window moves to another Space.
otherSpaceLookups = otherSpaceLookups.filter { pidIsLive($0.key) }
var lookups = otherSpaceLookups[app.pid] ?? OtherSpaceLookups()
lookups.elements = lookups.elements.filter { inventory.windowIds.contains($0.key) }
lookups.elements[capture.windowId] = window
otherSpaceLookups[app.pid] = lookups
let title = stringAttribute(window, kAXTitleAttribute as String) ?? capture.title ?? app.name
let renderer = TreeRenderer(
windowBounds: capture.bounds,
@@ -771,6 +997,8 @@ final class Provider {
.captured
} else if includeScreenshot && !canCaptureScreenshot {
.failed("Screen Recording permission is required for Orca Computer Use; grant permission or pass --no-screenshot to inspect accessibility state only.")
} else if includeScreenshot && visibility != .onScreen {
.failed("the window \(visibility.reason), so it has no pixels to capture; the accessibility tree is still current. Pass --no-screenshot to skip the capture.")
} else if includeScreenshot {
.failed("window screenshot capture returned no image; retry with --no-screenshot if accessibility state is sufficient.")
} else {
@@ -783,6 +1011,7 @@ final class Provider {
windowBounds: capture.bounds,
windowId: capture.windowId,
windowLayer: capture.layer,
windowVisibility: visibility,
treeText: renderTreeText(app: app, title: title, rootIndex: subtree?.root.index, lines: renderer.lines, focused: renderer.focusedSummary),
focusedElementId: renderer.focusedElementId,
screenshot: screenshot,
@@ -795,7 +1024,7 @@ final class Provider {
truncated: renderer.truncated,
maxDepthReached: renderer.maxDepthReached,
scope: subtree.map { SnapshotScope(rootIndex: $0.root.index, walkedIndexes: Set(indexes.order)) },
appWindowIds: windowCandidates.map(\.windowId).sorted()
appWindowIds: observable.filter { $0.visibility == .onScreen }.map(\.candidate.windowId).sorted()
)
}
@@ -824,8 +1053,8 @@ final class Provider {
"y": Int(snapshot.windowBounds.origin.y.rounded()),
"width": Int(snapshot.windowBounds.width.rounded()),
"height": Int(snapshot.windowBounds.height.rounded()),
"isMinimized": false,
"isOffscreen": false,
"isMinimized": snapshot.windowVisibility == .minimized,
"isOffscreen": snapshot.windowVisibility != .onScreen,
"screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)),
"platform": [
"layer": snapshot.windowLayer,
@@ -872,9 +1101,6 @@ final class Provider {
}
let modifiers = try KeyMap.parseModifiers(params["modifiers"]?.string)
let foreground = isForegroundRequest(params)
if foreground {
recoverWindow(snapshot.app, windowId: snapshot.windowId, windowBounds: snapshot.windowBounds)
}
if let elementIndex = try optionalInteger(params, "elementIndex") {
let record = try element(snapshot, elementIndex)
try requireEnabled(record, snapshot: snapshot)
@@ -897,6 +1123,9 @@ final class Provider {
actionName: actionName
)
}
if !foreground {
try requireBackgroundInputReachable(snapshot, input: "pointer click")
}
let point = try pointerTarget(record, snapshot: snapshot)
try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element)
guard unselectedRow else {
@@ -912,6 +1141,9 @@ final class Provider {
actionName: nil
)
}
if !foreground {
try requireBackgroundInputReachable(snapshot, input: "pointer click")
}
let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground)
return actionMetadata(
@@ -1125,6 +1357,10 @@ final class Provider {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
let pasteKeys = try Input.keyEvents("cmd+v")
if !isForegroundRequest(params) {
// Why: refuse before the clipboard is replaced, so clipboard history never records text that was not sent.
try requireBackgroundInputReachable(snapshot, input: "paste")
}
let insertion = TextInput.Insertion(into: focused, text: text)
try Input.withPasteboardText(text) {
try deliverKeys(pasteKeys, snapshot: snapshot, params: params)
@@ -1186,6 +1422,7 @@ final class Provider {
try requireTargetWindowFocused(snapshot, restoreWindowRequested: true)
try Input.drag(from: start, to: end, targetWindow: snapshot)
} else {
try requireBackgroundInputReachable(snapshot, input: "drag")
// Why: the mouse-down lands on whatever is under the start point, which can open a menu over the user.
guard let startElement = elementAtPoint(start, pid: snapshot.app.pid) ?? fromRecord?.element else {
throw unidentifiedBackgroundTarget
@@ -1293,6 +1530,28 @@ private func parsePid(_ query: String) -> pid_t? {
return pid
}
/// An app that isn't running, by absolute path, bundle ID or bundle name in the usual install folders.
private func installedApplicationURL(_ query: String) -> URL? {
let path = (query as NSString).expandingTildeInPath
if path.hasPrefix("/") {
return FileManager.default.fileExists(atPath: path) ? URL(fileURLWithPath: path) : nil
}
if let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: query) {
return url
}
let bundleName = query.lowercased().hasSuffix(".app") ? query : "\(query).app"
let folders = [
"/Applications",
"/Applications/Utilities",
"/System/Applications",
"/System/Applications/Utilities",
NSHomeDirectory() + "/Applications",
]
return folders
.map { URL(fileURLWithPath: $0).appendingPathComponent(bundleName) }
.first { FileManager.default.fileExists(atPath: $0.path) }
}
private func matches(_ app: AppDescriptor, query: String) -> Bool {
app.name.caseInsensitiveCompare(query) == .orderedSame ||
app.bundleId?.caseInsensitiveCompare(query) == .orderedSame
@@ -1357,18 +1616,13 @@ private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: A
}
}
private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement {
private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, inventory: WindowInventory) throws -> AXUIElement {
let observable = inventory.windows
let systemWide = AXUIElementCreateSystemWide()
if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) {
return window
}
if allowRecovery {
recoverWindow(app)
if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) {
return window
}
}
if visibleWindowCount > 0 {
if observable.contains(where: { $0.visibility == .onScreen }) {
var settledWindow: AXUIElement?
let outcome = PermissionTrustSettling.settle {
settledWindow = lookupUsableWindow(
@@ -1381,12 +1635,26 @@ private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleW
if let window = settledWindow, outcome.settled {
return window
}
throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but no accessibility window (AX reads stayed blocked for \(outcome.waitedMs)ms after retries). macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings.")
var windows: CFTypeRef?
if AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &windows) == .apiDisabled {
throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but macOS refuses accessibility reads (AX reads stayed blocked for \(outcome.waitedMs)ms after retries). macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings.")
}
}
if allowRecovery {
throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window even after --restore-window; ask the user to open or show one of its windows.")
let offScreen = observable.filter { $0.element != nil }
if let window = (offScreen.first { $0.visibility != .minimized } ?? offScreen.first)?.element {
return window
}
throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; it may have none open, or they may be minimized, hidden or on another Space. Ask the user to bring one on screen, or ask the user before retrying with --restore-window.")
if observable.contains(where: { $0.visibility == .onScreen }) {
throw ProviderError.coded("window_not_found", "app '\(app.name)' has visible windows but exposes no accessibility window; it may still be starting up. Run get-app-state again shortly.")
}
if !inventory.unreadableOtherSpace.isEmpty {
throw unreadableOtherSpaceError(app)
}
throw ProviderError.coded("window_not_found", "app '\(app.name)' has no open window, counting minimized, hidden and other-Space windows; open one in the app first.")
}
private func unreadableOtherSpaceError(_ app: AppDescriptor) -> ProviderError {
.coded("window_not_found", "app '\(app.name)' has a window on another Space that accessibility cannot read yet: macOS leaves other-Space windows (full-screen ones included) out of accessibility until something has read them once. Ask the user to show that window (switch to its Space) and run get-app-state while it is showing, or ask the user before retrying with --restore-window.")
}
private func lookupUsableWindow(
@@ -1628,6 +1896,21 @@ private func isForegroundRequest(_ params: [String: JSONValue]) -> Bool {
params["restoreWindow"]?.bool == true
}
/// Refuses, before anything is sent, background pointer or key input the window cannot receive while off screen.
private func requireBackgroundInputReachable(_ snapshot: Snapshot, input: String) throws {
guard snapshot.windowVisibility != .onScreen else { return }
throw ProviderError.coded("background_unavailable", "window \(Int(snapshot.windowId)) of \(snapshot.app.name) \(snapshot.windowVisibility.reason), so background \(input) cannot reach it without bringing it forward; nothing was sent. Use its elements instead (click --element-index, set-value, type-text --element-index). Ask the user before retrying with --restore-window.")
}
private func frontmostObservation(of app: AppDescriptor) -> ForegroundDelivery.Observation {
let frontmost = boolAttribute(AXUIElementCreateApplication(app.pid), kAXFrontmostAttribute as String) == true
return ForegroundDelivery.Observation(frontmost: frontmost, focusedWindowID: frontmost ? focusedWindowId(of: app) : nil)
}
private func focusedWindowId(of app: AppDescriptor) -> CGWindowID? {
copyElement(AXUIElementCreateApplication(app.pid), kAXFocusedWindowAttribute as String).flatMap(windowNumber)
}
private func backgroundTarget(_ snapshot: Snapshot) -> BackgroundInputTransport.Target {
BackgroundInputTransport.Target(pid: snapshot.app.pid, windowID: snapshot.windowId, windowFrame: snapshot.windowBounds)
}
@@ -1737,6 +2020,7 @@ private func scrollPointer(at point: CGPoint, direction: String, pages: Double,
try Input.scroll(at: point, wheel1: wheel1, wheel2: wheel2, targetWindow: snapshot)
return
}
try requireBackgroundInputReachable(snapshot, input: "scroll")
try withTransportErrors {
try BackgroundInputTransport.scroll(backgroundTarget(snapshot), at: point, wheel1: wheel1, wheel2: wheel2)
}
@@ -1750,6 +2034,7 @@ private func deliverKeys(_ events: [CGEvent], snapshot: Snapshot, params: [Strin
}
return
}
try requireBackgroundInputReachable(snapshot, input: "key input")
try withTransportErrors {
try BackgroundInputTransport.sendKeys(events, to: backgroundTarget(snapshot))
}
@@ -1786,16 +2071,14 @@ private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, foc
} ?? focused
}
/// Brings only the target window forward: no other window of the app, and no LaunchServices reopen that can open a new one.
private func recoverWindow(
_ app: AppDescriptor,
windowId: CGWindowID? = nil,
windowBounds: CGRect? = nil
windowId: CGWindowID?,
windowBounds: CGRect?,
fallback: AXUIElement?
) {
_ = app.app.unhide()
_ = app.app.activate(options: [.activateAllWindows])
if let bundleId = app.bundleId {
openBundle(bundleId)
}
let appElement = AXUIElementCreateApplication(app.pid)
let focusedWindow = copyElement(appElement, kAXFocusedWindowAttribute as String)
var cachedWindows: [AXUIElement]?
@@ -1805,26 +2088,23 @@ private func recoverWindow(
cachedWindows = value
return value
}
let targetWindow: AXUIElement?
if let focusedWindow,
(windowId == nil && windowBounds == nil || windowMatchesCapture(
focusedWindow,
windowId: windowId,
windowBounds: windowBounds
)) {
targetWindow = focusedWindow
} else {
let exactWindow = windowId.flatMap { targetId in
windows().first { windowNumber($0) == targetId }
}
targetWindow = exactWindow ?? windowBounds.flatMap { targetBounds in
windows().first { window in
absoluteFrame(window).map { windowFramesMatch($0, targetBounds) } == true
}
// Why: an exact id wins over a frame match, since full-screen and minimized windows can share the focused window's frame.
let exactWindow = windowId.flatMap { targetId in
windows().first { windowNumber($0) == targetId }
} ?? fallback
let sameFrameWindow = windowBounds.flatMap { targetBounds in
windows().first { window in
absoluteFrame(window).map { windowFramesMatch($0, targetBounds) } == true
}
}
if let window = targetWindow ?? focusedWindow ?? windows().first {
let window = exactWindow ?? sameFrameWindow ?? focusedWindow ?? windows().first
if let window {
// Why: un-minimize first, so activating the app has the target to bring forward.
_ = AXUIElementSetAttributeValue(window, kAXMinimizedAttribute as CFString, kCFBooleanFalse)
}
_ = app.app.activate(options: [])
_ = AXUIElementSetAttributeValue(appElement, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
if let window {
_ = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
_ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
_ = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
@@ -1832,16 +2112,6 @@ private func recoverWindow(
Thread.sleep(forTimeInterval: 0.4)
}
private func windowMatchesCapture(
_ window: AXUIElement,
windowId: CGWindowID?,
windowBounds: CGRect?
) -> Bool {
if let windowId, windowNumber(window) == windowId { return true }
guard let windowBounds, let frame = absoluteFrame(window) else { return false }
return windowFramesMatch(frame, windowBounds)
}
private func windowFramesMatch(_ lhs: CGRect, _ rhs: CGRect) -> Bool {
let tolerance: CGFloat = 2
return abs(lhs.minX - rhs.minX) <= tolerance &&
@@ -1850,16 +2120,6 @@ private func windowFramesMatch(_ lhs: CGRect, _ rhs: CGRect) -> Bool {
abs(lhs.height - rhs.height) <= tolerance
}
private func openBundle(_ bundleId: String) {
let process = Process()
process.executableURL = URL(fileURLWithPath: "/usr/bin/open")
process.arguments = ["-b", bundleId]
process.standardOutput = Pipe()
process.standardError = Pipe()
try? process.run()
process.waitUntilExit()
}
private func hasRequestedWindowSelector(_ params: [String: JSONValue]) -> Bool {
params["windowId"] != nil || params["windowIndex"] != nil
}
@@ -2969,7 +3229,6 @@ private struct WindowCandidate {
var value = Int(bounds.width * bounds.height)
if layer == 0 { value += 1_000_000_000 }
if title != nil && title?.isEmpty == false { value += 10_000_000 }
if isOnScreen { value += 1_000_000 }
if alpha >= 0.99 { value += 100_000 }
return value
}
@@ -3015,6 +3274,8 @@ private struct WindowCapture {
return WindowCapture(candidate: visibleWindows[windowIndex], captureImage: captureImage)
}
guard let best = candidates.sorted(by: { lhs, rhs in
// Why: the focused window's title can name a window on another Space; a visible window still comes first.
if lhs.isOnScreen != rhs.isOnScreen { return lhs.isOnScreen }
if let titleHint, lhs.title == titleHint, rhs.title != titleHint { return true }
if let titleHint, rhs.title == titleHint, lhs.title != titleHint { return false }
return lhs.score > rhs.score
@@ -3038,8 +3299,10 @@ private struct WindowCapture {
}
}
static func candidates(pid: pid_t) -> [WindowCandidate] {
guard let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else {
/// Off-screen windows include minimized, hidden and other-Space ones, but also closed windows apps keep; see `windowInventory`.
static func candidates(pid: pid_t, includeOffScreen: Bool = false) -> [WindowCandidate] {
let options: CGWindowListOption = includeOffScreen ? [.optionAll, .excludeDesktopElements] : [.optionOnScreenOnly]
guard let infos = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
return []
}
return infos.compactMap { info in
@@ -3057,7 +3320,8 @@ private struct WindowCapture {
guard alpha > 0.01 else { return nil }
let sharing = (info[kCGWindowSharingState as String] as? NSNumber).map { $0.intValue }
if sharing == 0 { return nil }
let isOnScreen = (info[kCGWindowIsOnscreen as String] as? Bool) ?? true
// Why: CoreGraphics omits the key for off-screen windows, and every window of an on-screen-only list is on screen.
let isOnScreen = (info[kCGWindowIsOnscreen as String] as? Bool) ?? !includeOffScreen
return WindowCandidate(
windowId: CGWindowID(number.uint32Value),
layer: layer,
@@ -3120,7 +3384,11 @@ private final class AsyncBox<T>: @unchecked Sendable {
}
private enum BlockingAsync {
static func run<T>(timeout: TimeInterval, operation: @escaping @Sendable () async throws -> T) throws -> T {
static func run<T>(
timeout: TimeInterval,
timeoutMessage: String = "screenshot capture timed out",
operation: @escaping @Sendable () async throws -> T
) throws -> T {
let semaphore = DispatchSemaphore(value: 0)
let box = AsyncBox<T>()
let task = Task.detached {
@@ -3133,7 +3401,7 @@ private enum BlockingAsync {
}
guard semaphore.wait(timeout: .now() + timeout) == .success else {
task.cancel()
throw ProviderError.coded("action_timeout", "screenshot capture timed out")
throw ProviderError.coded("action_timeout", timeoutMessage)
}
return try box.result!.get()
}
@@ -0,0 +1,27 @@
import CoreGraphics
/// Decides from what happened, not from the request, whether an action put the target in front of the user.
public enum ForegroundDelivery {
public struct Observation: Equatable {
public let frontmost: Bool
/// Read only while the app is frontmost, since only then is its key window the user's.
public let focusedWindowID: CGWindowID?
public init(frontmost: Bool, focusedWindowID: CGWindowID?) {
self.frontmost = frontmost
self.focusedWindowID = focusedWindowID
}
}
public static func tookForeground(
before: Observation,
after: Observation,
targetWindowID: CGWindowID,
restoreRequested: Bool
) -> Bool {
guard after.frontmost else { return false }
// Why: --restore-window input goes through the HID tap, which lands in whatever is in front.
if restoreRequested || !before.frontmost { return true }
return before.focusedWindowID != targetWindowID && after.focusedWindowID == targetWindowID
}
}
@@ -0,0 +1,56 @@
import ApplicationServices
import CoreGraphics
import Darwin
import Foundation
/// Finds an app's accessibility windows on other Spaces, which `kAXWindowsAttribute` leaves out,
/// by rebuilding element references from remote tokens (the technique AltTab uses).
/// Only finds windows some accessibility client has read before: an app numbers its elements in the order it first hands them out.
public enum RemoteAccessibilityWindows {
/// Windows are usually handed out first (ids 42-73 measured in apps open for days); a window first read after
/// a large tree walk gets a later id.
static let elementIdLimit: UInt64 = 20_000
/// About 12 µs per id; the budget bounds a slow app.
static let scanBudgetSeconds: CFAbsoluteTime = 0.5
/// The windows of `pid` whose window ids are in `wanted`.
public static func windows(pid: pid_t, wanted: Set<CGWindowID>) -> [CGWindowID: AXUIElement] {
guard let createWithRemoteToken, !wanted.isEmpty else { return [:] }
let deadline = CFAbsoluteTimeGetCurrent() + scanBudgetSeconds
var found: [CGWindowID: AXUIElement] = [:]
for elementId in 0..<elementIdLimit where CFAbsoluteTimeGetCurrent() < deadline {
guard let element = createWithRemoteToken(token(pid: pid, elementId: elementId) as CFData)?.takeRetainedValue() else {
continue
}
var role: CFTypeRef?
let status = AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &role)
// Why: an app that stops answering would otherwise cost the AX timeout for every remaining id.
if status == .cannotComplete { break }
guard status == .success, role as? String == kAXWindowRole as String,
let windowID = try? AccessibilityWindowIdentity.windowID(of: element),
wanted.contains(windowID)
else { continue }
found[windowID] = element
if found.count == wanted.count { break }
}
return found
}
/// The pid, a zero word, the "coco" tag, then the element id.
static func token(pid: pid_t, elementId: UInt64) -> Data {
var token = Data()
withUnsafeBytes(of: pid.littleEndian) { token.append(contentsOf: $0) }
withUnsafeBytes(of: Int32(0)) { token.append(contentsOf: $0) }
withUnsafeBytes(of: Int32(0x636F_636F).littleEndian) { token.append(contentsOf: $0) }
withUnsafeBytes(of: elementId.littleEndian) { token.append(contentsOf: $0) }
return token
}
private typealias CreateWithRemoteToken = @convention(c) (CFData) -> Unmanaged<AXUIElement>?
private static let createWithRemoteToken: CreateWithRemoteToken? = {
guard let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "_AXUIElementCreateWithRemoteToken") else { return nil }
// SAFETY: HIServices exports AXUIElementRef _AXUIElementCreateWithRemoteToken(CFDataRef), returning a +1 reference.
return unsafeBitCast(symbol, to: CreateWithRemoteToken.self)
}()
}
@@ -0,0 +1,40 @@
import CoreGraphics
import Darwin
import Foundation
/// Which windows belong to a Space. Minimized, hidden and other-Space windows do; a closed window an app keeps around does not.
public enum WindowSpaces {
/// Nil when SkyLight's lookup is missing, so closed windows can't be told apart from other-Space ones.
public static func windowsInAnySpace(_ windowIDs: [CGWindowID]) -> Set<CGWindowID>? {
guard let symbols else { return nil }
let connection = symbols.mainConnection()
return Set(windowIDs.filter { windowID in
let spaces = symbols.copySpacesForWindows(connection, allSpacesMask, [NSNumber(value: windowID)] as CFArray)?
.takeRetainedValue() as? [NSNumber]
return spaces?.isEmpty == false
})
}
private static let allSpacesMask: Int32 = 0x7
private typealias MainConnectionID = @convention(c) () -> Int32
private typealias CopySpacesForWindows = @convention(c) (Int32, Int32, CFArray) -> Unmanaged<CFArray>?
private struct Symbols {
let mainConnection: MainConnectionID
let copySpacesForWindows: CopySpacesForWindows
}
private static let symbols: Symbols? = {
let skyLight = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY)
guard let main = dlsym(skyLight, "SLSMainConnectionID"),
let copy = dlsym(skyLight, "SLSCopySpacesForWindows")
else { return nil }
// SAFETY: SkyLight exports CGSConnectionID SLSMainConnectionID(void) and
// CFArrayRef SLSCopySpacesForWindows(CGSConnectionID, CGSSpaceMask, CFArrayRef), returning a +1 array (as yabai uses them).
return Symbols(
mainConnection: unsafeBitCast(main, to: MainConnectionID.self),
copySpacesForWindows: unsafeBitCast(copy, to: CopySpacesForWindows.self)
)
}()
}
@@ -0,0 +1,25 @@
import CoreGraphics
import XCTest
@testable import OrcaComputerUseMacOSCore
final class ForegroundDeliveryTests: XCTestCase {
private func observation(_ frontmost: Bool, _ focusedWindowID: CGWindowID? = nil) -> ForegroundDelivery.Observation {
ForegroundDelivery.Observation(frontmost: frontmost, focusedWindowID: focusedWindowID)
}
func testBackgroundActionThatActivatesTheAppReportsForeground() {
XCTAssertTrue(ForegroundDelivery.tookForeground(before: observation(false), after: observation(true, 7), targetWindowID: 7, restoreRequested: false))
XCTAssertFalse(ForegroundDelivery.tookForeground(before: observation(false), after: observation(false), targetWindowID: 7, restoreRequested: false))
}
func testActiveAppCountsOnlyWhenTheTargetWindowBecameKey() {
XCTAssertTrue(ForegroundDelivery.tookForeground(before: observation(true, 3), after: observation(true, 7), targetWindowID: 7, restoreRequested: false))
XCTAssertFalse(ForegroundDelivery.tookForeground(before: observation(true, 7), after: observation(true, 7), targetWindowID: 7, restoreRequested: false))
XCTAssertFalse(ForegroundDelivery.tookForeground(before: observation(true, 3), after: observation(true, 3), targetWindowID: 7, restoreRequested: false))
}
func testRestoreReportsWhatHappenedNotWhatWasAsked() {
XCTAssertTrue(ForegroundDelivery.tookForeground(before: observation(true, 7), after: observation(true, 7), targetWindowID: 7, restoreRequested: true))
XCTAssertFalse(ForegroundDelivery.tookForeground(before: observation(false), after: observation(false), targetWindowID: 7, restoreRequested: true))
}
}
@@ -0,0 +1,117 @@
param([string]$RuntimePath = (Join-Path $PSScriptRoot "runtime.ps1"))
$ErrorActionPreference = "Stop"
function Assert-TestEqual($Actual, $Expected, [string]$Label) {
if ($Actual -cne $Expected) { throw "$Label expected '$Expected', received '$Actual'" }
}
Add-Type -TypeDefinition @"
using System;
using System.Collections.Generic;
// A combo box and a list box in a fake process 4242, answering the messages the runtime sends.
public static class OrcaDesktopWin32 {
public static readonly List<string> Calls = new List<string>();
public static string[] Items;
public static int Selected;
public static int Top;
public static int ListCount;
public static int ClientHeight;
public static void Reset(string[] items) {
Calls.Clear();
Items = items;
Selected = 0;
Top = 0;
ListCount = 100;
ClientHeight = 60;
}
public static bool IsWindow(IntPtr hwnd) { return true; }
public static uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId) { processId = 4242; return 1; }
public static int GetWindowLong(IntPtr hwnd, int index) { return 0; }
public static IntPtr GetParent(IntPtr hwnd) { return new IntPtr(101); }
public static int GetDlgCtrlID(IntPtr hwnd) { return 7; }
public static int GetClientHeight(IntPtr hwnd) { return ClientHeight; }
// CB_FINDSTRINGEXACT: case-insensitive, searching after start and wrapping.
public static long? FindControlString(IntPtr hwnd, uint message, int start, string name) {
for (var step = 1; step <= Items.Length; step++) {
var index = (start + step + Items.Length) % Items.Length;
if (string.Equals(Items[index], name, StringComparison.OrdinalIgnoreCase)) { return index; }
}
return -1;
}
public static string GetControlItemText(IntPtr hwnd, uint lengthMessage, uint textMessage, int index) { return Items[index]; }
public static long? SendControlMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam) {
switch (message) {
case 0x0146: return Items.Length;
case 0x0147: return Selected;
case 0x014E: Selected = wParam.ToInt32(); Calls.Add("select:" + Selected); return Selected;
case 0x0111: Calls.Add("command:" + hwnd + ":" + wParam.ToInt64().ToString("X")); return 0;
case 0x018E: return Top;
case 0x018B: return ListCount;
case 0x01A1: return 15;
case 0x0197: Top = wParam.ToInt32(); Calls.Add("top:" + Top); return 0;
default: return 0;
}
}
}
"@
$tokens = $null
$errors = $null
$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors)
if ($errors.Count) { throw ($errors | Out-String) }
$functions = @(
"Test-OrcaNativeClass", "Test-OrcaEditClass", "Get-OrcaNativeControl", "Assert-OrcaWindowOwner", "Get-OrcaProperty",
"Set-OrcaNativeComboValue", "Set-OrcaElementValue", "Invoke-OrcaNativeListScroll", "Test-OrcaSameRuntimeId"
)
foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
}
$process = [pscustomobject]@{ Id = 4242 }
function Get-OrcaRuntimeId($Element) { @($Element.id) }
function Test-OrcaElementDisabled($Element) { -not $Element.Current.IsEnabled }
function Set-OrcaNativeEditValue { $null }
function New-TestControl([string]$ClassName, [bool]$Enabled = $true) {
[pscustomobject]@{ id = 5; Current = [pscustomobject]@{ ClassName = $ClassName; NativeWindowHandle = 303; IsEnabled = $Enabled } }
}
$combo = New-TestControl "WindowsForms10.COMBOBOX.app.0.1"
# The exact-case option is picked among case-insensitive hits, and the app's change handler is told.
[OrcaDesktopWin32]::Reset(@("Alpha", "gamma", "Gamma"))
$written = Set-OrcaElementValue $combo $process 5 "Gamma"
Assert-TestEqual ([OrcaDesktopWin32]::Calls -join "|") "select:2|command:101:10007" "combo selection messages"
Assert-TestEqual $written.verification.state "verified" "combo verification"
# Two options with the exact name can't be told apart, so nothing is selected.
[OrcaDesktopWin32]::Reset(@("Gamma", "Beta", "Gamma"))
$message = try { Set-OrcaElementValue $combo $process 5 "Gamma" | Out-Null; $null } catch { $_.Exception.Message }
if ($message -notlike "*more than one option is named 'Gamma'*") { throw "ambiguous combo, received '$message'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "ambiguous combo sends nothing"
# A disabled combo box is locked against the user, so it is refused before any message.
[OrcaDesktopWin32]::Reset(@("Alpha", "Gamma"))
$message = try { Set-OrcaElementValue (New-TestControl "ComboBox" $false) $process 5 "Gamma" | Out-Null; $null } catch { $_.Exception.Message }
Assert-TestEqual $message "element_not_enabled: element 5 is disabled, so no input was sent; do what enables it first, then run get-app-state and retry once its line no longer lists disabled" "disabled combo"
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "disabled combo sends nothing"
# A page is the rows the list shows: 60 px at 15 px a row is 4 rows.
$list = New-TestControl "ListBox"
[OrcaDesktopWin32]::Reset(@())
Assert-TestEqual (Invoke-OrcaNativeListScroll $list $list $process "down" 2) $true "list scroll"
Assert-TestEqual ([OrcaDesktopWin32]::Calls -join "|") "top:8" "list scroll top row"
# At the end already, there is nothing to move and nothing falls back to ScrollPattern.
[OrcaDesktopWin32]::Reset(@())
[OrcaDesktopWin32]::Top = 96
Assert-TestEqual (Invoke-OrcaNativeListScroll $list $list $process "down" 1) $true "list at its end"
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "list at its end sends nothing"
# A list that shows all its rows leaves the scroll to an enclosing ScrollPattern.
[OrcaDesktopWin32]::Reset(@())
[OrcaDesktopWin32]::ListCount = 3
Assert-TestEqual (Invoke-OrcaNativeListScroll $list $list $process "down" 1) $false "short list"
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "short list sends nothing"
Write-Output "windows-native-controls-tests-ok"
@@ -107,7 +107,7 @@ $functions = @(
"Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation",
"Assert-OrcaInputAllowed", "Test-OrcaTargetTookForeground", "Invoke-OrcaPrimaryAction", "Get-OrcaScrollPattern",
"Get-OrcaScrollPercent", "Get-OrcaScrollViewSize", "Get-OrcaRuntimeId", "Test-OrcaSameRuntimeId", "Wait-OrcaScrollPercentChange", "Invoke-OrcaScrollPercent",
"Invoke-OrcaScrollPattern"
"Invoke-OrcaScrollPattern", "Invoke-OrcaNativeListScroll", "Invoke-OrcaNativeButtonClick"
)
foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
@@ -301,9 +301,9 @@ if ($uiaTypes) {
$element
}
$ignored = New-TestScrollElement 30.0
Assert-TestEqual (Invoke-OrcaScrollPatternReal $ignored $ignored "down" 1) $false "whole page the provider ignores"
Assert-TestEqual (Invoke-OrcaScrollPatternReal $ignored $ignored $null "down" 1) $false "whole page the provider ignores"
$atEnd = New-TestScrollElement 100.0
Assert-TestEqual (Invoke-OrcaScrollPatternReal $atEnd $atEnd "down" 1) $true "whole page at the end of the content"
Assert-TestEqual (Invoke-OrcaScrollPatternReal $atEnd $atEnd $null "down" 1) $true "whole page at the end of the content"
} else {
Write-Output "skipping UI Automation pattern checks: UIAutomationClient is unavailable"
}
@@ -0,0 +1,189 @@
param([string]$RuntimePath = (Join-Path $PSScriptRoot "runtime.ps1"))
$ErrorActionPreference = "Stop"
function Assert-TestEqual($Actual, $Expected, [string]$Label) {
if ($Actual -cne $Expected) { throw "$Label expected '$Expected', received '$Actual'" }
}
Add-Type -TypeDefinition @"
using System;
using System.Collections.Generic;
public static class OrcaDesktopWin32 {
public static readonly List<string> Calls = new List<string>();
public static IntPtr Foreground;
public static string EditText;
public static int[] Selection;
public static int EditOwner;
public static bool IgnoreInsertion;
public static long Limit;
public static void Reset(string text, int start, int end) {
Calls.Clear();
Foreground = new IntPtr(202);
EditText = text;
Selection = new[] { start, end };
EditOwner = 4242;
IgnoreInsertion = false;
Limit = 30000;
}
public static int GetProcessIntegrityLevel(int processId) { return 0x2000; }
public static IntPtr GetForegroundWindow() { return Foreground; }
public static bool SetForegroundWindow(IntPtr hwnd) { Calls.Add("activate"); return false; }
public static bool IsIconic(IntPtr hwnd) { return false; }
public static bool ShowWindow(IntPtr hwnd, int command) { Calls.Add("restore"); return true; }
public static bool IsWindow(IntPtr hwnd) { return true; }
public static int GetWindowLong(IntPtr hwnd, int index) { return 0x0004; }
public static uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId) {
processId = hwnd == new IntPtr(303) ? (uint)EditOwner : (hwnd == new IntPtr(101) ? 4242u : 1u);
return 1;
}
public static string GetEditText(IntPtr hwnd, int maxLength) { return EditText; }
public static int[] GetEditSelection(IntPtr hwnd) { return Selection; }
public static bool ReplaceEditSelection(IntPtr hwnd, string text) {
Calls.Add("replace:" + text);
if (!IgnoreInsertion) {
EditText = EditText.Substring(0, Selection[0]) + text + EditText.Substring(Selection[1]);
}
return true;
}
public static bool SendUnicodeText(string text) { Calls.Add("keys:" + text); return true; }
public static long? SendControlMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam) { return message == 0x00D5 ? Limit : 0; }
}
"@
$tokens = $null
$errors = $null
$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors)
if ($errors.Count) { throw ($errors | Out-String) }
$functions = @(
"Test-OrcaWindowFocused", "Wait-OrcaWindowFocused", "Assert-OrcaKeyboardFocus", "New-OrcaForegroundRefusal",
"Assert-OrcaWindowTarget", "Get-OrcaRequiredString", "Invoke-OrcaOperation",
"Assert-OrcaInputAllowed", "Test-OrcaTargetTookForeground", "Get-OrcaProperty", "Test-OrcaSensitiveElement",
"Test-OrcaEditClass", "Get-OrcaNativeEdit", "Assert-OrcaWindowOwner", "Test-OrcaNativeClass", "ConvertTo-OrcaEditNewlines",
"Invoke-OrcaEditTextInsertion", "Invoke-OrcaElementTextInsertion", "Send-OrcaText",
"Test-OrcaElementDisabled", "Assert-OrcaStillFocused"
)
foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
}
$EditReadbackLimit = 1048576
function Find-OrcaProcess { [pscustomobject]@{ MainWindowHandle = 101; Id = 4242; ProcessName = "Fixture" } }
function Get-OrcaAppName($Process) { $Process.ProcessName }
function Get-OrcaRootElement { $null }
function Get-OrcaWindowFrame { @{ x = 100; y = 200 } }
function Find-OrcaElement($RootElement, $Record) { if ($null -eq $Record) { $null } else { $Record.live } }
function Invoke-OrcaValueTextInsertion { $script:valueWrites++; $null }
function Send-OrcaKey { [OrcaDesktopWin32]::Calls.Add("sendkeys") }
function Send-OrcaHotkey { [OrcaDesktopWin32]::Calls.Add("sendkeys") }
function Send-OrcaPasteText { [OrcaDesktopWin32]::Calls.Add("sendkeys") }
function New-OrcaSnapshot { [pscustomobject]@{} }
function Test-OrcaTextPatternReadOnly { $false }
function Restore-OrcaWindow { [OrcaDesktopWin32]::Calls.Add("activate"); $null }
function New-TestElement([string]$ClassName, [int64]$Handle, [bool]$Enabled = $true, [string]$ControlType = "ControlType.Document") {
$live = [pscustomobject]@{
Current = [pscustomobject]@{
ClassName = $ClassName; NativeWindowHandle = $Handle; IsPassword = $false; IsEnabled = $Enabled; Name = "Body"
AutomationId = ""; LocalizedControlType = "document"; ControlType = [pscustomobject]@{ ProgrammaticName = $ControlType }
}
}
[pscustomobject]@{ index = 7; live = $live }
}
function Invoke-TestOperation($Operation) {
$message = $null
$result = $null
try { $result = Invoke-OrcaOperation $Operation } catch { $message = $_.Exception.Message }
[pscustomobject]@{ result = $result; message = $message }
}
# Background keyboard verbs refuse before any key leaves, and say how to enter text without the foreground.
foreach ($operation in @(
[pscustomobject]@{ tool = "type_text"; text = "hello" },
[pscustomobject]@{ tool = "paste_text"; text = "hello" },
[pscustomobject]@{ tool = "press_key"; key = "Return" },
[pscustomobject]@{ tool = "hotkey"; key = "Ctrl+A" }
)) {
[OrcaDesktopWin32]::Reset("", 0, 0)
$operation | Add-Member -NotePropertyName app -NotePropertyValue "Fixture"
$outcome = Invoke-TestOperation $operation
if ($outcome.message -notlike "background_unavailable: *takes the user's keyboard; nothing was sent.*type-text --element-index*Ask the user before retrying with --restore-window.") {
throw "$($operation.tool) background refusal, received '$($outcome.message)'"
}
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "$($operation.tool) background refusal sends nothing"
}
# Typing into a native edit replaces its selection in the background and reads the result back.
[OrcaDesktopWin32]::Reset("one`r`nthree", 3, 3)
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "`ntwo"; element = (New-TestElement "Edit" 303) })
if ($null -ne $outcome.message) { throw "edit insertion failed: $($outcome.message)" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls -join "|") "replace:`r`ntwo" "edit insertion sends one EM_REPLACESEL with CRLF line breaks"
Assert-TestEqual ([OrcaDesktopWin32]::EditText) "one`r`ntwo`r`nthree" "edit insertion result"
Assert-TestEqual $outcome.result.action.actionName "EM_REPLACESEL" "edit insertion action"
Assert-TestEqual $outcome.result.action.delivery "background" "edit insertion delivery"
Assert-TestEqual $outcome.result.action.verification.state "verified" "edit insertion verification"
# A field that ignores the insertion is an error, not a typed result.
[OrcaDesktopWin32]::Reset("same", 4, 4)
[OrcaDesktopWin32]::IgnoreInsertion = $true
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "x"; element = (New-TestElement "Edit" 303) })
if ($outcome.message -notlike "text_insertion_unconfirmed:*") { throw "ignored insertion, received '$($outcome.message)'" }
# Replacing a selection with the same text changes nothing yet is the requested result.
[OrcaDesktopWin32]::Reset("abc", 0, 3)
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "abc"; element = (New-TestElement "Edit" 303) })
if ($null -ne $outcome.message) { throw "identical replacement failed: $($outcome.message)" }
Assert-TestEqual $outcome.result.action.verification.state "verified" "identical replacement verification"
# An edit cuts text past its limit, so an insertion that would overflow it is refused before anything is sent.
[OrcaDesktopWin32]::Reset("old", 3, 3)
[OrcaDesktopWin32]::Limit = 5
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "abc"; element = (New-TestElement "Edit" 303) })
if ($outcome.message -notlike "element 7 text is not settable: the result would be 6 characters, past the field's 5-character limit") { throw "over-limit insertion, received '$($outcome.message)'" }
Assert-TestEqual ([OrcaDesktopWin32]::EditText) "old" "over-limit insertion leaves the field untouched"
# A password field can't be read back, yet its limit still holds.
[OrcaDesktopWin32]::Reset("", 0, 0)
[OrcaDesktopWin32]::Limit = 2
$secret = New-TestElement "Edit" 303
$secret.live.Current.IsPassword = $true
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "abc"; element = $secret })
if ($outcome.message -notlike "*past the field's 2-character limit") { throw "over-limit password insertion, received '$($outcome.message)'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "over-limit password insertion sends nothing"
# Edit messages would bypass a disabled field's lock, so a disabled field is refused untouched.
[OrcaDesktopWin32]::Reset("", 0, 0)
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "x"; element = (New-TestElement "Edit" 303 $false) })
if ($outcome.message -notlike "element 7 text is not settable: the field is disabled") { throw "disabled edit, received '$($outcome.message)'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "disabled edit sends nothing"
# A recycled window handle owned by another process is never messaged.
[OrcaDesktopWin32]::Reset("", 0, 0)
[OrcaDesktopWin32]::EditOwner = 999
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "x"; element = (New-TestElement "Edit" 303) })
if ($outcome.message -notlike "stale element:*") { throw "recycled edit handle, received '$($outcome.message)'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "recycled edit handle sends nothing"
# An element with no background text route refuses instead of typing at whatever has focus.
[OrcaDesktopWin32]::Reset("", 0, 0)
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "x"; element = (New-TestElement "Chrome_RenderWidgetHostHWND" 404) })
if ($outcome.message -notlike "background_unavailable: element 7 accepts text only as keys*") { throw "no text route, received '$($outcome.message)'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "no text route sends nothing"
# List and tree items take ValuePattern too, where a write would rename them, so they are refused untouched.
[OrcaDesktopWin32]::Reset("", 0, 0)
$script:valueWrites = 0
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "x"; element = (New-TestElement "" 0 $true "ControlType.ListItem") })
if ($outcome.message -notlike "element 7 text is not settable: it is not a text field") { throw "list item, received '$($outcome.message)'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count + $script:valueWrites) 0 "list item sends nothing"
# Foreground typing re-checks focus right before SendInput, so a user's switch never receives the text.
[OrcaDesktopWin32]::Reset("", 0, 0)
[OrcaDesktopWin32]::Foreground = [IntPtr]101
$outcome = Invoke-TestOperation ([pscustomobject]@{ tool = "type_text"; app = "Fixture"; text = "caf$([char]0x00E9)"; restoreWindow = $true })
if ($null -ne $outcome.message) { throw "foreground typing failed: $($outcome.message)" }
Assert-TestEqual (@([OrcaDesktopWin32]::Calls | Where-Object { $_ -like "keys:*" }) -join "|") "keys:caf$([char]0x00E9)" "foreground typing sends Unicode input"
Write-Output "windows-text-entry-tests-ok"
+720 -48
View File
@@ -89,9 +89,6 @@ public static class OrcaDesktopWin32 {
[DllImport("user32.dll")]
public static extern bool ScreenToClient(IntPtr hwnd, ref POINT point);
[DllImport("user32.dll", EntryPoint = "PostMessageW")]
public static extern bool PostMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam);
[DllImport("user32.dll")]
public static extern bool ShowWindow(IntPtr hwnd, int nCmdShow);
@@ -143,6 +140,27 @@ public static class OrcaDesktopWin32 {
[DllImport("user32.dll", EntryPoint = "SendMessageTimeoutW", SetLastError = true)]
private static extern IntPtr SendMessageTimeout(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam, uint flags, uint timeout, out IntPtr result);
[DllImport("user32.dll", EntryPoint = "SendMessageTimeoutW", CharSet = CharSet.Unicode)]
private static extern IntPtr SendMessageTimeoutString(IntPtr hwnd, uint message, IntPtr wParam, string lParam, uint flags, uint timeout, out IntPtr result);
[DllImport("user32.dll", EntryPoint = "SendMessageTimeoutW", CharSet = CharSet.Unicode)]
private static extern IntPtr SendMessageTimeoutBuffer(IntPtr hwnd, uint message, IntPtr wParam, System.Text.StringBuilder lParam, uint flags, uint timeout, out IntPtr result);
[DllImport("user32.dll")]
public static extern bool IsWindow(IntPtr hwnd);
[DllImport("user32.dll", EntryPoint = "PostMessageW")]
public static extern bool PostMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam);
[DllImport("user32.dll")]
public static extern IntPtr GetParent(IntPtr hwnd);
[DllImport("user32.dll")]
public static extern int GetDlgCtrlID(IntPtr hwnd);
[DllImport("user32.dll", EntryPoint = "GetWindowLongW")]
public static extern int GetWindowLong(IntPtr hwnd, int index);
[DllImport("user32.dll")]
private static extern bool IsHungAppWindow(IntPtr hwnd);
@@ -175,6 +193,90 @@ public static class OrcaDesktopWin32 {
return Marshal.GetLastWin32Error() == errorTimeout || IsHungAppWindow(hwnd);
}
const uint AbortIfHung = 0x0002;
const uint EditMessageTimeoutMilliseconds = 2000;
// EM_REPLACESEL: user32 marshals its string across processes and the edit needs no focus; false when it did not answer.
public static bool ReplaceEditSelection(IntPtr hwnd, string text) {
IntPtr result;
return SendMessageTimeoutString(hwnd, 0x00C2, new IntPtr(1), text, AbortIfHung, EditMessageTimeoutMilliseconds, out result) != IntPtr.Zero;
}
// WM_GETTEXT, at most maxLength characters; null when the window did not answer.
public static string GetEditText(IntPtr hwnd, int maxLength) {
IntPtr length;
if (SendMessageTimeout(hwnd, 0x000E, IntPtr.Zero, IntPtr.Zero, AbortIfHung, EditMessageTimeoutMilliseconds, out length) == IntPtr.Zero) { return null; }
if (length.ToInt64() <= 0) { return ""; }
var capacity = (int)Math.Min(Math.Max(0L, length.ToInt64()), (long)maxLength) + 1;
var buffer = new System.Text.StringBuilder(capacity);
IntPtr copied;
if (SendMessageTimeoutBuffer(hwnd, 0x000D, new IntPtr(capacity), buffer, AbortIfHung, EditMessageTimeoutMilliseconds, out copied) == IntPtr.Zero) { return null; }
return buffer.ToString();
}
// A control message with no pointer arguments; null when the window did not answer.
public static long? SendControlMessage(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam) {
IntPtr result;
if (SendMessageTimeout(hwnd, message, wParam, lParam, AbortIfHung, EditMessageTimeoutMilliseconds, out result) == IntPtr.Zero) { return null; }
return result.ToInt64();
}
// An exact-name search such as CB_FINDSTRINGEXACT, whose string user32 marshals across processes; null when unanswered.
public static long? FindControlString(IntPtr hwnd, uint message, int start, string name) {
IntPtr result;
if (SendMessageTimeoutString(hwnd, message, new IntPtr(start), name, AbortIfHung, EditMessageTimeoutMilliseconds, out result) == IntPtr.Zero) { return null; }
return result.ToInt64();
}
// An item's text through a length/text message pair such as CB_GETLBTEXTLEN/CB_GETLBTEXT; null when unanswered.
public static string GetControlItemText(IntPtr hwnd, uint lengthMessage, uint textMessage, int index) {
IntPtr length;
if (SendMessageTimeout(hwnd, lengthMessage, new IntPtr(index), IntPtr.Zero, AbortIfHung, EditMessageTimeoutMilliseconds, out length) == IntPtr.Zero || length.ToInt64() < 0) { return null; }
var buffer = new System.Text.StringBuilder((int)length.ToInt64() + 1);
IntPtr copied;
if (SendMessageTimeoutBuffer(hwnd, textMessage, new IntPtr(index), buffer, AbortIfHung, EditMessageTimeoutMilliseconds, out copied) == IntPtr.Zero) { return null; }
return buffer.ToString();
}
public static int GetClientHeight(IntPtr hwnd) {
RECT rect;
return GetClientRect(hwnd, out rect) ? rect.Bottom - rect.Top : 0;
}
// EM_GETSEL as [start, end]; null when the edit did not answer or an offset is past the 16-bit return value.
public static int[] GetEditSelection(IntPtr hwnd) {
IntPtr result;
if (SendMessageTimeout(hwnd, 0x00B0, IntPtr.Zero, IntPtr.Zero, AbortIfHung, EditMessageTimeoutMilliseconds, out result) == IntPtr.Zero) { return null; }
var packed = result.ToInt64() & 0xFFFFFFFFL;
if (packed == 0xFFFFFFFFL) { return null; }
return new[] { (int)(packed & 0xFFFF), (int)(packed >> 16) };
}
// One SendInput batch of KEYEVENTF_UNICODE characters, so any layout types them and no other input interleaves.
public static bool SendUnicodeText(string text) {
const uint keyboardInput = 1;
const uint keyUp = 0x0002;
const uint unicode = 0x0004;
const byte returnKey = 0x0D;
const byte tabKey = 0x09;
var inputs = new List<INPUT>();
foreach (var unit in text) {
if (unit == '\r') { continue; }
// Why: Return and Tab go as keys, as SendKeys sent them, so dialogs and forms act on them.
if (unit == '\n' || unit == '\t') {
var key = unit == '\n' ? returnKey : tabKey;
inputs.Add(KeyboardInput(keyboardInput, key, 0));
inputs.Add(KeyboardInput(keyboardInput, key, keyUp));
continue;
}
inputs.Add(UnicodeInput(unit, unicode));
inputs.Add(UnicodeInput(unit, unicode | keyUp));
}
var values = inputs.ToArray();
if (values.Length == 0) { return true; }
return SendInput((uint)values.Length, values, Marshal.SizeOf(typeof(INPUT))) == (uint)values.Length;
}
// Mandatory integrity RID (0x2000 medium, 0x3000 high), or -1 when the process cannot be queried.
public static int GetProcessIntegrityLevel(int processId) {
const uint processQueryLimitedInformation = 0x1000;
@@ -359,6 +461,15 @@ public static class OrcaDesktopWin32 {
};
}
private static INPUT UnicodeInput(char unit, uint flags) {
return new INPUT {
type = 1,
data = new INPUTUNION {
keyboard = new KEYBDINPUT { scanCode = unit, flags = flags }
}
};
}
private static INPUT MouseInput(uint type, uint flags) {
return new INPUT {
type = type,
@@ -367,6 +478,123 @@ public static class OrcaDesktopWin32 {
}
};
}
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
private struct STARTUPINFO {
public int cb;
public string lpReserved;
public string lpDesktop;
public string lpTitle;
public int dwX;
public int dwY;
public int dwXSize;
public int dwYSize;
public int dwXCountChars;
public int dwYCountChars;
public int dwFillAttribute;
public int dwFlags;
public short wShowWindow;
public short cbReserved2;
public IntPtr lpReserved2;
public IntPtr hStdInput;
public IntPtr hStdOutput;
public IntPtr hStdError;
}
[StructLayout(LayoutKind.Sequential)]
private struct STARTUPINFOEX {
public STARTUPINFO StartupInfo;
public IntPtr lpAttributeList;
}
[StructLayout(LayoutKind.Sequential)]
private struct PROCESS_INFORMATION {
public IntPtr hProcess;
public IntPtr hThread;
public int dwProcessId;
public int dwThreadId;
}
[DllImport("kernel32.dll", EntryPoint = "CreateProcessW", SetLastError = true, CharSet = CharSet.Unicode)]
private static extern bool CreateProcess(string applicationName, System.Text.StringBuilder commandLine, IntPtr processAttributes, IntPtr threadAttributes, bool inheritHandles, uint creationFlags, IntPtr environment, string currentDirectory, ref STARTUPINFOEX startupInfo, out PROCESS_INFORMATION processInformation);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool InitializeProcThreadAttributeList(IntPtr attributeList, int attributeCount, int flags, ref IntPtr size);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool UpdateProcThreadAttribute(IntPtr attributeList, uint flags, IntPtr attribute, IntPtr value, IntPtr size, IntPtr previousValue, IntPtr returnSize);
[DllImport("kernel32.dll")]
private static extern void DeleteProcThreadAttributeList(IntPtr attributeList);
[DllImport("user32.dll")]
private static extern IntPtr GetShellWindow();
[DllImport("user32.dll", SetLastError = true)]
private static extern bool SetWindowPos(IntPtr hwnd, IntPtr insertAfter, int x, int y, int width, int height, uint flags);
// Starts an app without handing it the caller's right to take the foreground (design from agent-desktop #615):
// the desktop shell is its parent, and its first ShowWindow is SW_SHOWNOACTIVATE.
public static int LaunchWithoutForeground(string executable, string workingDirectory) {
const uint processCreateProcess = 0x0080;
const uint extendedStartupInfoPresent = 0x00080000;
const int procThreadAttributeParentProcess = 0x00020000;
const int startfUseShowWindow = 0x0001;
const short swShowNoActivate = 4;
uint shellProcessId;
GetWindowThreadProcessId(GetShellWindow(), out shellProcessId);
if (shellProcessId == 0) {
throw new InvalidOperationException("no desktop shell is running to start the app under");
}
var shell = OpenProcess(processCreateProcess, false, (int)shellProcessId);
if (shell == IntPtr.Zero) {
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error(), "could not open the desktop shell process");
}
var attributes = IntPtr.Zero;
var parentHandle = IntPtr.Zero;
var attributesInitialized = false;
try {
var size = IntPtr.Zero;
InitializeProcThreadAttributeList(IntPtr.Zero, 1, 0, ref size);
attributes = Marshal.AllocHGlobal(size);
if (!InitializeProcThreadAttributeList(attributes, 1, 0, ref size)) {
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
}
attributesInitialized = true;
// CreateProcess reads the handle through this pointer, so it must outlive the call.
parentHandle = Marshal.AllocHGlobal(IntPtr.Size);
Marshal.WriteIntPtr(parentHandle, shell);
if (!UpdateProcThreadAttribute(attributes, 0, (IntPtr)procThreadAttributeParentProcess, parentHandle, (IntPtr)IntPtr.Size, IntPtr.Zero, IntPtr.Zero)) {
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
}
var startup = new STARTUPINFOEX();
startup.StartupInfo.cb = Marshal.SizeOf(typeof(STARTUPINFOEX));
startup.StartupInfo.dwFlags = startfUseShowWindow;
startup.StartupInfo.wShowWindow = swShowNoActivate;
startup.lpAttributeList = attributes;
PROCESS_INFORMATION process;
var commandLine = new System.Text.StringBuilder("\"" + executable + "\"");
if (!CreateProcess(executable, commandLine, IntPtr.Zero, IntPtr.Zero, false, extendedStartupInfoPresent, IntPtr.Zero, workingDirectory, ref startup, out process)) {
throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error());
}
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
return process.dwProcessId;
} finally {
if (attributesInitialized) { DeleteProcThreadAttributeList(attributes); }
if (attributes != IntPtr.Zero) { Marshal.FreeHGlobal(attributes); }
if (parentHandle != IntPtr.Zero) { Marshal.FreeHGlobal(parentHandle); }
CloseHandle(shell);
}
}
// Bottom of the z-order without activating: the window stays restored and drivable but covers nothing.
public static bool SendWindowToBack(IntPtr hwnd) {
const uint swpNoSize = 0x0001;
const uint swpNoMove = 0x0002;
const uint swpNoActivate = 0x0010;
return SetWindowPos(hwnd, new IntPtr(1), 0, 0, 0, 0, swpNoSize | swpNoMove | swpNoActivate);
}
}
"@
@@ -375,6 +603,7 @@ public static class OrcaDesktopWin32 {
$MaxNodes = 1200
$MaxDepth = 64
$TextLimit = 500
$EditReadbackLimit = 1048576
$MaxScreenshotPngBytes = 900000
$MaxScreenshotEdge = 1280
$MinScreenshotScale = 0.25
@@ -390,20 +619,6 @@ $BlockedAppFragments = @(
"proton pass"
)
$WindowsMessages = @{
Char = 0x0102
KeyDown = 0x0100
KeyUp = 0x0101
MouseMove = 0x0200
LeftDown = 0x0201
LeftUp = 0x0202
RightDown = 0x0204
RightUp = 0x0205
MiddleDown = 0x0207
MiddleUp = 0x0208
Wheel = 0x020A
}
$MouseEvents = @{
LeftDown = 0x0002
LeftUp = 0x0004
@@ -620,13 +835,19 @@ function Wait-OrcaWindowFocused([IntPtr]$WindowHandle, [int]$TimeoutMilliseconds
Test-OrcaWindowFocused $WindowHandle
}
function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation, $RestoreFailure = $null) {
function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation, $RestoreFailure = $null, $Process = $null) {
if (Test-OrcaWindowFocused $WindowHandle) { return }
if ([bool]$Operation.restoreWindow) {
$reason = if ($null -ne $RestoreFailure) { " ($RestoreFailure)" } else { "" }
throw "window_not_focused: keyboard input requires the target window to be focused; restoreWindow was requested but the target window is still not focused$reason, so nothing was sent; ask the user to bring it forward or check desktop permissions"
}
throw "window_not_focused: keyboard input requires the target window to be focused, and focusing it would take over the user's keyboard; nothing was sent. Use set-value for editable elements, or ask the user before retrying with --restore-window."
$verb = ([string]$Operation.tool) -replace "_", "-"
$alternative = if ($Operation.tool -in @("type_text", "paste_text")) {
"To enter text in the background, use type-text --element-index <field> or set-value."
} else {
"To press a button in the background, click it with click --element-index; to enter text, use type-text --element-index <field>."
}
throw (New-OrcaForegroundRefusal $Process "$verb sends keys, which go only to the window in front" $alternative "takes the user's keyboard")
}
# Raises the target for real mouse input, then returns its frame read after the raise (restoring moves it).
@@ -638,9 +859,9 @@ function Restore-OrcaPointerWindow($Process, $RootElement) {
Get-OrcaWindowFrame $Process $RootElement
}
function New-OrcaForegroundRefusal($Process, [string]$Need, [string]$Alternative = "") {
function New-OrcaForegroundRefusal($Process, [string]$Need, [string]$Alternative = "", [string]$Takes = "moves the user's pointer") {
$hint = if ([string]::IsNullOrEmpty($Alternative)) { "" } else { " $Alternative" }
"background_unavailable: $Need, which on Windows brings $(Get-OrcaAppName $Process) to the front and moves the user's pointer; nothing was sent.$hint Ask the user before retrying with --restore-window."
"background_unavailable: $Need, which on Windows brings $(Get-OrcaAppName $Process) to the front and $Takes; nothing was sent.$hint Ask the user before retrying with --restore-window."
}
function Get-OrcaCursorPosition {
@@ -710,6 +931,213 @@ function Get-OrcaValueText($Element) {
}
}
function Test-OrcaNativeClass([string]$ClassName, [string]$BaseClass) {
# Why: WinForms controls are superclasses named WindowsForms10.<BASE>.app..., and answer the base class's messages.
$ClassName -match "^(?:WindowsForms\d+\.)?(?:$BaseClass)(?:\.|$)"
}
function Test-OrcaEditClass([string]$ClassName) {
# Why: user32 edits and RichEdit answer EM_ and WM_GETTEXT messages from other processes.
Test-OrcaNativeClass $ClassName "Edit|RichEdit\w*"
}
# Only for elements Get-OrcaValueText left empty, so never a sensitive one (it answers "[redacted]" for those).
function Get-OrcaEditValueText([IntPtr]$Handle) {
# Why: the client-side edit proxy gives a multiline edit (Notepad's document) no ValuePattern, so its text is read directly.
$text = [OrcaDesktopWin32]::GetEditText($Handle, $TextLimit + 1)
if ($null -eq $text) { return "" }
if ($text.Length -gt $TextLimit) { return $text.Substring(0, $TextLimit) + "..." }
$text
}
function Get-OrcaNativeEdit($Element, $Process) {
$className = Get-OrcaProperty $Element "ClassName"
if (-not (Test-OrcaEditClass $className)) { return $null }
$handle = try { [IntPtr][int64]$Element.Current.NativeWindowHandle } catch { [IntPtr]::Zero }
if ($handle -eq [IntPtr]::Zero) { return $null }
[pscustomobject]@{
handle = $handle
processId = [int]$Process.Id
rich = $className -match "RichEdit"
sensitive = Test-OrcaSensitiveElement $Element
}
}
function Assert-OrcaWindowOwner([IntPtr]$Handle, [int]$ProcessId) {
# Why: a window handle can be recycled for another process between the tree read and the message.
$owner = [uint32]0
if ([OrcaDesktopWin32]::IsWindow($Handle)) {
[void][OrcaDesktopWin32]::GetWindowThreadProcessId($Handle, [ref]$owner)
}
if ($owner -ne $ProcessId) {
throw "stale element: the control's window no longer belongs to the target app; run get-app-state again and use a fresh element index"
}
}
# The element's own window when it is a native control of BaseClass owned by the target process.
function Get-OrcaNativeControl($Element, $Process, [string]$BaseClass) {
if (-not (Test-OrcaNativeClass (Get-OrcaProperty $Element "ClassName") $BaseClass)) { return $null }
$handle = try { [IntPtr][int64]$Element.Current.NativeWindowHandle } catch { [IntPtr]::Zero }
if ($handle -eq [IntPtr]::Zero) { return $null }
Assert-OrcaWindowOwner $handle ([int]$Process.Id)
$handle
}
function ConvertTo-OrcaEditNewlines([string]$Text, [bool]$Collapse) {
$normalized = $Text -replace "`r`n|`r", "`n"
if ($Collapse) { return $normalized }
$normalized -replace "`n", "`r`n"
}
function Invoke-OrcaEditTextInsertion($Edit, [int]$Index, [string]$Text, [switch]$ReplaceAll) {
Assert-OrcaWindowOwner $Edit.handle $Edit.processId
$style = [OrcaDesktopWin32]::GetWindowLong($Edit.handle, -16)
if (($style -band 0x0800) -ne 0) { throw "element $Index text is not settable: the field is read-only" }
$insert = ConvertTo-OrcaEditNewlines $Text $false
$before = if ($Edit.sensitive) { $null } else { [OrcaDesktopWin32]::GetEditText($Edit.handle, $EditReadbackLimit) }
$selection = [OrcaDesktopWin32]::GetEditSelection($Edit.handle)
# Why: an edit cuts text past its EM_SETLIMITTEXT limit, which would leave a partial write; refuse it untouched instead.
$limit = [OrcaDesktopWin32]::SendControlMessage($Edit.handle, 0x00D5, [IntPtr]::Zero, [IntPtr]::Zero)
if ($null -ne $limit -and $limit -gt 0) {
# Why: RichEdit counts a line break as one character, so its lengths are compared without CRLF.
# Why: a readback cut at the read limit holds only a prefix, so the field reports its own length then.
$current = if ($null -ne $before -and $before.Length -lt $EditReadbackLimit) { (ConvertTo-OrcaEditNewlines $before $Edit.rich).Length } else { [OrcaDesktopWin32]::SendControlMessage($Edit.handle, 0x000E, [IntPtr]::Zero, [IntPtr]::Zero) }
$replaced = if ($ReplaceAll) { $current } elseif ($null -ne $selection) { [Math]::Abs($selection[1] - $selection[0]) } else { 0 }
$length = $current - $replaced + (ConvertTo-OrcaEditNewlines $Text $Edit.rich).Length
if ($null -ne $current -and $length -gt $limit) { throw "element $Index text is not settable: the result would be $length characters, past the field's $limit-character limit" }
}
Assert-OrcaWindowOwner $Edit.handle $Edit.processId
if (-not [OrcaDesktopWin32]::ReplaceEditSelection($Edit.handle, $insert)) {
throw "window_not_responding: the text field did not answer the insertion in time; the text may still land, so run get-app-state before retrying"
}
$action = [pscustomobject]@{ path = "accessibility"; actionName = "EM_REPLACESEL"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "readback_unsupported" } }
$after = if ($null -eq $before) { $null } else { [OrcaDesktopWin32]::GetEditText($Edit.handle, $EditReadbackLimit) }
# Why: a text past the read limit can't show the whole result, so it neither proves nor disproves the insertion.
if ($null -eq $after -or $before.Length -ge $EditReadbackLimit -or $after.Length -ge $EditReadbackLimit) { return $action }
# Why: RichEdit counts a line break as one character in its selection but may read it back as CRLF.
$current = if ($Edit.rich) { ConvertTo-OrcaEditNewlines $before $true } else { $before }
$actual = if ($Edit.rich) { ConvertTo-OrcaEditNewlines $after $true } else { $after }
$expected = $null
if ($null -ne $selection -and [Math]::Max($selection[0], $selection[1]) -le $current.Length) {
$start = [Math]::Min($selection[0], $selection[1])
$end = [Math]::Max($selection[0], $selection[1])
$expected = $current.Substring(0, $start) + (ConvertTo-OrcaEditNewlines $Text $Edit.rich) + $current.Substring($end)
}
if ($null -ne $expected -and $actual -ceq $expected) {
$action.verification = [pscustomobject]@{ state = "verified"; property = "elementText" }
} elseif ($after -ceq $before) {
throw "text_insertion_unconfirmed: element $Index did not change after the insertion; run get-app-state to check the field before retrying"
} elseif ($null -ne $expected) {
$action.verification = [pscustomobject]@{ state = "unverified"; reason = "text_mismatch" }
}
$action
}
function Set-OrcaTextCaret($Element, [string]$Leading) {
# Why: a ValuePattern write resets WPF's caret to the start, so the next insertion would land there.
try {
# Why: UIA moves by caret positions, which keep a surrogate pair, a combining sequence and a CRLF together.
$positions = (New-Object Globalization.StringInfo $Leading).LengthInTextElements - ([regex]::Matches($Leading, "`r`n")).Count
$start = [Windows.Automation.Text.TextPatternRangeEndpoint]::Start
$end = [Windows.Automation.Text.TextPatternRangeEndpoint]::End
$leadingRange = $Element.GetCurrentPattern([Windows.Automation.TextPattern]::Pattern).DocumentRange.Clone()
$leadingRange.MoveEndpointByRange($end, $leadingRange, $start)
[void]$leadingRange.MoveEndpointByUnit($end, [Windows.Automation.Text.TextUnit]::Character, $positions)
# Why: a provider that counts positions differently would put the caret elsewhere, so only a confirmed prefix moves it.
if ($leadingRange.GetText(-1) -cne $Leading) { return }
$leadingRange.MoveEndpointByRange($start, $leadingRange, $end)
$leadingRange.Select()
} catch {}
}
function Get-OrcaTextSelectionOffsets($Element) {
try {
$text = $Element.GetCurrentPattern([Windows.Automation.TextPattern]::Pattern)
$ranges = @($text.GetSelection())
if ($ranges.Count -ne 1) { return $null }
$leading = $text.DocumentRange.Clone()
$leading.MoveEndpointByRange([Windows.Automation.Text.TextPatternRangeEndpoint]::End, $ranges[0], [Windows.Automation.Text.TextPatternRangeEndpoint]::Start)
$start = $leading.GetText(-1).Length
@($start, ($start + $ranges[0].GetText(-1).Length))
} catch {
$null
}
}
function Invoke-OrcaValueTextInsertion($Element, [int]$Index, [string]$Text) {
$pattern = try { $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern) } catch { $null }
if ($null -eq $pattern) { return $null }
if ($pattern.Current.IsReadOnly) { throw "element $Index text is not settable: the field is read-only" }
# Why: Chromium applies a value write as replaced inner text, which editors with their own model miss or revert.
if ((Get-OrcaProperty $Element "FrameworkId") -eq "Chrome") { return $null }
if (Test-OrcaSensitiveElement $Element) {
throw "element $Index text is not settable at its caret: it is a password field, whose text can't be read; use set-value to replace it"
}
$current = [string]$pattern.Current.Value
$selection = Get-OrcaTextSelectionOffsets $Element
# Why: without a readable selection the caret is unknown; appending matches macOS's default insertion point.
if ($null -eq $selection -or $selection[1] -gt $current.Length) { $selection = @($current.Length, $current.Length) }
$next = $current.Substring(0, $selection[0]) + $Text + $current.Substring($selection[1])
$pattern.SetValue($next)
# Why: Chromium applies the write asynchronously, so poll briefly like the range readback.
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
while ($true) {
$actual = try { [string]$pattern.Current.Value } catch { $null }
if ($actual -ceq $next -or $stopwatch.ElapsedMilliseconds -ge 500) { break }
Start-Sleep -Milliseconds 25
}
if ($actual -ceq $next) { Set-OrcaTextCaret $Element ($current.Substring(0, $selection[0]) + $Text) }
$verification = if ($actual -ceq $next) {
[pscustomobject]@{ state = "verified"; property = "elementText" }
} elseif ($null -eq $actual) {
[pscustomobject]@{ state = "unverified"; reason = "readback_unsupported" }
} elseif ($actual -ceq $current) {
[pscustomobject]@{ state = "unverified"; reason = "accessibility_action_unasserted" }
} else {
[pscustomobject]@{ state = "unverified"; reason = "text_mismatch" }
}
[pscustomobject]@{ path = "accessibility"; actionName = "setValue"; fallbackReason = $null; verification = $verification }
}
function Test-OrcaTextPatternReadOnly($Element) {
try {
$range = $Element.GetCurrentPattern([Windows.Automation.TextPattern]::Pattern).DocumentRange
$range.GetAttributeValue([Windows.Automation.TextPattern]::IsReadOnlyAttribute) -eq $true
} catch {
$false
}
}
function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, $RootElement, $Operation, [string]$Text) {
if ($null -eq $Element) {
throw "stale element: element $($Record.index) is gone; run get-app-state again and use a fresh element index"
}
$index = [int]$Record.index
# Why: edit messages bypass a disabled field's lock, which set-value's UI Automation write honours.
if (Test-OrcaElementDisabled $Element) { throw "element $index text is not settable: the field is disabled" }
$edit = Get-OrcaNativeEdit $Element $Process
if ($null -ne $edit) { return Invoke-OrcaEditTextInsertion $edit $index $Text }
# Why: list and tree items also take ValuePattern, where a write renames them rather than typing.
$controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
if ($controlType -notin @("ControlType.Edit", "ControlType.Document")) {
throw "element $index text is not settable: it is not a text field"
}
$action = Invoke-OrcaValueTextInsertion $Element $index $Text
if ($null -ne $action) { return $action }
# Why: a read-only document with no ValuePattern (WPF RichTextBox) says so only through TextPattern; keys would do nothing.
if (Test-OrcaTextPatternReadOnly $Element) { throw "element $index text is not settable: the field is read-only" }
if (-not [bool]$Operation.restoreWindow) {
throw (New-OrcaForegroundRefusal $Process "element $index accepts text only as keys, which go only to the window in front" "" "takes the user's keyboard")
}
$failure = Restore-OrcaWindow $Process $RootElement
Assert-OrcaKeyboardFocus ([IntPtr]$Process.MainWindowHandle) $Operation $failure $Process
try { $Element.SetFocus() } catch {
throw "window_not_focused: element $index did not take keyboard focus; no text was sent"
}
Send-OrcaText ([IntPtr]$Process.MainWindowHandle) $Text
[pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = "actionUnsupported"; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
}
function Get-OrcaRangeValueText($Element) {
try {
$pattern = $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern)
@@ -849,11 +1277,15 @@ function Test-OrcaPlainTextSubtree($Element, [int]$MaxDepth = 4) {
function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
$controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
$nativeWindowHandle = try { [int64]$Element.Current.NativeWindowHandle } catch { 0 }
$className = Get-OrcaProperty $Element "ClassName"
$patterns = @($Element.GetSupportedPatterns())
$value = Get-OrcaValueText $Element
if ([string]::IsNullOrEmpty($value) -and $patterns -contains [Windows.Automation.RangeValuePattern]::Pattern) {
$value = Get-OrcaRangeValueText $Element
}
if ([string]::IsNullOrEmpty($value) -and $nativeWindowHandle -ne 0 -and (Test-OrcaEditClass $className)) {
$value = Get-OrcaEditValueText ([IntPtr]$nativeWindowHandle)
}
[pscustomobject]@{
index = $Index
runtimeId = @(Get-OrcaRuntimeId $Element)
@@ -861,7 +1293,7 @@ function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
name = Get-OrcaProperty $Element "Name"
controlType = $controlType
localizedControlType = Get-OrcaProperty $Element "LocalizedControlType"
className = Get-OrcaProperty $Element "ClassName"
className = $className
value = $value
isSelected = Test-OrcaElementSelected $Element
isDisabled = Test-OrcaElementDisabled $Element
@@ -1211,6 +1643,104 @@ function Get-OrcaWindowList([string]$Query) {
}
}
function Resolve-OrcaLaunchExecutable([string]$Query) {
$needle = $Query.Trim()
if ([System.IO.Path]::IsPathRooted($needle)) {
if (Test-Path -LiteralPath $needle -PathType Leaf) { return $needle }
throw "appNotFound(`"$Query`")"
}
$exe = if ($needle.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) { $needle } else { "$needle.exe" }
$command = Get-Command -Name ([WildcardPattern]::Escape($exe)) -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
if ($null -ne $command) { return [string]$command.Source }
# Why: Start-Process finds apps such as chrome.exe through App Paths, which PATH lookup misses.
foreach ($hive in @("HKCU:", "HKLM:")) {
$entry = Get-ItemProperty -LiteralPath "$hive\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\$exe" -ErrorAction SilentlyContinue
if ($null -eq $entry -or [string]::IsNullOrWhiteSpace([string]$entry.'(default)')) { continue }
$path = [Environment]::ExpandEnvironmentVariables(([string]$entry.'(default)').Trim('"'))
if (Test-Path -LiteralPath $path -PathType Leaf) { return $path }
}
throw "appNotFound(`"$Query`")"
}
function Wait-OrcaLaunchedWindow([int]$ProcessId, [string]$ProcessName, [DateTime]$StartedAt, [int]$TimeoutMilliseconds) {
$deadline = [DateTime]::UtcNow.AddMilliseconds($TimeoutMilliseconds)
while ([DateTime]::UtcNow -lt $deadline) {
$process = Get-Process -Id $ProcessId -ErrorAction SilentlyContinue
if ($null -ne $process -and $process.MainWindowHandle -ne 0) { return $process }
if ($null -eq $process) {
# Why: stubs such as Windows 11's notepad.exe start the real app under the same name and exit.
$handoff = Get-OrcaWindowProcesses | Where-Object {
$_.ProcessName -ieq $ProcessName -and $_.StartTime -ge $StartedAt
} | Select-Object -First 1
if ($null -ne $handoff) { return $handoff }
}
Start-Sleep -Milliseconds 100
}
Get-Process -Id $ProcessId -ErrorAction SilentlyContinue
}
function New-OrcaLaunchResult($Process, [bool]$Launched) {
$owner = [uint32]0
[void][OrcaDesktopWin32]::GetWindowThreadProcessId([OrcaDesktopWin32]::GetForegroundWindow(), [ref]$owner)
# Why: measured, not taken from the request; an app may raise itself however it was started.
$delivery = if ($owner -eq $Process.Id) { "foreground" } else { "background" }
[pscustomobject]@{ ok = $true; app = New-OrcaAppRecord $Process; launched = $Launched; delivery = $delivery }
}
function Find-OrcaRunningLaunchTarget([string]$ProcessName, [string]$ExecutablePath = "") {
# Why: starting a running app again usually opens a second window over the user's work; tray-only apps count too.
$session = [System.Diagnostics.Process]::GetCurrentProcess().SessionId
$running = @(Get-Process -Name ([WildcardPattern]::Escape($ProcessName)) -ErrorAction SilentlyContinue | Where-Object {
# A resolved executable must be that file, not another copy with the same name.
$_.SessionId -eq $session -and ([string]::IsNullOrEmpty($ExecutablePath) -or [string]$_.Path -ieq $ExecutablePath)
}) | Sort-Object { $_.MainWindowHandle -eq 0 } | Select-Object -First 1
if ($null -ne $running) { Assert-OrcaProcessAllowed $running }
$running
}
function Invoke-OrcaLaunchApp($Operation) {
$query = ([string]$Operation.app).Trim()
if ([string]::IsNullOrWhiteSpace($query)) { throw 'appNotFound("")' }
# A bare name means any running copy; a path means that file, checked once it resolves.
if (-not [System.IO.Path]::IsPathRooted($query)) {
$queryName = if ($query.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) { $query.Substring(0, $query.Length - 4) } else { $query }
$running = Find-OrcaRunningLaunchTarget $queryName
if ($null -ne $running) { return New-OrcaLaunchResult $running $false }
}
$executable = [System.IO.Path]::GetFullPath((Resolve-OrcaLaunchExecutable $query))
# Why: an App Paths alias can name a differently named executable.
$running = Find-OrcaRunningLaunchTarget ([System.IO.Path]::GetFileNameWithoutExtension($executable)) $executable
if ($null -ne $running) { return New-OrcaLaunchResult $running $false }
$name = [System.IO.Path]::GetFileNameWithoutExtension($executable).ToLowerInvariant().Replace(" ", "")
foreach ($fragment in $BlockedAppFragments) {
if ($name.Contains($fragment.Replace(" ", ""))) { throw "appBlocked(`"$name`")" }
}
$activate = [bool]$Operation.activate
# Why: Orca runs this script with ELECTRON_RUN_AS_NODE=1, which would start an Electron app as bare Node.
Remove-Item Env:ELECTRON_RUN_AS_NODE -ErrorAction SilentlyContinue
$startedAt = (Get-Date).AddSeconds(-1)
try {
$processId = if ($activate) {
(Start-Process -FilePath $executable -PassThru).Id
} else {
[OrcaDesktopWin32]::LaunchWithoutForeground($executable, [System.IO.Path]::GetDirectoryName($executable))
}
} catch {
throw "app_launch_failed: could not launch '$query': $($_.Exception.Message)"
}
$process = Wait-OrcaLaunchedWindow $processId ([System.IO.Path]::GetFileNameWithoutExtension($executable)) $startedAt 10000
if ($null -eq $process) {
throw "app_launch_failed: the process started for '$query' exited; if it handed off to another process, find that one with list-apps"
}
$result = New-OrcaLaunchResult $process $true
# Why: an app that still took the foreground keeps the keyboard; burying it would hide where typing goes.
if (-not $activate -and $result.delivery -eq "background" -and $process.MainWindowHandle -ne 0) {
[void][OrcaDesktopWin32]::SendWindowToBack([IntPtr]$process.MainWindowHandle)
}
$result
}
function Get-OrcaHandshake {
[pscustomobject]@{
platform = "win32"
@@ -1218,13 +1748,13 @@ function Get-OrcaHandshake {
providerVersion = "1.0.0"
protocolVersion = 1
supports = [pscustomobject]@{
apps = [pscustomobject]@{ list = $true; bundleIds = $false; pids = $true }
apps = [pscustomobject]@{ list = $true; bundleIds = $false; pids = $true; launch = $true }
windows = [pscustomobject]@{ list = $true; targetById = $true; targetByIndex = $true; focus = $false; moveResize = $false }
observation = [pscustomobject]@{ screenshot = $true; annotatedScreenshot = $false; elementFrames = $true; ocr = $false; disabledState = $true }
actions = [pscustomobject]@{
click = $true
typeText = $true
typeTextElement = $false
typeTextElement = $true
pressKey = $true
hotkey = $true
pasteText = $true
@@ -1264,8 +1794,27 @@ function Find-OrcaElement($RootElement, $Record) {
$null
}
# Why: UI Automation's Invoke and Toggle focus a WinForms button first, which activates its background window;
# BM_CLICK runs the button's own click (and check box or radio state) with no focus change.
function Invoke-OrcaNativeButtonClick($Element, $Process) {
if ($null -eq $Process) { return $false }
# Why: measured on WinForms, whose buttons click on BM_CLICK directly; a user32 button simulates the mouse and
# sets focus, which can fail in an inactive dialog, so those keep the UI Automation pattern.
if ((Get-OrcaProperty $Element "ClassName") -notmatch "^WindowsForms\d+\.BUTTON\.") { return $false }
$button = Get-OrcaNativeControl $Element $Process "Button"
if ($null -eq $button) { return $false }
# Why: a group box shares the Button class but presses nothing; real buttons offer one of these patterns.
$patterns = @(try { $Element.GetSupportedPatterns() } catch { @() })
$pressable = @([Windows.Automation.InvokePattern]::Pattern, [Windows.Automation.TogglePattern]::Pattern, [Windows.Automation.SelectionItemPattern]::Pattern)
if (@($patterns | Where-Object { $pressable -contains $_ }).Count -eq 0) { return $false }
# A click that opens a modal dialog doesn't return until the dialog closes, so an unanswered BM_CLICK still counts as sent.
[void][OrcaDesktopWin32]::SendControlMessage($button, 0x00F5, [IntPtr]::Zero, [IntPtr]::Zero)
$true
}
# False only when the element offers none of these patterns; a pattern that fails is an error, never a cue for a real click.
function Invoke-OrcaPrimaryAction($Element, $Index) {
function Invoke-OrcaPrimaryAction($Element, $Index, $Process) {
if (Invoke-OrcaNativeButtonClick $Element $Process) { return $true }
foreach ($pattern in @(
[Windows.Automation.InvokePattern]::Pattern,
[Windows.Automation.SelectionItemPattern]::Pattern,
@@ -1339,7 +1888,87 @@ function Test-OrcaRangeValueMatches([string]$Actual, [double]$Expected) {
[Math]::Abs($actualNumber - $Expected) -le [Math]::Min(1e-6 * [Math]::Max(1.0, [Math]::Abs($Expected)), 1e-3)
}
function Set-OrcaElementValue($Element, [string]$Value) {
function Set-OrcaNativeComboValue($Element, $Process, [string]$Value) {
$combo = Get-OrcaNativeControl $Element $Process "ComboBox"
if ($null -eq $combo) { return $null }
# Why: an owner-drawn combo without CBS_HASSTRINGS keeps item data, not names, so its items can't be found by name.
$style = [OrcaDesktopWin32]::GetWindowLong($combo, -16)
if (($style -band 0x0030) -ne 0 -and ($style -band 0x0200) -eq 0) { return $null }
$count = [OrcaDesktopWin32]::SendControlMessage($combo, 0x0146, [IntPtr]::Zero, [IntPtr]::Zero)
if ($null -eq $count -or $count -le 0) { return $null }
# Why: CB_FINDSTRINGEXACT ignores case and wraps, so walk its hits once and keep those whose text matches exactly.
$exact = @()
$first = [OrcaDesktopWin32]::FindControlString($combo, 0x0158, -1, $Value)
$hit = $first
for ($step = 0; $null -ne $hit -and $hit -ge 0 -and $step -lt $count; $step++) {
$text = [OrcaDesktopWin32]::GetControlItemText($combo, 0x0149, 0x0148, [int]$hit)
if ($text -ceq $Value) { $exact += [int]$hit }
$hit = [OrcaDesktopWin32]::FindControlString($combo, 0x0158, [int]$hit, $Value)
# Why: a search or read the combo didn't answer could hide a second option with the same name.
if ($null -eq $text -or $null -eq $hit) {
throw "window_not_responding: the combo box did not answer its option search in time; nothing was selected"
}
if ($hit -eq $first) { break }
}
if ($exact.Count -eq 0) { return $null }
if ($exact.Count -gt 1) { throw "element value is not settable by name: more than one option is named '$Value'" }
# CB_SETCURSEL changes the selection without opening the list; CBN_SELCHANGE runs the app's own change handler.
Assert-OrcaWindowOwner $combo ([int]$Process.Id)
if ($null -eq [OrcaDesktopWin32]::SendControlMessage($combo, 0x014E, [IntPtr]$exact[0], [IntPtr]::Zero)) {
throw "window_not_responding: the combo box did not answer in time; the selection may still change, so run get-app-state before retrying"
}
$parent = [OrcaDesktopWin32]::GetParent($combo)
if ($parent -ne [IntPtr]::Zero) {
Assert-OrcaWindowOwner $parent ([int]$Process.Id)
$command = [IntPtr]((1 -shl 16) -bor ([OrcaDesktopWin32]::GetDlgCtrlID($combo) -band 0xFFFF))
[void][OrcaDesktopWin32]::SendControlMessage($parent, 0x0111, $command, $combo)
}
# Why: the change handler ran inside the send above, so the selection now is the one the app kept.
$selected = [OrcaDesktopWin32]::SendControlMessage($combo, 0x0147, [IntPtr]::Zero, [IntPtr]::Zero)
# Why: a handler that clears the selection leaves CB_ERR (-1), which contradicts the write rather than hiding it.
$actual = if ($null -eq $selected) { $null } elseif ($selected -lt 0) { "" } else { [OrcaDesktopWin32]::GetControlItemText($combo, 0x0149, 0x0148, [int]$selected) }
$verification = if ($actual -ceq $Value) {
[pscustomobject]@{ state = "verified"; property = "value" }
} elseif ($null -eq $actual) {
[pscustomobject]@{ state = "unverified"; reason = "readback_unsupported" }
} else {
[pscustomobject]@{ state = "unverified"; reason = "value_mismatch"; actualPreview = $actual }
}
[pscustomobject]@{ verification = $verification }
}
function Set-OrcaNativeEditValue($Element, $Process, [int]$Index, [string]$Value) {
$edit = Get-OrcaNativeEdit $Element $Process
if ($null -eq $edit) { return $null }
# Why: UI Automation's ValuePattern focuses the field first, which activates its background window; selecting
# everything and replacing it runs the edit's own change notification, which WM_SETTEXT skips for multiline edits.
Assert-OrcaWindowOwner $edit.handle $edit.processId
if ($null -eq [OrcaDesktopWin32]::SendControlMessage($edit.handle, 0x00B1, [IntPtr]::Zero, [IntPtr](-1))) {
throw "window_not_responding: the text field did not answer in time; nothing was written"
}
$action = Invoke-OrcaEditTextInsertion $edit $Index $Value -ReplaceAll
$verification = if ($action.verification.state -eq "verified") {
[pscustomobject]@{ state = "verified"; property = "value" }
} elseif ($action.verification.reason -eq "text_mismatch") {
[pscustomobject]@{ state = "unverified"; reason = "value_mismatch" }
} else {
$action.verification
}
[pscustomobject]@{ verification = $verification }
}
function Set-OrcaElementValue($Element, $Process, [int]$Index, [string]$Value) {
if ($null -ne $Process) {
# Why: these write native controls without focusing them; a disabled one is locked against the user, so it is refused.
$native = (Test-OrcaEditClass (Get-OrcaProperty $Element "ClassName")) -or (Test-OrcaNativeClass (Get-OrcaProperty $Element "ClassName") "ComboBox")
if ($native -and (Test-OrcaElementDisabled $Element)) {
throw "element_not_enabled: element $Index is disabled, so no input was sent; do what enables it first, then run get-app-state and retry once its line no longer lists disabled"
}
$combo = Set-OrcaNativeComboValue $Element $Process $Value
if ($null -ne $combo) { return $combo }
$edit = Set-OrcaNativeEditValue $Element $Process $Index $Value
if ($null -ne $edit) { return $edit }
}
try {
$pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
if (-not $pattern.Current.IsReadOnly) {
@@ -1482,8 +2111,44 @@ function Invoke-OrcaScrollPercent($Pattern, [bool]$Vertical, [bool]$Forward, [do
$null -ne $after -and $after -ne $Before
}
function Invoke-OrcaScrollPattern($Element, $RootElement, [string]$Direction, [double]$Pages) {
# Why: ScrollPattern on a list box focuses it first, which activates its background window; LB_SETTOPINDEX pages the
# list itself. A page is the rows the list shows.
function Invoke-OrcaNativeListScroll($Element, $RootElement, $Process, [string]$Direction, [double]$Pages) {
if ($null -eq $Process -or $Direction -notin @("up", "down")) { return $false }
$rootId = Get-OrcaRuntimeId $RootElement
$current = $Element
$list = $null
# Why: an item scrolls with its list box, the window a wheel at its center would scroll.
while ($null -ne $current -and $null -eq $list) {
$list = Get-OrcaNativeControl $current $Process "ListBox"
if (Test-OrcaSameRuntimeId (Get-OrcaRuntimeId $current) $rootId) { break }
$current = try { [Windows.Automation.TreeWalker]::RawViewWalker.GetParent($current) } catch { $null }
}
if ($null -eq $list) { return $false }
# Why: rows of an LBS_OWNERDRAWVARIABLE list differ in height, so a page can't be counted in rows, and an
# LBS_MULTICOLUMN list pages sideways.
if (([OrcaDesktopWin32]::GetWindowLong($list, -16) -band 0x0220) -ne 0) { return $false }
$top = [OrcaDesktopWin32]::SendControlMessage($list, 0x018E, [IntPtr]::Zero, [IntPtr]::Zero)
$count = [OrcaDesktopWin32]::SendControlMessage($list, 0x018B, [IntPtr]::Zero, [IntPtr]::Zero)
$itemHeight = [OrcaDesktopWin32]::SendControlMessage($list, 0x01A1, [IntPtr]::Zero, [IntPtr]::Zero)
if ($null -eq $top -or $null -eq $count -or $null -eq $itemHeight -or $itemHeight -le 0 -or $count -le 0) { return $false }
$rows = [Math]::Max(1, [Math]::Floor([OrcaDesktopWin32]::GetClientHeight($list) / $itemHeight))
$step = [int][Math]::Max(1, [Math]::Round($rows * $Pages))
$target = if ($Direction -eq "down") { $top + $step } else { $top - $step }
# Why: a list that shows every row leaves the scroll to an enclosing ScrollPattern.
if ($count -le $rows) { return $false }
$target = [Math]::Max(0, [Math]::Min($count - $rows, $target))
# Why: at that end already there is nothing left to move, as ScrollPattern reports at the end of its content.
if ($target -eq $top) { return $true }
if ($null -eq [OrcaDesktopWin32]::SendControlMessage($list, 0x0197, [IntPtr][int]$target, [IntPtr]::Zero)) {
throw "window_not_responding: the list did not answer in time; it may still scroll, so run get-app-state before retrying"
}
$true
}
function Invoke-OrcaScrollPattern($Element, $RootElement, $Process, [string]$Direction, [double]$Pages) {
if ($null -eq $Element) { return $false }
if (Invoke-OrcaNativeListScroll $Element $RootElement $Process $Direction $Pages) { return $true }
$vertical = $Direction -in @("up", "down")
$forward = $Direction -in @("down", "right")
$pattern = Get-OrcaScrollPattern $Element $RootElement $vertical
@@ -1621,19 +2286,10 @@ function Send-OrcaDrag($From, $To) {
}
function Send-OrcaText([IntPtr]$WindowHandle, [string]$Text) {
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
$hasNonAscii = $false
foreach ($character in $Text.ToCharArray()) {
if ([int][char]$character -gt 0x7F) { $hasNonAscii = $true; break }
Assert-OrcaStillFocused $WindowHandle
if (-not [OrcaDesktopWin32]::SendUnicodeText($Text)) {
throw "SendInput did not deliver the text; inspect the field before retrying"
}
if ($hasNonAscii) {
foreach ($character in $Text.ToCharArray()) {
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.Char, [IntPtr][int][char]$character, [IntPtr]::Zero)
Start-Sleep -Milliseconds 8
}
return
}
[System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysText $Text))
}
function Get-OrcaVirtualKey([string]$Key) {
@@ -1648,8 +2304,15 @@ function Get-OrcaVirtualKey([string]$Key) {
throw "Unsupported key: $Key"
}
function Assert-OrcaStillFocused([IntPtr]$WindowHandle) {
# Why: keys go to whatever is in front, so a user switching windows since the focus check must not receive them.
if (-not (Test-OrcaWindowFocused $WindowHandle)) {
throw "window_not_focused: the target window lost focus before the keys were sent; nothing was sent"
}
}
function Send-OrcaKey([IntPtr]$WindowHandle, [string]$Key) {
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
Assert-OrcaStillFocused $WindowHandle
[System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysKey $Key))
}
@@ -1690,7 +2353,7 @@ function ConvertTo-OrcaSendKeysHotkey([string]$KeySpec) {
function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) {
$keys = ConvertTo-OrcaSendKeysHotkey $KeySpec
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
Assert-OrcaStillFocused $WindowHandle
[System.Windows.Forms.SendKeys]::SendWait($keys)
}
@@ -1771,6 +2434,9 @@ function Invoke-OrcaOperation($Operation) {
$list = Get-OrcaWindowList $Operation.app
return [pscustomobject]@{ ok = $true; app = $list.app; windows = @($list.windows) }
}
if ($Operation.tool -eq "launch_app") {
return Invoke-OrcaLaunchApp $Operation
}
if ($Operation.tool -eq "get_app_state") {
return [pscustomobject]@{ ok = $true; snapshot = New-OrcaSnapshot $Operation.app $includeScreenshot $Operation.windowId $Operation.windowIndex ([bool]$Operation.restoreWindow) $Operation.root }
}
@@ -1785,9 +2451,10 @@ function Invoke-OrcaOperation($Operation) {
$fromElement = Find-OrcaElement $root $Operation.fromElement
$toElement = Find-OrcaElement $root $Operation.toElement
$handle = [IntPtr]$process.MainWindowHandle
if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text")) {
$typesIntoElement = $Operation.tool -eq "type_text" -and $null -ne $Operation.element
if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text") -and -not $typesIntoElement) {
$restoreFailure = if ($restoreWindow) { Restore-OrcaWindow $process $root } else { $null }
Assert-OrcaKeyboardFocus $handle $Operation $restoreFailure
Assert-OrcaKeyboardFocus $handle $Operation $restoreFailure $process
}
$foregroundBefore = [OrcaDesktopWin32]::GetForegroundWindow()
# Why: a stale or frameless target must fail before any refusal or raise; a minimized window has no frames until restored.
@@ -1812,7 +2479,7 @@ function Invoke-OrcaOperation($Operation) {
else { $null }
$alternative = if ($null -eq $Operation.element) { "Click an element by --element-index instead when it lists Invoke, Select or Toggle." } else { "" }
if ($null -eq $pointerNeed) {
if (Invoke-OrcaPrimaryAction $element $Operation.element.index) {
if (Invoke-OrcaPrimaryAction $element $Operation.element.index $process) {
$action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }
break
}
@@ -1860,7 +2527,7 @@ function Invoke-OrcaOperation($Operation) {
$at = Get-OrcaPointerPoint $null $null $Operation.x $Operation.y "x" "y" $windowFrame
$scrollTarget = Find-OrcaScrollContainerAt $root $at.x $at.y ($Operation.direction -in @("up", "down"))
}
if (Invoke-OrcaScrollPattern $scrollTarget $root $Operation.direction $pages) {
if (Invoke-OrcaScrollPattern $scrollTarget $root $process $Operation.direction $pages) {
$action = [pscustomobject]@{ path = "accessibility"; actionName = "scroll"; fallbackReason = $null }
break
}
@@ -1904,8 +2571,13 @@ function Invoke-OrcaOperation($Operation) {
$action = [pscustomobject]@{ path = "synthetic"; actionName = "drag"; fallbackReason = $null }
}
"type_text" {
Send-OrcaText $handle (Get-OrcaRequiredString $Operation.text "text")
$action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
$text = Get-OrcaRequiredString $Operation.text "text"
if ($typesIntoElement) {
$action = Invoke-OrcaElementTextInsertion $element $Operation.element $process $root $Operation $text
} else {
Send-OrcaText $handle $text
$action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
}
}
"press_key" {
Send-OrcaKey $handle (Get-OrcaRequiredString $Operation.key "key")
@@ -1920,7 +2592,7 @@ function Invoke-OrcaOperation($Operation) {
$action = [pscustomobject]@{ path = "clipboard"; actionName = "paste"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "clipboard_paste" } }
}
"set_value" {
$written = if ($null -ne $element) { Set-OrcaElementValue $element ([string]$Operation.value) } else { $null }
$written = if ($null -ne $element) { Set-OrcaElementValue $element $process ([int]$Operation.element.index) ([string]$Operation.value) } else { $null }
if ($null -eq $written) {
throw "element value is not settable"
}
+11
View File
@@ -22,5 +22,16 @@
"command": ["node", "config/scripts/typecheck/language-server/agent-typescript-lsp.mjs"],
"extensions": [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"]
}
},
"mcp": {
"orca-component-fixtures": {
"type": "local",
"command": [
"node",
"--input-type=module",
"-e",
"import { existsSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; let dir = process.cwd(); for (;;) { const file = join(dir, 'config/scripts/component-fixtures/fixtures-mcp-server.mjs'); if (existsSync(file)) { await import(pathToFileURL(file).href); break; } const parent = dirname(dir); if (parent === dir) throw new Error('Cannot find the component fixtures server'); dir = parent; }"
]
}
}
}
+3 -3
View File
@@ -209,7 +209,7 @@ patchedDependencies:
'@xterm/xterm@6.1.0-beta.336': 4b7fbfe7a7f626843ab88535cc6e2181872ea7c107876b15406a295182bc3c2c
glob@13.0.6: 163edae2cab27bf7a8cf791c9a0ee368b596bfef898e490b3b77e7e44a8f0223
i18next-cli@1.74.2: 2ee32eef742615cd133e1a6940f027deb3723a8430bb7953bba3a6769b0aac47
lint-staged@16.4.0: 7333b3837f80a7fbd045964db6d76ba4fc118e49134bdbabb00585b6b7b60673
lint-staged@16.4.0: b3b398fa76585526ea579cbb07949ce69c2456afb5ab0b49e10de2de4ea76f6d
minimatch@10.2.5: 4324868e550a69eee9addc4801001bee415e98339c9b2b88d6ab4fc6028741c6
minimatch@10.2.6: 4324868e550a69eee9addc4801001bee415e98339c9b2b88d6ab4fc6028741c6
minimatch@9.0.9: c356feccb9147b4d6a1cc72b10270defcb95d14cb42ee74ddfe7219341417161
@@ -544,7 +544,7 @@ importers:
version: 0.18.10
lint-staged:
specifier: ^16.4.0
version: 16.4.0(patch_hash=7333b3837f80a7fbd045964db6d76ba4fc118e49134bdbabb00585b6b7b60673)
version: 16.4.0(patch_hash=b3b398fa76585526ea579cbb07949ce69c2456afb5ab0b49e10de2de4ea76f6d)
lowlight:
specifier: ^3.3.0
version: 3.3.0
@@ -13410,7 +13410,7 @@ snapshots:
linkifyjs@4.3.3: {}
lint-staged@16.4.0(patch_hash=7333b3837f80a7fbd045964db6d76ba4fc118e49134bdbabb00585b6b7b60673):
lint-staged@16.4.0(patch_hash=b3b398fa76585526ea579cbb07949ce69c2456afb5ab0b49e10de2de4ea76f6d):
dependencies:
commander: 14.0.3
listr2: 9.0.5
+2 -2
View File
@@ -64,7 +64,7 @@ ORCA exec --page <browserPageId> --command "get text @e3" --json
Browser rules:
- Actions (`click`, `fill`, `type`, `keypress`, `select`, `check`, `hover`, `scroll`, `goto`, `back`, `reload`, `mouse` and the rest) wait briefly for the page to settle and then report what they caused in `result.effects` (text mode prints it under the action): `url`/`title` when they changed, `newTabs` with their `browserPageId`, blocked or opened `popups`, `downloads` started, `consoleErrors`, a `passkeyRequest` still waiting, and `page`: `"unchanged"`, or the snapshot lines the action added or replaced, with their refs (`full: true` means the whole new page). `loading: true` means a navigation was still running: `wait --load load`, then `snapshot`. No `page` means the page could not be read: run `snapshot`. `settled: false` means the page was still changing when it was read. An action that opens a JavaScript dialog fails with `browser_dialog_opened` instead. One that opens a file picker fails with `browser_file_chooser_opened`: Orca closes the picker so the system dialog never covers the user's screen; use `ORCA upload` on the file input or its button instead.
- Actions (`click`, `fill`, `type`, `keypress`, `select`, `check`, `hover`, `scroll`, `goto`, `back`, `reload`, `mouse` and the rest) wait briefly for the page to settle and then report what they caused in `result.effects` (text mode prints it under the action): `url`/`title` when they changed, `newTabs` with their `browserPageId`, blocked or opened `popups`, `downloads` started, `consoleErrors`, a `passkeyRequest` still waiting, `blockedPrint` when Orca refused the page's system print dialog (run `ORCA pdf` to save the page instead), and `page`: `"unchanged"`, or the snapshot lines the action added or replaced, with their refs (`full: true` means the whole new page). `loading: true` means a navigation was still running: `wait --load load`, then `snapshot`. No `page` means the page could not be read: run `snapshot`. `settled: false` means the page was still changing when it was read. An action that opens a JavaScript dialog fails with `browser_dialog_opened` instead. One that opens a file picker fails with `browser_file_chooser_opened`: Orca closes the picker so the system dialog never covers the user's screen; use `ORCA upload` on the file input or its button instead.
- Snapshot again only when `page.truncated` is true, after `wait`, after a tab switch, on a `browser_stale_ref`, or when a result has no `effects` (older Orca, or a server-hosted page).
- Pass `--expect-url <url>` with `--page` when acting on the wrong resource would matter, such as writing or sending in one conversation of a chat app. Use the exact absolute URL from `result.tab.url` in `ORCA tab current --json`, or `url` from a `snapshot --json` or `goto`. Path, query and fragment must all match, so a page that rewrites its URL while you work refuses until you read the URL again. If the page's URL differs, or a navigation to another URL is under way, the command fails with `browser_expected_url_mismatch`, and `error.data` holds `expectedUrl`, `actualUrl`, `browserPageId` and `mayHaveStarted`. `mayHaveStarted: false` means nothing ran; `true` means a step of a longer command (a typed run, a `select-all`) may have happened, so look at the page before retrying. Success adds `browserPageId` and `checkedUrl`, the URL last checked; the action itself may have navigated since, and `effects.url` says where. The check runs right before the action and again before each later step, but it is not a lock: a route change in the instant after a check can still receive the action. It compares the top-level page only, so it says nothing about an iframe or an app that shows different content at one URL; read the page for those. `exec` refuses the flag, because one raw batch can navigate and retarget. A runtime, client or desktop that cannot check the URL refuses with `incompatible_runtime` or `browser_expect_url_unsupported` rather than acting unguarded.
- Refs like `@e1` are assigned by `snapshot`, scoped to one tab, and invalidated by navigation or tab switch.
@@ -74,7 +74,7 @@ Browser rules:
- When the user says they left notes or comments on a page, read them with `ORCA tab annotations --page <browserPageId>` (the current tab without `--page`). It prints the notes they haven't sent, the same text the tab's Copy button gives, and `--json` returns them in `result.annotations`. Notes the user already sent arrive in your chat instead.
- Passkey, fingerprint, Face ID, Windows Hello and security-key prompts can't be completed by you. When `snapshot` or an action's `effects` has `passkeyRequest`, the page is waiting on one from that origin and will wait for minutes. Ask the user to finish the sign-in, or run `ORCA dialog dismiss --page <browserPageId>` to cancel it so the page offers another way to sign in.
- A popup window a page opens (sign-in, payment) is listed in `ORCA tab list --json` as a `popup-<n>` page with `openerBrowserPageId`. If a click seems to do nothing, check `tab list`, then drive the popup with `--page popup-<n>`. When it closes, continue on the opener page.
- Browser commands default to the current worktree. `--page` reaches the page in any worktree unless you also pass `--worktree <selector>`. Use `--worktree all` only intentionally.
- Browser commands default to the worktree you are in: your cwd's worktree, else your Orca terminal's. Outside both (a shell or IDE Orca did not open, with a cwd outside any worktree), and on every `--environment` command, always pass `--page <browserPageId>` or `--worktree <selector>` (not `all`); find pages with `ORCA tab list --json`. Without either, a local command acts on the tab the user is looking at, and an `--environment` command is refused. `--page` reaches the page in any worktree unless you also pass `--worktree <selector>`. Use `--worktree all` only intentionally.
- `tab list` shows every tab in the tab bar. After a restart, tabs the user has not opened report `loaded: false`; `tab switch` or any `--page` command loads one in the background without changing the user's visible tab.
- `tab create --split <right|left|down|up>` opens the page in a new pane beside the calling terminal instead of as another tab; focus stays in the terminal. When a browser pane is already open beside it, the page opens as that pane's visible tab instead of adding another pane.
- Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command "tab ..."`, so Orca keeps UI state synchronized.
+7 -6
View File
@@ -87,6 +87,7 @@ For apps with multiple windows or ambiguous titles, run `list-windows` first. Pr
ORCA computer permissions --json
ORCA computer capabilities --json
ORCA computer list-apps --json
ORCA computer launch-app <app> --json
ORCA computer list-windows --app <app> --json
ORCA computer get-app-state --app <app> --json
ORCA computer wait --app <app> (--text <text> | --enabled <text> | --window-title <text>) [--timeout <ms>] --json
@@ -129,7 +130,7 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.
- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.
- `type-text --element-index <index>` inserts at that text field's caret, replacing its selection, without focusing it or bringing its window forward, and reads the text back. It works on macOS and Linux; where `capabilities` lacks `actions.typeTextElement` it fails with `unsupported_capability`; then click the field, confirm it is the focused text receiver, and type without the index. A `value_not_settable` error means that element does not accept text insertion.
- `type-text --element-index <index>` inserts at that text field's caret, replacing its selection, without focusing it or bringing its window forward, and reads the text back. It works on macOS, Linux and Windows; where `capabilities` lacks `actions.typeTextElement` it fails with `unsupported_capability`; then click the field, confirm it is the focused text receiver, and type without the index. A `value_not_settable` error means that element does not accept text insertion.
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
@@ -146,22 +147,22 @@ These need the foreground, so ask the user first:
- An action fails with `background_unavailable` and no semantic action (`set-value`, `perform-secondary-action`, `type-text --element-index`) can do the job.
- A background action reports success but the UI does not change, and no semantic action can do the job.
- On macOS, drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.
- On Windows, keyboard input (`press-key`, `hotkey`, `paste-text` and `type-text`) needs the target window focused; otherwise it is refused with `window_not_focused` and nothing is sent. For editable fields use `set-value`.
- On Windows, keyboard input (`press-key`, `hotkey`, `paste-text` and `type-text` without `--element-index`) needs the target window in front; otherwise it is refused with `background_unavailable` and nothing is sent. Enter text with `type-text --element-index` or `set-value`.
- On Linux and Windows, actions that need the real pointer, and on Linux keys sent while the target window is not active (Return in a focused GTK text field works). They are refused with `background_unavailable`; see below.
On macOS, a window that is minimized, hidden (Cmd+H) or on another Space cannot be read, so `get-app-state` fails with `window_not_found`. Ask the user to bring it on screen.
On macOS, `list-windows` and `get-app-state` also read windows that are minimized, hidden (Cmd+H) or on another Space, full-screen ones included; `isMinimized` and `isOffscreen` say which. Element actions work there (`click --element-index`, `set-value`, `type-text --element-index`, secondary actions), while coordinate clicks, scrolls, drags, keys and pastes are refused with `background_unavailable` and nothing is sent. Minimized and hidden windows have no screenshot. A window on another Space that nothing has read yet fails with `window_not_found`; ask the user to show it once.
On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks or drags that start on popup menus, menu buttons, native combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, native combo box or `<select>`, set the option with `set-value` on that element.
On macOS, every action without `--restore-window` runs in the background: the target app is not brought forward, the user's frontmost app and key window stay put, and the real pointer does not move. `capabilities` reports this as `supports.input.background`.
On Linux, element clicks through the element's accessibility action, `set-value`, `type-text --element-index` and secondary actions run without raising the window or moving the pointer, and Return in a focused GTK text field runs its activate action. Anything that needs the real pointer (clicks with no accessibility action, right, middle, multiple or modified clicks, scrolling, drags) or the focused window (other keys, typing with no focused editable field) is refused with `background_unavailable` and nothing is sent; so is a click that would open a menu, a keyboard-grabbing popup or a dialog (a GTK 4 or Qt label ending in `…`). Ask the user before retrying with `--restore-window`, which raises only the target window, refuses a window on another workspace and puts the cursor back. On Wayland, synthetic pointer and key input never reaches native windows, so it is refused even with the flag. `action.delivery` is measured: `foreground` when the input went to the window in front or the target came forward.
On Linux, element clicks through the element's accessibility action (or, for a list row with none, its list's selection), `set-value`, `type-text --element-index` and secondary actions run without raising the window or moving the pointer, and Return in a focused GTK text field runs its activate action. Scrolling moves the nearest scroll bar under the element or point by whole viewports, or pages a web document, also in the background. Anything that needs the real pointer (other clicks with no accessibility action, right, middle, multiple or modified clicks, scrolls in a view with no scroll bar or document, drags) or the focused window (other keys, typing with no focused editable field) is refused with `background_unavailable` and nothing is sent; so is a click that would open a menu, a keyboard-grabbing popup or a dialog (a GTK 4 or Qt label ending in `…`). Ask the user before retrying with `--restore-window`, which raises only the target window, refuses a window on another workspace and puts the cursor back. On Wayland, synthetic pointer and key input never reaches native windows, so it is refused even with the flag. `action.delivery` is measured: `foreground` when the input went to the window in front or the target came forward.
On Windows, these actions go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), scrolls of an element or of the scrollable area under `--x`/`--y` (fractional `--pages` too), and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Clicks by coordinates, right, middle, modified and multi-clicks, clicks on elements with no `Invoke`, `Select` or `Toggle` action, scrolls over nothing scrollable, and drags need the real mouse, so they are refused with `background_unavailable` and nothing is sent. With `--restore-window` (after asking the user), only the target window comes forward, never one on another virtual desktop, and the pointer goes back where it was.
On Windows, these actions go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), `type-text --element-index`, scrolls of an element or of the scrollable area under `--x`/`--y` (fractional `--pages` too), and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). WinForms buttons, check boxes and radio buttons, and native text boxes, combo boxes and list boxes (user32 and WinForms), are driven through their own window messages, which leave the window where it is. User32 buttons, check boxes and radio buttons use UI Automation, so they can bring their window forward. Other apps can bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Clicks by coordinates, right, middle, modified and multi-clicks, clicks on elements with no `Invoke`, `Select` or `Toggle` action, scrolls over nothing scrollable, and drags need the real mouse, so they are refused with `background_unavailable` and nothing is sent. `type-text` without `--element-index`, `press-key`, `hotkey` and `paste-text` send keys, which need the window in front, so they are refused the same way; enter text with `type-text --element-index` or `set-value` instead. With `--restore-window` (after asking the user), only the target window comes forward, never one on another virtual desktop, and the pointer goes back where it was.
## Starting Apps
`orca computer` does not launch apps. On macOS, start one without bringing it forward with `open -g -a <App>` (or `open -g -b <bundle-id>`); plain `open -a` brings it to the front. Some apps, such as many Electron apps, still bring their own window forward when it opens. On Windows and Linux a newly started app usually comes to the front, so ask the user before starting one.
Start a closed app with `ORCA computer launch-app <app>`, not `open -a` or `Start-Process`: those bring the app to the front and take the user's keyboard. `launch-app` asks macOS and Windows to start it in the background and leaves an app that is already running alone. Some apps, such as many Electron apps, still bring their own window forward when they open, so read `delivery`: if it says `foreground`, tell the user the app came to the front. Pass `--activate` only when the user asked for the app to come forward. On Linux it fails with `unsupported_capability`, because a newly started app usually comes to the front there; ask the user before starting one another way.
## Screenshots
@@ -30,17 +30,15 @@ describe('command-scoped repeatable flags', () => {
expect(parsed.get('path')).toBe('/c')
})
it.each([
['--agent', '--agent=b'],
['--agent=a', '--agent', '--agent=b'],
['--agent=a', '--agent'],
['--agent', '--agent']
])('rejects a missing occurrence even among valid values: %j', (...options) => {
const parsed = parseArgs(['example', 'search', ...options], paths, specs)
expect(() => validateCommandAndFlags(specs, parsed)).toThrow('Flag --agent requires a value.')
})
it.each([['--agent=a', '--agent', '--agent=b']])(
'rejects a missing occurrence even among valid values: %j',
(...options) => {
const parsed = parseArgs(['example', 'search', ...options], paths, specs)
expect(() => validateCommandAndFlags(specs, parsed)).toThrow('Flag --agent requires a value.')
}
)
it.each(['label', 'skill'])('also rejects erased missing legacy --%s values', (flag) => {
it.each(['label'])('also rejects erased missing legacy --%s values', (flag) => {
const spec = { ...leaf, allowedFlags: [flag], repeatableFlags: [] }
const parsed = parseArgs(['ex', 'find', `--${flag}=a`, `--${flag}`, `--${flag}=b`], paths, [
spec
+3 -93
View File
@@ -1,12 +1,7 @@
import { describe, expect, it } from 'vitest'
import type { CommandSpec } from '../discovery/command-spec'
import {
findCommandSpec,
normalizeCommandPositionals,
parseArgs,
validateCommandAndFlags
} from './args'
import { normalizeCommandPositionals, parseArgs, validateCommandAndFlags } from './args'
describe('parseArgs', () => {
it('keeps an empty string as a flag value', () => {
@@ -17,39 +12,12 @@ describe('parseArgs', () => {
expect(parsed.flags.get('json')).toBe(true)
})
it('accepts a flag value that starts with -- via the = form', () => {
const parsed = parseArgs(['terminal', 'send', '--text=--help'])
expect(parsed.commandPath).toEqual(['terminal', 'send'])
expect(parsed.flags.get('text')).toBe('--help')
})
it('splits --flag=value on the first = so values may contain =', () => {
const parsed = parseArgs(['set', 'cookie', '--value=a=b=c'])
expect(parsed.flags.get('value')).toBe('a=b=c')
})
it('treats --flag= as an empty string value', () => {
const parsed = parseArgs(['--value='])
expect(parsed.flags.get('value')).toBe('')
})
it('does not consume a command token after an unknown flag', () => {
const parsed = parseArgs(['--jso', 'worktree', 'list'], [['worktree', 'list']])
expect(parsed.commandPath).toEqual(['worktree', 'list'])
expect(parsed.flags.get('jso')).toBe(true)
})
it('does not consume a command token after a flag valid on another command', () => {
const parsed = parseArgs(['--workspace', 'worktree', 'list'], [['worktree', 'list']])
expect(parsed.commandPath).toEqual(['worktree', 'list'])
expect(parsed.flags.get('workspace')).toBe(true)
})
it('recognizes a command path with a boolean flag between its segments', () => {
const parsed = parseArgs(['--jso', 'worktree', '--json', 'list'], [['worktree', 'list']])
@@ -69,30 +37,6 @@ describe('parseArgs', () => {
expect(parsed.flags.get('repo')).toBe('id:abc')
})
it('still consumes a pre-command flag value before a later command path', () => {
const parsed = parseArgs(['--workspace', 'team-1', 'linear', 'list'], [['linear', 'list']])
expect(parsed.commandPath).toEqual(['linear', 'list'])
expect(parsed.flags.get('workspace')).toBe('team-1')
})
it('preserves existing pre-command selector values', () => {
const parsed = parseArgs(['--repo', 'id:abc', 'worktree', 'list'], [['worktree', 'list']])
expect(parsed.commandPath).toEqual(['worktree', 'list'])
expect(parsed.flags.get('repo')).toBe('id:abc')
})
it('preserves a project selector before the project command', () => {
const parsed = parseArgs(
['--project', 'github:stablyai/orca', 'project', 'setups'],
[['project', 'setups']]
)
expect(parsed.commandPath).toEqual(['project', 'setups'])
expect(parsed.flags.get('project')).toBe('github:stablyai/orca')
})
it('preserves a selector value that is also a registered command', () => {
const parsed = parseArgs(
['--environment', 'status', 'worktree', 'list'],
@@ -113,16 +57,6 @@ describe('parseArgs', () => {
expect(parsed.flags.get('environment')).toBe('worktree')
})
it.each([
['--project', 'project', 'project', 'setups'],
['--project=project', 'project', 'setups']
])('preserves a command-named project selector in %j', (...args) => {
const parsed = parseArgs(args, [['project', 'setups']])
expect(parsed.commandPath).toEqual(['project', 'setups'])
expect(parsed.flags.get('project')).toBe('project')
})
it('normalizes partial positionals without conflicting later flag-supplied args', () => {
const parsed = normalizeCommandPositionals(
[
@@ -151,12 +85,6 @@ describe('parseArgs', () => {
expect(parsed.flags.get('permission')).toBe('android.permission.CAMERA')
expect(parsed.positionalFlagConflicts).toEqual([])
})
it('does not apply repeated flag encoding to ordinary string flags', () => {
const parsed = parseArgs(['linear', 'list', '--workspace', 'old', '--workspace', 'new'])
expect(parsed.flags.get('workspace')).toBe('new')
})
})
describe('command aliases', () => {
@@ -170,20 +98,9 @@ describe('command aliases', () => {
summary: 'Remove a worktree',
usage: 'orca worktree rm --worktree <selector>',
allowedFlags: ['worktree', 'force']
},
{
path: ['repo', 'show'],
summary: 'Show a repo',
usage: 'orca repo show --repo <selector>',
allowedFlags: ['repo'],
positionalArgs: ['repo']
}
]
it('resolves an aliased path to the canonical spec', () => {
expect(findCommandSpec(specs, ['worktree', 'remove'])?.path).toEqual(['worktree', 'rm'])
})
it('canonicalizes an aliased command path during normalization', () => {
const normalized = normalizeCommandPositionals(
specs,
@@ -193,13 +110,6 @@ describe('command aliases', () => {
expect(normalized.commandPath).toEqual(['worktree', 'rm'])
expect(normalized.flags.get('force')).toBe(true)
})
it('binds a trailing positional after canonicalizing the base path', () => {
const normalized = normalizeCommandPositionals(specs, parseArgs(['repo', 'show', 'id:abc']))
expect(normalized.commandPath).toEqual(['repo', 'show'])
expect(normalized.flags.get('repo')).toBe('id:abc')
})
})
describe('validateCommandAndFlags', () => {
@@ -212,13 +122,13 @@ describe('validateCommandAndFlags', () => {
}
]
it.each(['environment', 'pairing-code'])('rejects --%s without a value', (flag) => {
it.each(['environment'])('rejects --%s without a value', (flag) => {
const parsed = parseArgs([`--${flag}`, 'demo'], [['demo']])
expect(() => validateCommandAndFlags(specs, parsed)).toThrow(`Flag --${flag} requires a value.`)
})
it.each(['environment', 'pairing-code'])('rejects an empty --%s= value', (flag) => {
it.each(['environment'])('rejects an empty --%s= value', (flag) => {
const parsed = parseArgs(['demo', `--${flag}=`])
expect(() => validateCommandAndFlags(specs, parsed)).toThrow(`Flag --${flag} requires a value.`)
+27 -1
View File
@@ -4,6 +4,7 @@ import { getOptionalStringFlag, getPresentStringFlag } from '../arguments/flags'
import type { RuntimeClient } from '../runtime/client'
import { RuntimeClientError } from '../runtime/types'
import { assertLocalCwdWorktreeSelector } from '../hosts/local-environment-selector'
import { quoteCliCommandArgument } from '../arguments/shell-command-quote'
import {
getBrowserWorktreeSelector,
normalizeWorktreeSelectorForCaller,
@@ -46,6 +47,31 @@ async function getEnforceableExpectUrl(
return expectUrl
}
/** A paired server's fallback is the tab a person is looking at, so a remote command must name its target. */
export async function getScopedBrowserWorktreeSelector(
flags: Map<string, string | boolean>,
cwd: string,
client: RuntimeClient
): Promise<string | undefined> {
const worktree = await getBrowserWorktreeSelector(flags, cwd, client)
if (worktree === undefined && client.isRemote) {
const environment = client.environmentSelector
const remote = environment
? ` --environment ${quoteCliCommandArgument(environment)}`
: ' --pairing-code <the same code>'
throw new RuntimeClientError(
'invalid_argument',
'Remote browser commands need --page <browserPageId> or a server-side --worktree <selector> (not `all`); without one they would act on the tab the user is looking at. Nothing was sent.',
{
nextSteps: [
`Find the page with \`orca tab list${remote} --json\` and pass its browserPageId as --page.`
]
}
)
}
return worktree
}
async function getBrowserPageTarget(
flags: Map<string, string | boolean>,
cwd: string,
@@ -53,7 +79,7 @@ async function getBrowserPageTarget(
): Promise<BrowserCliTarget> {
const page = getOptionalStringFlag(flags, 'page')
if (!page) {
return { worktree: await getBrowserWorktreeSelector(flags, cwd, client) }
return { worktree: await getScopedBrowserWorktreeSelector(flags, cwd, client) }
}
const explicitWorktree = getOptionalStringFlag(flags, 'worktree')
@@ -0,0 +1,86 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const { callMock, runtimeClientConstructorMock, serveOrcaAppMock, getDefaultUserDataPathMock } =
vi.hoisted(() => ({
callMock: vi.fn(),
runtimeClientConstructorMock: vi.fn(),
serveOrcaAppMock: vi.fn(),
getDefaultUserDataPathMock: vi.fn(() => '/tmp/orca-user-data')
}))
vi.mock('../runtime/client', async () => {
const { createRuntimeClientModuleMock } = await import('../index-test-harness.js')
return createRuntimeClientModuleMock({
callMock,
runtimeClientConstructorMock,
serveOrcaAppMock,
getDefaultUserDataPathMock
})
})
vi.mock('../runtime/launch', async (importOriginal) => ({
...(await importOriginal<Record<string, unknown>>()),
serveOrcaApp: serveOrcaAppMock
}))
vi.mock('../runtime/metadata', async (importOriginal) => ({
...(await importOriginal<Record<string, unknown>>()),
getDefaultUserDataPath: getDefaultUserDataPathMock
}))
vi.mock(
'../../shared/remote-runtime/host-store/runtime-environment-store',
async (importOriginal) => ({
...(await importOriginal<Record<string, unknown>>()),
listEnvironments: () => [{ id: 'env-1', name: 'server' }]
})
)
import { main } from '../index'
import { okFixture } from '../runtime/fixtures/runtime-rpc-fixtures'
const browserCalls = (): string[] =>
callMock.mock.calls.map(([method]) => String(method)).filter((m) => m.startsWith('browser.'))
// Unscoped, a paired server resolves the tab its person is looking at.
describe('browser commands over --environment', () => {
beforeEach(() => {
callMock.mockReset()
callMock.mockImplementation(async (method: string) =>
method === 'ssh.listTargetSummaries'
? okFixture('ssh', { targets: [] })
: okFixture('result', { browserPageId: 'page-1' })
)
vi.spyOn(console, 'log').mockImplementation(() => {})
vi.spyOn(console, 'error').mockImplementation(() => {})
process.exitCode = undefined
})
afterEach(() => {
vi.restoreAllMocks()
process.exitCode = undefined
})
it.each([
[['click', '--element', '@e1']],
[['goto', '--url', 'https://example.com']],
[['tab', 'current']],
[['tab', 'create', '--url', 'https://example.com']],
[['click', '--element', '@e1', '--worktree', 'all']]
])('refuses %j without --page or --worktree, sending nothing', async (args) => {
await main([...args, '--environment', 'server', '--json'], '/tmp/not-an-orca-worktree')
expect(browserCalls()).toEqual([])
expect(process.exitCode).toBe(1)
})
it('sends a command that names its page', async () => {
await main(
['click', '--element', '@e1', '--page', 'page-1', '--environment', 'server', '--json'],
'/tmp/not-an-orca-worktree'
)
expect(browserCalls()).toEqual(['browser.click'])
expect(process.exitCode).toBeUndefined()
})
})
+5
View File
@@ -204,6 +204,11 @@ export function formatBrowserActionEffects(result: unknown): string {
if (effects.passkeyRequest) {
lines.push(formatPasskeyRequest(effects.passkeyRequest))
}
if (effects.blockedPrint) {
lines.push(
`${effects.blockedPrint.origin} tried to open the system print dialog, which Orca blocked so it wouldn't take over the user's screen. To save the page as a PDF, run \`pdf\`.`
)
}
if (effects.loading) {
lines.push('Still loading; run `wait --load load`, then `snapshot`.')
} else if (effects.page === undefined) {
+4 -4
View File
@@ -22,12 +22,12 @@ import {
import { RuntimeClientError } from '../runtime/types'
import { isBrowserTabSplitDirection } from '../../shared/browser-tab-split-direction'
import { getBrowserWorktreeSelector } from '../workspace-selection/workspace-selector'
import { getBrowserCommandTarget } from './browser-command-target'
import { getBrowserCommandTarget, getScopedBrowserWorktreeSelector } from './browser-command-target'
export const BROWSER_TAB_HANDLERS: Record<string, CommandHandler> = {
'open-url': async ({ flags, client, cwd, json }) => {
const url = getRequiredStringFlag(flags, 'url')
const worktree = await getBrowserWorktreeSelector(flags, cwd, client)
const worktree = await getScopedBrowserWorktreeSelector(flags, cwd, client)
const result = await client.call<{ browserPageId: string }>(
'browser.openUrl',
{ url, worktree },
@@ -54,7 +54,7 @@ export const BROWSER_TAB_HANDLERS: Record<string, CommandHandler> = {
printResult(result, json, formatTabAnnotations)
},
'tab current': async ({ flags, client, cwd, json }) => {
const worktree = await getBrowserWorktreeSelector(flags, cwd, client)
const worktree = await getScopedBrowserWorktreeSelector(flags, cwd, client)
const result = await client.call<BrowserTabCurrentResult>('browser.tabCurrent', { worktree })
printResult(result, json, formatTabShow)
},
@@ -82,7 +82,7 @@ export const BROWSER_TAB_HANDLERS: Record<string, CommandHandler> = {
if (split !== undefined && !isBrowserTabSplitDirection(split)) {
throw new RuntimeClientError('invalid_argument', '--split must be right, left, down, or up')
}
const worktree = await getBrowserWorktreeSelector(flags, cwd, client)
const worktree = await getScopedBrowserWorktreeSelector(flags, cwd, client)
// Why the caller's tab: the split should sit beside the terminal that ran the command, not
// wherever the user last clicked. Spread conditionally so a plain create keeps its old payload.
const callerTabId = process.env.ORCA_TAB_ID
+7
View File
@@ -4,6 +4,7 @@ import { quoteCliCommandArgument } from '../arguments/shell-command-quote'
import type {
ComputerActionMetadata,
ComputerActionResult,
ComputerLaunchAppResult,
ComputerListAppsResult,
ComputerListWindowsResult,
ComputerSnapshotResult
@@ -108,6 +109,12 @@ export function formatListApps(result: ComputerListAppsResult): string {
.join('\n')
}
export function formatLaunchApp(result: ComputerLaunchAppResult): string {
const verb = result.launched ? 'Launched' : 'Already running:'
const place = result.delivery === 'foreground' ? 'in the foreground' : 'in the background'
return `${verb} ${result.app.name} pid:${result.app.pid} ${place}`
}
export function formatListWindows(result: ComputerListWindowsResult): string {
if (result.windows.length === 0) {
return `No windows found for ${result.app.name}.`
+12 -2
View File
@@ -1,5 +1,6 @@
import type {
ComputerActionResult,
ComputerLaunchAppResult,
ComputerListAppsResult,
ComputerListWindowsResult,
ComputerProviderCapabilities,
@@ -18,6 +19,7 @@ import { RuntimeClientError } from '../runtime/types'
import {
formatComputerAction,
formatGetAppState,
formatLaunchApp,
formatListApps,
formatListWindows
} from './computer-format'
@@ -49,6 +51,7 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
},
'computer permissions': async ({ flags, client, json }) => {
const id = getComputerPermissionSetupId(flags)
// Agents run this on errors; System Settings over the user's work is only worth it when something is missing.
const result = await client.call<{
platform: NodeJS.Platform
helperAppPath: string | null
@@ -57,7 +60,7 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
launchQueued?: boolean
permissions?: { id: string; status: string }[]
nextStep?: string | null
}>('computer.permissions', id ? { id } : {})
}>('computer.permissions', id ? { id, skipGranted: true } : {})
printResult(result, json, (value) => {
if (value.platform !== 'darwin') {
return 'Computer-use permission setup is only required on macOS.'
@@ -78,6 +81,13 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
.join('\n')
})
},
'computer launch-app': async ({ flags, client, json }) => {
const result = await client.call<ComputerLaunchAppResult>('computer.launchApp', {
app: getRequiredStringFlag(flags, 'app'),
...(flags.get('activate') === true ? { activate: true } : {})
})
printResult(result, json, formatLaunchApp)
},
'computer list-windows': async ({ flags, client, json }) => {
const result = await client.call<ComputerListWindowsResult>('computer.listWindows', {
app: getRequiredStringFlag(flags, 'app')
@@ -268,7 +278,7 @@ function getComputerPermissionSetupId(
function formatComputerCapabilities(value: ComputerProviderCapabilities): string {
return [
`${value.provider} (${value.platform}, protocol ${value.protocolVersion})`,
` Apps: list=${value.supports.apps.list} bundleIds=${value.supports.apps.bundleIds} pids=${value.supports.apps.pids}`,
` Apps: list=${value.supports.apps.list} bundleIds=${value.supports.apps.bundleIds} pids=${value.supports.apps.pids} launch=${value.supports.apps.launch === true}`,
` Windows: list=${value.supports.windows.list} targetById=${value.supports.windows.targetById} targetByIndex=${value.supports.windows.targetByIndex}`,
` Observation: screenshot=${value.supports.observation.screenshot} elementFrames=${value.supports.observation.elementFrames} annotatedScreenshot=${value.supports.observation.annotatedScreenshot}`,
` Actions: ${Object.entries(value.supports.actions)
@@ -24,6 +24,9 @@ const COMMAND_SCOPED_FLAG_HELP: Record<string, Record<string, string>> = {
query:
'--query <text> Show only elements whose line contains this text, with their ancestors'
},
'computer launch-app': {
activate: '--activate Start the app in front; only when the user asked for it'
},
'computer wait': {
text: '--text <text> Wait until an element line contains this text',
enabled:
@@ -189,6 +189,7 @@ export const ROOT_HELP_TEXT_PRIMARY = [
' computer capabilities Show computer-use provider capabilities',
' computer permissions Show or open computer-use permission setup',
' computer list-apps List running apps available to computer-use',
' computer launch-app Start a closed app without taking the foreground',
' computer list-windows List visible windows for a target app',
' computer get-app-state Capture a compact accessibility snapshot of an app',
' computer wait Wait until an app shows text, enables a control, or opens a window',
@@ -185,6 +185,7 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [
'computer capabilities',
'computer list-apps',
'computer permissions',
'computer launch-app',
'computer list-windows',
'computer get-app-state',
'computer wait',
+8 -3
View File
@@ -308,8 +308,13 @@ export class RuntimeClient {
}
async openOrca(timeoutMs = 15_000): Promise<RuntimeRpcSuccess<CliStatusResult>> {
return this.remoteSelected
? this.getCliStatus()
: openLocalOrca(() => this.getCliStatus(), timeoutMs)
if (this.remoteSelected) {
return this.getCliStatus()
}
return openLocalOrca(
() => this.getCliStatus(),
(options) => this.call<{ requested: boolean }>('desktop.openWindowInBackground', {}, options),
timeoutMs
)
}
}
-50
View File
@@ -1,50 +0,0 @@
import '../../shared/remote-runtime/host-store/environment-store-worker-test-fixture'
import { describe, expect, it } from 'vitest'
import { RuntimeRpcEnvelopeSchema } from './envelope-schema'
describe('RuntimeRpcEnvelopeSchema', () => {
it('accepts and strips additive future success fields', () => {
const parsed = RuntimeRpcEnvelopeSchema.safeParse({
id: 'req-1',
ok: true,
result: { value: 1, futureResultField: true },
futureEnvelopeField: 'new',
_meta: { runtimeId: 'runtime-1', futureMetaField: 2 }
})
expect(parsed).toEqual({
success: true,
data: {
id: 'req-1',
ok: true,
result: { value: 1, futureResultField: true },
_meta: { runtimeId: 'runtime-1' }
}
})
})
it('accepts and strips additive future failure fields', () => {
const parsed = RuntimeRpcEnvelopeSchema.safeParse({
id: 'req-1',
ok: false,
futureEnvelopeField: 'new',
error: {
code: 'failed',
message: 'Nope',
data: { retryable: true },
futureErrorField: 2
},
_meta: { runtimeId: 'runtime-1', futureMetaField: 3 }
})
expect(parsed).toEqual({
success: true,
data: {
id: 'req-1',
ok: false,
error: { code: 'failed', message: 'Nope', data: { retryable: true } },
_meta: { runtimeId: 'runtime-1' }
}
})
})
})
+13 -5
View File
@@ -21,6 +21,7 @@ import {
superviseForegroundServe
} from './serve-update-supervisor'
import { RuntimeClientError } from './types'
import { BACKGROUND_OPEN_ARG } from '../../shared/background-open-argv'
import {
getExecutableAppArgs,
resolveAppRoot,
@@ -69,7 +70,7 @@ export function launchOrcaApp(): void {
const overrideExecutable = process.env.ORCA_APP_EXECUTABLE
if (typeof overrideExecutable === 'string' && overrideExecutable.trim().length > 0) {
const pinned = pinLaunchUserDataPath(
getExecutableAppArgs(overrideExecutable),
[...getExecutableAppArgs(overrideExecutable), BACKGROUND_OPEN_ARG],
stripElectronRunAsNode(process.env),
userDataPath
)
@@ -93,7 +94,7 @@ export function launchOrcaApp(): void {
}
const pinned = pinLaunchUserDataPath(
getExecutableAppArgs(process.execPath),
[...getExecutableAppArgs(process.execPath), BACKGROUND_OPEN_ARG],
stripElectronRunAsNode(process.env),
userDataPath
)
@@ -107,12 +108,19 @@ export function launchOrcaApp(): void {
)
}
/** Nondefault profiles need a new instance: open drops env and only new instances receive --args. */
/** Nondefault profiles need a new instance: open drops env and only new instances receive --args. -g keeps the user's app in front. */
export function getMacOpenArgs(appBundlePath: string, userDataPath: string): string[] {
if (isSameUserDataPath(userDataPath, getPlatformUserDataPath())) {
return [appBundlePath]
return ['-g', appBundlePath, '--args', BACKGROUND_OPEN_ARG]
}
return ['-n', appBundlePath, '--args', `--user-data-dir=${userDataPath}`]
return [
'-g',
'-n',
appBundlePath,
'--args',
`--user-data-dir=${userDataPath}`,
BACKGROUND_OPEN_ARG
]
}
function spawnDetached(command: string, args: string[], options: SpawnOptions): void {
+71
View File
@@ -0,0 +1,71 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { CliStatusResult } from '../../shared/runtime-types'
import { launchOrcaApp } from './launch'
import { openLocalOrca } from './open-local-orca'
import type { RuntimeRpcSuccess } from './types'
vi.mock('./launch', () => ({ launchOrcaApp: vi.fn() }))
function status(desktopWindowStatus: string): RuntimeRpcSuccess<CliStatusResult> {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: openLocalOrca reads only runtime.reachable and app.desktopWindowStatus.
return {
ok: true,
result: { runtime: { reachable: true }, app: { desktopWindowStatus } }
} as unknown as RuntimeRpcSuccess<CliStatusResult>
}
describe('openLocalOrca', () => {
beforeEach(() => {
vi.mocked(launchOrcaApp).mockClear()
})
it('asks a running Orca to show its window instead of relaunching it', async () => {
const readStatus = vi
.fn()
.mockResolvedValueOnce(status('openable'))
.mockResolvedValueOnce(status('available'))
const openWindowInBackground = vi.fn(async () => ({
id: '1',
ok: true as const,
result: { requested: true },
_meta: { runtimeId: 'runtime' }
}))
await openLocalOrca(readStatus, openWindowInBackground, 1_000)
expect(openWindowInBackground).toHaveBeenCalledOnce()
expect(launchOrcaApp).not.toHaveBeenCalled()
})
it('does not relaunch an older Orca whose window is already available', async () => {
const available = status('available')
await expect(
openLocalOrca(
async () => available,
async () => {
throw new Error('method_not_found')
},
1_000
)
).resolves.toBe(available)
expect(launchOrcaApp).not.toHaveBeenCalled()
})
it('launches Orca when an older headless runtime cannot open its own window', async () => {
const readStatus = vi
.fn()
.mockResolvedValueOnce(status('openable'))
.mockResolvedValueOnce(status('available'))
await openLocalOrca(
readStatus,
async () => {
throw new Error('method_not_found')
},
1_000
)
expect(launchOrcaApp).toHaveBeenCalledOnce()
})
})
+26 -3
View File
@@ -3,21 +3,34 @@ import { launchOrcaApp } from './launch'
import { RuntimeClientError, RuntimeRpcFailureError, type RuntimeRpcSuccess } from './types'
type ReadStatus = () => Promise<RuntimeRpcSuccess<CliStatusResult>>
type OpenWindowInBackground = (options: {
timeoutMs: number
}) => Promise<RuntimeRpcSuccess<{ requested: boolean }>>
// The runtime only queues a window request; a stuck one must not hold `orca open` for a full RPC timeout.
const OPEN_WINDOW_RPC_TIMEOUT_MS = 5_000
export async function openLocalOrca(
readStatus: ReadStatus,
openWindowInBackground: OpenWindowInBackground,
timeoutMs: number
): Promise<RuntimeRpcSuccess<CliStatusResult>> {
const initial = await getStatusForOpen(readStatus)
const windowStatus = initial?.result.app.desktopWindowStatus
// A blocked runtime cannot open a window; a second app would only hit its instance lock.
if (initial?.result.app.desktopWindowStatus === 'blocked') {
if (windowStatus === 'blocked') {
throwDesktopActivationBlocked()
}
launchOrcaApp()
if (initial?.result.app.desktopWindowStatus === 'available') {
// Relaunching a running Orca would hand it the foreground, so ask it to show its own window.
const asked =
initial?.result.runtime.reachable === true && (await askToOpenWindow(openWindowInBackground))
if (initial && windowStatus === 'available') {
return initial
}
if (!asked) {
launchOrcaApp()
}
const startedAt = Date.now()
while (Date.now() - startedAt < timeoutMs) {
@@ -55,6 +68,16 @@ async function getStatusForOpen(
}
}
async function askToOpenWindow(openWindowInBackground: OpenWindowInBackground): Promise<boolean> {
try {
const response = await openWindowInBackground({ timeoutMs: OPEN_WINDOW_RPC_TIMEOUT_MS })
return response.result.requested === true
} catch {
// Older runtimes lack the method; any failure falls back to relaunching, as open always did.
return false
}
}
function throwDesktopActivationBlocked(): never {
throw new RuntimeClientError(
'desktop_activation_blocked',
@@ -1,18 +1,10 @@
import '../../shared/remote-runtime/host-store/environment-store-worker-test-fixture'
import { MacServeStartupAbortError, serveSignalExitError } from './serve-signal-exit-diagnostic'
import { EventEmitter } from 'node:events'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
SERVE_CHILD_FORCE_KILL_SCHEDULING_MARGIN_MS,
superviseForegroundServe
} from './serve-update-supervisor'
import {
QUIT_RENDERER_ACK_TIMEOUT_MS,
WILL_QUIT_TEARDOWN_DEADLINE_MS
} from '../../shared/quit-teardown-deadline'
import { superviseForegroundServe } from './serve-update-supervisor'
class FakeChildProcess extends EventEmitter {
kill = vi.fn()
@@ -41,67 +33,7 @@ afterEach(() => {
vi.useRealTimers()
})
describe('serveSignalExitError', () => {
it('classifies darwin SIGABRT as a startup abort', () => {
const error = serveSignalExitError('SIGABRT', 'darwin')
expect(error).toBeInstanceOf(MacServeStartupAbortError)
})
})
describe('superviseForegroundServe signal exits', () => {
it('lets pre-commit and committed Electron quit deadlines finish before force-killing serve', async () => {
setPlatform('linux')
vi.useFakeTimers()
const child = new FakeChildProcess()
const supervised = superviseChild(child)
process.emit('SIGTERM', 'SIGTERM')
expect(child.kill).toHaveBeenCalledOnce()
expect(child.kill).toHaveBeenLastCalledWith('SIGTERM')
await vi.advanceTimersByTimeAsync(QUIT_RENDERER_ACK_TIMEOUT_MS + WILL_QUIT_TEARDOWN_DEADLINE_MS)
expect(child.kill).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(SERVE_CHILD_FORCE_KILL_SCHEDULING_MARGIN_MS)
expect(child.kill).toHaveBeenLastCalledWith('SIGKILL')
expect(child.kill).toHaveBeenCalledTimes(2)
child.emit('exit', null, 'SIGKILL')
await expect(supervised).rejects.toThrow('Orca serve exited via SIGKILL.')
})
it('lets a shared-console Windows child handle Ctrl-C gracefully', async () => {
setPlatform('win32')
vi.useFakeTimers()
const child = new FakeChildProcess()
const supervised = superviseChild(child)
process.emit('SIGINT', 'SIGINT')
expect(child.kill).not.toHaveBeenCalled()
child.emit('exit', 0, null)
await expect(supervised).resolves.toBe(0)
})
it('forwards Linux terminal hangup and removes the listener after exit', async () => {
setPlatform('linux')
const listenersBefore = process.listeners('SIGHUP')
const child = new FakeChildProcess()
const supervised = superviseChild(child)
expect(process.listeners('SIGHUP')).toHaveLength(listenersBefore.length + 1)
process.emit('SIGHUP', 'SIGHUP')
expect(child.kill).toHaveBeenCalledWith('SIGHUP')
child.emit('exit', null, 'SIGHUP')
await expect(supervised).resolves.toBe(0)
expect(process.listeners('SIGHUP')).toEqual(listenersBefore)
const killCallsAfterExit = child.kill.mock.calls.length
process.emit('SIGHUP', 'SIGHUP')
expect(child.kill).toHaveBeenCalledTimes(killCallsAfterExit)
})
it('treats a child exit through the caller-forwarded SIGINT as graceful', async () => {
setPlatform('linux')
const child = new FakeChildProcess()
+3 -1
View File
@@ -7,6 +7,7 @@ import { RemoteRuntimeClientError } from '../../shared/remote-runtime/remote-run
import { AGENT_LAUNCH_RUNTIME_CAPABILITY } from '../../shared/agent-launch/contracts/agent-launch-runtime-capability'
import { AGENT_SESSION_RENAME_ONLY_CLIENT_CAPABILITY } from '../../shared/agent-session/structured-agent-session-surface-capabilities'
import { REPO_SEARCH_QUALIFIED_REFS_RUNTIME_CAPABILITY } from '../../shared/worktree-runtime-capabilities'
import { CLI_AGENT_CALLER_CLIENT_CAPABILITY } from '../../shared/remote-runtime/cli-agent-caller-capability'
import { sendRemoteRuntimeRequestOnSocket } from '../../shared/remote-runtime/requests/remote-runtime-request-socket'
import type { RuntimeStatus } from '../../shared/runtime-types'
import { RuntimeClientError, type RuntimeRpcResponse } from './types'
@@ -15,7 +16,8 @@ import { RuntimeClientError, type RuntimeRpcResponse } from './types'
const CLI_CLIENT_CAPABILITIES = [
AGENT_LAUNCH_RUNTIME_CAPABILITY,
REPO_SEARCH_QUALIFIED_REFS_RUNTIME_CAPABILITY,
AGENT_SESSION_RENAME_ONLY_CLIENT_CAPABILITY
AGENT_SESSION_RENAME_ONLY_CLIENT_CAPABILITY,
CLI_AGENT_CALLER_CLIENT_CAPABILITY
] as const
export async function sendWebSocketRequest<TResult>(
+9 -8
View File
@@ -737,7 +737,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [
"",
"Browser rules:",
"",
"- Actions (`click`, `fill`, `type`, `keypress`, `select`, `check`, `hover`, `scroll`, `goto`, `back`, `reload`, `mouse` and the rest) wait briefly for the page to settle and then report what they caused in `result.effects` (text mode prints it under the action): `url`/`title` when they changed, `newTabs` with their `browserPageId`, blocked or opened `popups`, `downloads` started, `consoleErrors`, a `passkeyRequest` still waiting, and `page`: `\"unchanged\"`, or the snapshot lines the action added or replaced, with their refs (`full: true` means the whole new page). `loading: true` means a navigation was still running: `wait --load load`, then `snapshot`. No `page` means the page could not be read: run `snapshot`. `settled: false` means the page was still changing when it was read. An action that opens a JavaScript dialog fails with `browser_dialog_opened` instead. One that opens a file picker fails with `browser_file_chooser_opened`: Orca closes the picker so the system dialog never covers the user's screen; use `ORCA upload` on the file input or its button instead.",
"- Actions (`click`, `fill`, `type`, `keypress`, `select`, `check`, `hover`, `scroll`, `goto`, `back`, `reload`, `mouse` and the rest) wait briefly for the page to settle and then report what they caused in `result.effects` (text mode prints it under the action): `url`/`title` when they changed, `newTabs` with their `browserPageId`, blocked or opened `popups`, `downloads` started, `consoleErrors`, a `passkeyRequest` still waiting, `blockedPrint` when Orca refused the page's system print dialog (run `ORCA pdf` to save the page instead), and `page`: `\"unchanged\"`, or the snapshot lines the action added or replaced, with their refs (`full: true` means the whole new page). `loading: true` means a navigation was still running: `wait --load load`, then `snapshot`. No `page` means the page could not be read: run `snapshot`. `settled: false` means the page was still changing when it was read. An action that opens a JavaScript dialog fails with `browser_dialog_opened` instead. One that opens a file picker fails with `browser_file_chooser_opened`: Orca closes the picker so the system dialog never covers the user's screen; use `ORCA upload` on the file input or its button instead.",
"- Snapshot again only when `page.truncated` is true, after `wait`, after a tab switch, on a `browser_stale_ref`, or when a result has no `effects` (older Orca, or a server-hosted page).",
"- Pass `--expect-url <url>` with `--page` when acting on the wrong resource would matter, such as writing or sending in one conversation of a chat app. Use the exact absolute URL from `result.tab.url` in `ORCA tab current --json`, or `url` from a `snapshot --json` or `goto`. Path, query and fragment must all match, so a page that rewrites its URL while you work refuses until you read the URL again. If the page's URL differs, or a navigation to another URL is under way, the command fails with `browser_expected_url_mismatch`, and `error.data` holds `expectedUrl`, `actualUrl`, `browserPageId` and `mayHaveStarted`. `mayHaveStarted: false` means nothing ran; `true` means a step of a longer command (a typed run, a `select-all`) may have happened, so look at the page before retrying. Success adds `browserPageId` and `checkedUrl`, the URL last checked; the action itself may have navigated since, and `effects.url` says where. The check runs right before the action and again before each later step, but it is not a lock: a route change in the instant after a check can still receive the action. It compares the top-level page only, so it says nothing about an iframe or an app that shows different content at one URL; read the page for those. `exec` refuses the flag, because one raw batch can navigate and retarget. A runtime, client or desktop that cannot check the URL refuses with `incompatible_runtime` or `browser_expect_url_unsupported` rather than acting unguarded.",
"- Refs like `@e1` are assigned by `snapshot`, scoped to one tab, and invalidated by navigation or tab switch.",
@@ -747,7 +747,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [
"- When the user says they left notes or comments on a page, read them with `ORCA tab annotations --page <browserPageId>` (the current tab without `--page`). It prints the notes they haven't sent, the same text the tab's Copy button gives, and `--json` returns them in `result.annotations`. Notes the user already sent arrive in your chat instead.",
"- Passkey, fingerprint, Face ID, Windows Hello and security-key prompts can't be completed by you. When `snapshot` or an action's `effects` has `passkeyRequest`, the page is waiting on one from that origin and will wait for minutes. Ask the user to finish the sign-in, or run `ORCA dialog dismiss --page <browserPageId>` to cancel it so the page offers another way to sign in.",
"- A popup window a page opens (sign-in, payment) is listed in `ORCA tab list --json` as a `popup-<n>` page with `openerBrowserPageId`. If a click seems to do nothing, check `tab list`, then drive the popup with `--page popup-<n>`. When it closes, continue on the opener page.",
"- Browser commands default to the current worktree. `--page` reaches the page in any worktree unless you also pass `--worktree <selector>`. Use `--worktree all` only intentionally.",
"- Browser commands default to the worktree you are in: your cwd's worktree, else your Orca terminal's. Outside both (a shell or IDE Orca did not open, with a cwd outside any worktree), and on every `--environment` command, always pass `--page <browserPageId>` or `--worktree <selector>` (not `all`); find pages with `ORCA tab list --json`. Without either, a local command acts on the tab the user is looking at, and an `--environment` command is refused. `--page` reaches the page in any worktree unless you also pass `--worktree <selector>`. Use `--worktree all` only intentionally.",
"- `tab list` shows every tab in the tab bar. After a restart, tabs the user has not opened report `loaded: false`; `tab switch` or any `--page` command loads one in the background without changing the user's visible tab.",
"- `tab create --split <right|left|down|up>` opens the page in a new pane beside the calling terminal instead of as another tab; focus stays in the terminal. When a browser pane is already open beside it, the page opens as that pane's visible tab instead of adding another pane.",
"- Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command \"tab ...\"`, so Orca keeps UI state synchronized.",
@@ -1108,6 +1108,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
"ORCA computer permissions --json",
"ORCA computer capabilities --json",
"ORCA computer list-apps --json",
"ORCA computer launch-app <app> --json",
"ORCA computer list-windows --app <app> --json",
"ORCA computer get-app-state --app <app> --json",
"ORCA computer wait --app <app> (--text <text> | --enabled <text> | --window-title <text>) [--timeout <ms>] --json",
@@ -1150,7 +1151,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
"- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.",
"- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.",
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.",
"- `type-text --element-index <index>` inserts at that text field's caret, replacing its selection, without focusing it or bringing its window forward, and reads the text back. It works on macOS and Linux; where `capabilities` lacks `actions.typeTextElement` it fails with `unsupported_capability`; then click the field, confirm it is the focused text receiver, and type without the index. A `value_not_settable` error means that element does not accept text insertion.",
"- `type-text --element-index <index>` inserts at that text field's caret, replacing its selection, without focusing it or bringing its window forward, and reads the text back. It works on macOS, Linux and Windows; where `capabilities` lacks `actions.typeTextElement` it fails with `unsupported_capability`; then click the field, confirm it is the focused text receiver, and type without the index. A `value_not_settable` error means that element does not accept text insertion.",
"- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.",
"- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.",
"- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.",
@@ -1167,22 +1168,22 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
"- An action fails with `background_unavailable` and no semantic action (`set-value`, `perform-secondary-action`, `type-text --element-index`) can do the job.",
"- A background action reports success but the UI does not change, and no semantic action can do the job.",
"- On macOS, drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.",
"- On Windows, keyboard input (`press-key`, `hotkey`, `paste-text` and `type-text`) needs the target window focused; otherwise it is refused with `window_not_focused` and nothing is sent. For editable fields use `set-value`.",
"- On Windows, keyboard input (`press-key`, `hotkey`, `paste-text` and `type-text` without `--element-index`) needs the target window in front; otherwise it is refused with `background_unavailable` and nothing is sent. Enter text with `type-text --element-index` or `set-value`.",
"- On Linux and Windows, actions that need the real pointer, and on Linux keys sent while the target window is not active (Return in a focused GTK text field works). They are refused with `background_unavailable`; see below.",
"",
"On macOS, a window that is minimized, hidden (Cmd+H) or on another Space cannot be read, so `get-app-state` fails with `window_not_found`. Ask the user to bring it on screen.",
"On macOS, `list-windows` and `get-app-state` also read windows that are minimized, hidden (Cmd+H) or on another Space, full-screen ones included; `isMinimized` and `isOffscreen` say which. Element actions work there (`click --element-index`, `set-value`, `type-text --element-index`, secondary actions), while coordinate clicks, scrolls, drags, keys and pastes are refused with `background_unavailable` and nothing is sent. Minimized and hidden windows have no screenshot. A window on another Space that nothing has read yet fails with `window_not_found`; ask the user to show it once.",
"",
"On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks or drags that start on popup menus, menu buttons, native combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, native combo box or `<select>`, set the option with `set-value` on that element.",
"",
"On macOS, every action without `--restore-window` runs in the background: the target app is not brought forward, the user's frontmost app and key window stay put, and the real pointer does not move. `capabilities` reports this as `supports.input.background`.",
"",
"On Linux, element clicks through the element's accessibility action, `set-value`, `type-text --element-index` and secondary actions run without raising the window or moving the pointer, and Return in a focused GTK text field runs its activate action. Anything that needs the real pointer (clicks with no accessibility action, right, middle, multiple or modified clicks, scrolling, drags) or the focused window (other keys, typing with no focused editable field) is refused with `background_unavailable` and nothing is sent; so is a click that would open a menu, a keyboard-grabbing popup or a dialog (a GTK 4 or Qt label ending in `…`). Ask the user before retrying with `--restore-window`, which raises only the target window, refuses a window on another workspace and puts the cursor back. On Wayland, synthetic pointer and key input never reaches native windows, so it is refused even with the flag. `action.delivery` is measured: `foreground` when the input went to the window in front or the target came forward.",
"On Linux, element clicks through the element's accessibility action (or, for a list row with none, its list's selection), `set-value`, `type-text --element-index` and secondary actions run without raising the window or moving the pointer, and Return in a focused GTK text field runs its activate action. Scrolling moves the nearest scroll bar under the element or point by whole viewports, or pages a web document, also in the background. Anything that needs the real pointer (other clicks with no accessibility action, right, middle, multiple or modified clicks, scrolls in a view with no scroll bar or document, drags) or the focused window (other keys, typing with no focused editable field) is refused with `background_unavailable` and nothing is sent; so is a click that would open a menu, a keyboard-grabbing popup or a dialog (a GTK 4 or Qt label ending in `…`). Ask the user before retrying with `--restore-window`, which raises only the target window, refuses a window on another workspace and puts the cursor back. On Wayland, synthetic pointer and key input never reaches native windows, so it is refused even with the flag. `action.delivery` is measured: `foreground` when the input went to the window in front or the target came forward.",
"",
"On Windows, these actions go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), scrolls of an element or of the scrollable area under `--x`/`--y` (fractional `--pages` too), and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Clicks by coordinates, right, middle, modified and multi-clicks, clicks on elements with no `Invoke`, `Select` or `Toggle` action, scrolls over nothing scrollable, and drags need the real mouse, so they are refused with `background_unavailable` and nothing is sent. With `--restore-window` (after asking the user), only the target window comes forward, never one on another virtual desktop, and the pointer goes back where it was.",
"On Windows, these actions go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), `type-text --element-index`, scrolls of an element or of the scrollable area under `--x`/`--y` (fractional `--pages` too), and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). WinForms buttons, check boxes and radio buttons, and native text boxes, combo boxes and list boxes (user32 and WinForms), are driven through their own window messages, which leave the window where it is. User32 buttons, check boxes and radio buttons use UI Automation, so they can bring their window forward. Other apps can bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Clicks by coordinates, right, middle, modified and multi-clicks, clicks on elements with no `Invoke`, `Select` or `Toggle` action, scrolls over nothing scrollable, and drags need the real mouse, so they are refused with `background_unavailable` and nothing is sent. `type-text` without `--element-index`, `press-key`, `hotkey` and `paste-text` send keys, which need the window in front, so they are refused the same way; enter text with `type-text --element-index` or `set-value` instead. With `--restore-window` (after asking the user), only the target window comes forward, never one on another virtual desktop, and the pointer goes back where it was.",
"",
"## Starting Apps",
"",
"`orca computer` does not launch apps. On macOS, start one without bringing it forward with `open -g -a <App>` (or `open -g -b <bundle-id>`); plain `open -a` brings it to the front. Some apps, such as many Electron apps, still bring their own window forward when it opens. On Windows and Linux a newly started app usually comes to the front, so ask the user before starting one.",
"Start a closed app with `ORCA computer launch-app <app>`, not `open -a` or `Start-Process`: those bring the app to the front and take the user's keyboard. `launch-app` asks macOS and Windows to start it in the background and leaves an app that is already running alone. Some apps, such as many Electron apps, still bring their own window forward when they open, so read `delivery`: if it says `foreground`, tell the user the app came to the front. Pass `--activate` only when the user asked for the app to come forward. On Linux it fails with `unsupported_capability`, because a newly started app usually comes to the front there; ask the user before starting one another way.",
"",
"## Screenshots",
"",
+12
View File
@@ -34,6 +34,18 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
usage: 'orca computer permissions [--id <accessibility|screenshots>] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'id']
},
{
path: ['computer', 'launch-app'],
summary: 'Start a closed app without taking the foreground',
usage: 'orca computer launch-app <name|bundle|path> [--activate] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'app', 'activate'],
positionalArgs: ['app'],
notes: [
'Leaves an app that is already running as it is. Use this instead of `open -a` or Start-Process, which bring the app to the front.',
'Use --activate only when the user asked for the app to come forward: it takes the keyboard from whatever they are doing. It applies only when launch-app starts the app.',
'macOS and Windows only.'
]
},
{
path: ['computer', 'list-windows'],
summary: 'List windows for an app available to computer-use',
+1 -1
View File
@@ -16,7 +16,7 @@ export const CORE_COMMAND_SPECS: CommandSpec[] = [
usage: 'orca open [--json]',
allowedFlags: [...GLOBAL_FLAGS],
notes: [
"It can bring Orca's window to the front. Agents should run it only when the user asks or `orca status --json` reports Orca is not running or still cannot connect after about 30 seconds of retries, never to retry a failed command."
'Starts Orca without taking the foreground and does nothing when an Orca window is already open. Agents should run it only when the user asks or `orca status --json` reports Orca is not running or still cannot connect after about 30 seconds of retries, never to retry a failed command.'
],
examples: ['orca open', 'orca open --json']
},
@@ -0,0 +1,313 @@
import { createStructuredAgentSessionCatalogReads } from '../../native-chat/agent-session-wire/options/structured-agent-session-catalog-reads'
import type { StructuredAgentSessionHostDeps } from '../../native-chat/agent-session-wire/host/structured-agent-session-host-types'
import type { createAgentModelCatalogSuccessWriter } from '../../native-chat/agent-model-catalog/agent-model-catalog-entry-lifetime'
import { createAcpModelCatalogProbe } from './acp-model-catalog-probe'
import { AgentModelCatalogStore } from '../../native-chat/agent-model-catalog/agent-model-catalog-store'
import { createAgentModelCatalogService } from '../../native-chat/agent-model-catalog/agent-model-catalog-service'
import { NewSessionResponseSchema } from '../generated/acp-protocol.generated'
import recorded from '../fixtures/opencode-v2-start-options.json'
import { expect, it, vi, onTestFinished } from 'vitest'
import { acpLaunchSpecFor } from '../launch/acp-launch-registry'
import {
readSessionModels,
type AcpModelCatalogSessionConnection
} from './acp-listing-session-models'
import type { runAgentModelCatalogListing } from '../../native-chat/agent-model-catalog/agent-model-catalog-probe-runner'
import { runAgentModelCatalogSession } from '../../native-chat/agent-model-catalog/agent-model-catalog-probe-runner'
const LEARNED_EFFORTS = [
{ value: 'low', label: 'Low' },
{ value: 'high', label: 'High' },
{ value: 'max', label: 'Max' },
{ value: 'default', label: 'Default' }
]
it('publishes inventory before a slow optional effort reply and retains its valid late result', async () => {
vi.useFakeTimers()
onTestFinished(() => {
vi.useRealTimers()
})
const late =
Promise.withResolvers<
Awaited<ReturnType<AcpModelCatalogSessionConnection['setConfigOption']>>
>()
const observations: Awaited<ReturnType<typeof readSessionModels>>[] = []
const connection: AcpModelCatalogSessionConnection = {
initialize: async () => ({
protocolVersion: 1,
agentCapabilities: { sessionCapabilities: { delete: {} } }
}),
start: async () => ({
kind: 'new',
sessionId: 'listing',
response: NewSessionResponseSchema.parse(recorded)
}),
setConfigOption: () => late.promise,
deleteSession: async () => ({}),
close: async () => {}
}
const run = runAgentModelCatalogSession<
AcpModelCatalogSessionConnection,
Awaited<ReturnType<typeof readSessionModels>>
>(
() => connection,
(connection, ready) =>
readSessionModels(
acpLaunchSpecFor('opencode2')!,
{ kind: 'session', listingNamesConfiguredModel: false },
connection,
'/workspace',
undefined,
['opencode/big-pickle'],
undefined,
ready,
(result) => observations.push(result)
),
{ label: 'opencode2', timeoutMs: null }
)
await vi.advanceTimersByTimeAsync(45_000)
expect(observations[0]?.models.find((row) => row.id === 'opencode/big-pickle')).toMatchObject({
efforts: [],
effortsKnown: false
})
const options = structuredClone(recorded.configOptions)
options.find((option) => option.id === 'model')!.currentValue = 'opencode/big-pickle'
late.resolve({
configOptions: NewSessionResponseSchema.parse({ ...recorded, configOptions: options })
.configOptions!
})
const result = await run
expect(result.models.find((row) => row.id === 'opencode/big-pickle')?.efforts).toEqual([
{ value: 'low', label: 'Low' },
{ value: 'high', label: 'High' },
{ value: 'max', label: 'Max' },
{ value: 'default', label: 'Default' }
])
expect(result.attemptedModels).toEqual(['opencode/big-pickle'])
expect(
observations.at(-1)?.models.find((row) => row.id === 'opencode/big-pickle')?.efforts
).toEqual(result.models.find((row) => row.id === 'opencode/big-pickle')?.efforts)
expect(
observations[0]?.models.find((row) => row.id === 'opencode/big-pickle')?.effortsKnown
).toBe(false)
})
function accumulatedCatalogHarness(count: number) {
const baseSpec = acpLaunchSpecFor('opencode2')!
const spec = {
...baseSpec,
managedRuntime: undefined,
account: {
...baseSpec.account,
environment: (_home: unknown, env: Record<string, string>) => env
}
}
const config = (selected = 'm0') =>
NewSessionResponseSchema.parse({
...recorded,
configOptions: recorded.configOptions.map((option) =>
option.id === 'model'
? {
...option,
currentValue: selected,
options: Array.from({ length: count }, (_, index) => ({
value: `m${index}`,
name: `Model ${index}`
}))
}
: option
)
})
const requests: {
model: string
reply: PromiseWithResolvers<
Awaited<ReturnType<AcpModelCatalogSessionConnection['setConfigOption']>>
>
}[] = []
const connect = vi.fn((): AcpModelCatalogSessionConnection => ({
initialize: async () => ({
protocolVersion: 1,
agentCapabilities: { sessionCapabilities: { delete: {} } }
}),
start: async () => ({ kind: 'new', sessionId: 'listing', response: config() }),
setConfigOption: (_id, model) => {
const reply =
Promise.withResolvers<
Awaited<ReturnType<AcpModelCatalogSessionConnection['setConfigOption']>>
>()
requests.push({ model, reply })
return reply.promise
},
deleteSession: async () => ({}),
close: async () => {}
}))
const store = new AgentModelCatalogStore()
const writes = vi.spyOn(
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: measure the production publication boundary required by the scaling regression.
store as unknown as { writeSuccess: ReturnType<typeof createAgentModelCatalogSuccessWriter> },
'writeSuccess'
)
const probe = createAcpModelCatalogProbe(spec, {
resolveEnvironment: async () => ({}),
resolveCommand: () => process.execPath,
probeVersion: async () => ({ version: '2.1.0' }),
homePath: '/account',
dataDirectory: '/unused',
connectSession: connect
})
const service = createAgentModelCatalogService({
store,
getRecord: () => undefined,
getConversation: () => undefined,
drivesRecord: () => true,
resolveAccountHome: async () => ({ variable: 'HOME', path: '/account' }),
probes: { opencode2: probe }
})
const externalRead = createStructuredAgentSessionCatalogReads(() => ({
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: sessionless catalog reads only access modelCatalog; no chat or journal is mocked.
deps: { modelCatalog: service } as StructuredAgentSessionHostDeps,
serialize: async (_id, task) => task(),
scopeOwned: async () => null
}))
const read = async (waitForListing = true, selectedModel?: string) => {
const result = await externalRead({
agent: 'opencode2',
workspacePath: '/workspace',
waitForListing,
selectedModel
})
if (result.origin === 'unknown') {
throw new Error('Expected the listing inventory')
}
return result
}
onTestFinished(() => service.stop())
const answer = (index: number) => {
const request = requests[index]!
request.reply.resolve({ configOptions: config(request.model).configOptions! })
}
return { store, writes, service, requests, connect, read, answer }
}
it.each([10, 100])(
'accumulates %i untargeted menus without publishing the full catalog per reply',
async (count) => {
vi.useFakeTimers()
onTestFinished(() => {
vi.useRealTimers()
})
const h = accumulatedCatalogHarness(count)
const first = h.read()
await vi.advanceTimersByTimeAsync(0)
expect((await first).models).toHaveLength(count)
let delivered = false
const observer = h.read().then((answer) => {
delivered = true
return answer
})
await vi.advanceTimersByTimeAsync(0)
for (let index = 0; index < count - 1; index++) {
h.answer(index)
await vi.advanceTimersByTimeAsync(0)
if (index < count - 2) {
expect(delivered).toBe(false)
}
}
const final = await observer
expect(final.models?.every((model) => model.effortsKnown !== false)).toBe(true)
expect(h.connect).toHaveBeenCalledOnce()
// One initial inventory and one terminal observation, independent of the target count.
expect(h.writes.mock.calls.length).toBeLessThanOrEqual(2)
expect(
h.writes.mock.calls.reduce((rows, call) => rows + call[2].models.length, 0)
).toBeLessThanOrEqual(count * 2)
}
)
it.each(['snapshot', 'deadline', 'selected'])(
'exposes accumulated menus at a %s boundary while the next switch is unanswered',
async (boundary) => {
vi.useFakeTimers()
onTestFinished(() => {
vi.useRealTimers()
})
const h = accumulatedCatalogHarness(5)
const first = h.read()
await first
const observer = h.read()
h.answer(0)
await vi.advanceTimersByTimeAsync(6_000)
// m1 has answered; m2 is still pending, beyond the old optional sweep cutoff.
if (boundary === 'snapshot') {
expect((await h.read(false)).models?.find((row) => row.id === 'm1')?.efforts).toEqual(
LEARNED_EFFORTS
)
} else if (boundary === 'selected') {
expect((await h.read(true, 'm1')).models?.find((row) => row.id === 'm1')?.efforts).toEqual(
LEARNED_EFFORTS
)
} else {
await vi.advanceTimersByTimeAsync(24_000)
}
expect((await observer).models?.find((row) => row.id === 'm1')?.efforts).toEqual(
LEARNED_EFFORTS
)
let followed = false
const follow = h.read().then((answer) => {
followed = true
return answer
})
await vi.advanceTimersByTimeAsync(1_000)
expect(followed).toBe(false)
// New selected demand joins the same owner and is promoted after its outstanding switch.
const selected = h.read(true, 'm4')
h.answer(1)
await vi.advanceTimersByTimeAsync(6_000)
expect(h.requests[2]?.model).toBe('m4')
h.answer(2)
await vi.advanceTimersByTimeAsync(0)
expect((await selected).models?.find((row) => row.id === 'm4')?.efforts).toEqual(
LEARNED_EFFORTS
)
await follow
h.answer(3)
await vi.advanceTimersByTimeAsync(0)
expect(h.connect).toHaveBeenCalledOnce()
}
)
it('keeps a slow OpenCode 1.x version response and selects its command listing', async () => {
vi.useFakeTimers()
onTestFinished(() => {
vi.useRealTimers()
})
const version = Promise.withResolvers<{ version: string }>()
const requested = Promise.withResolvers<void>()
const base = acpLaunchSpecFor('opencode')!
const session = vi.fn()
const command = vi.fn<typeof runAgentModelCatalogListing>(
async () => 'opencode/slow-valid-model\n'
)
const probe = createAcpModelCatalogProbe(
{ ...base, account: { ...base.account, environment: (_home, env) => env } },
{
resolveEnvironment: async () => ({}),
resolveCommand: () => process.execPath,
probeVersion: async () => {
requested.resolve()
return version.promise
},
dataDirectory: '/unused',
homePath: '/account',
runListing: command,
connectSession: session
}
)
const result = probe({ variable: 'HOME', path: '/account' }, { workspacePath: '/workspace' })
await requested.promise
await vi.advanceTimersByTimeAsync(45_000)
version.resolve({ version: '1.18.31' })
expect((await result).models.map((row) => row.id)).toEqual(['opencode/slow-valid-model'])
expect(session).not.toHaveBeenCalled()
expect(command.mock.calls[0]?.[1]).toMatchObject({ timeoutMs: null })
})
@@ -13,7 +13,8 @@ import type {
InitializeResponse,
SetSessionConfigOptionResponse
} from '../generated/acp-protocol.generated'
import { withTimeout } from '../../../shared/promise-timeout-fallback'
import type { AgentModelCatalogReady } from '../../native-chat/agent-model-catalog/agent-model-catalog-probe-runner'
import type { AgentModelCatalogProgress } from '../../native-chat/agent-model-catalog/agent-model-catalog-discovery'
import { AcpAgentError } from '../transport/acp-errors'
/** Session-free capability boundary: no session methods are exposed. */
@@ -61,7 +62,15 @@ export async function readSessionModels(
operation?: StructuredAgentResolutionOperation,
selectedModels?: readonly string[],
launchedModel?: string,
ready?: (cleanup: () => Promise<unknown>) => void
ready?: AgentModelCatalogReady<{ models: AgentSessionModelOption[]; attemptedModels: string[] }>,
publish?: (result: { models: AgentSessionModelOption[]; attemptedModels: string[] }) => void,
deferred?: {
defer: (
read: () => { models: AgentSessionModelOption[]; attemptedModels: string[] },
known: (model: string) => boolean
) => void
demanded: NonNullable<AgentModelCatalogProgress['demanded']>
}
): Promise<{ models: AgentSessionModelOption[]; attemptedModels: string[] }> {
const initialized = await connection.initialize()
assertStructuredAgentResolutionCurrent(operation)
@@ -80,12 +89,16 @@ export async function readSessionModels(
})
}
}
// Startup and menu discovery have separate budgets; cleanup also runs on cancellation.
ready?.(cleanup)
const options = new AcpStructuredOptions(spec.dialect)
options.adoptSession(started.response, 'new')
let models = options.read().models
let models = [...options.read().models]
const attemptedModels: string[] = []
const partial = () => {
stopped = true
return { models: [...models], attemptedModels: [...attemptedModels] }
}
ready?.(cleanup, models.length > 0 ? partial : undefined)
const progress = () => publish?.({ models: [...models], attemptedModels: [...attemptedModels] })
try {
if (launchedModel !== undefined) {
attemptedModels.push(launchedModel)
@@ -96,49 +109,77 @@ export async function readSessionModels(
row.id === running ? { ...row, efforts: [], effortsKnown: false as const } : row
)
}
progress()
} else if (spec.dialect.effortChangesModel) {
const byId = new Map(models.map((model) => [model.id, model]))
const targets = selectedModels ? selectedModels.flatMap((id) => byId.get(id) ?? []) : models
const sweep = async () => {
for (const model of targets) {
if (stopped) {
return
}
assertStructuredAgentResolutionCurrent(operation)
if (model.effortsKnown !== false) {
continue
}
attemptedModels.push(model.id)
try {
for (let attempt = 0; attempt < 2 && !stopped; attempt++) {
const write = options.write('model', model.id)
if (write?.method !== 'config') {
break
}
const reply = await connection.setConfigOption(write.configId, write.value)
if (stopped) {
return
}
assertStructuredAgentResolutionCurrent(operation)
options.adoptConfigOptions(reply.configOptions)
const observed = options.read()
if (observed.current.model !== model.id) {
continue
}
const known = observed.models.find((row) => row.id === model.id)
if (known) {
models = models.map((row) => (row.id === model.id ? known : row))
}
break
}
} catch {
assertStructuredAgentResolutionCurrent(operation)
progress()
const indices = new Map(models.map((model, index) => [model.id, index]))
const remaining = new Set(selectedModels ?? models.map((model) => model.id))
const visited = new Set<string>()
const known = (id: string) => {
const index = indices.get(id)
return index !== undefined && models[index].effortsKnown !== false
}
const snapshot = () => ({ models: [...models], attemptedModels: [...attemptedModels] })
while (!stopped) {
const demanded = deferred?.demanded() ?? new Set(selectedModels)
for (const id of demanded) {
if (!visited.has(id) && indices.has(id)) {
remaining.add(id)
}
}
const id = [...demanded].find((id) => remaining.has(id)) ?? remaining.values().next().value
if (id === undefined) {
break
}
remaining.delete(id)
visited.add(id)
const index = indices.get(id)
if (index === undefined || known(id)) {
continue
}
assertStructuredAgentResolutionCurrent(operation)
let conclusive = false
try {
for (let attempt = 0; attempt < 2 && !stopped; attempt++) {
const write = options.write('model', id)
if (write?.method !== 'config') {
conclusive = true
break
}
const reply = await connection.setConfigOption(write.configId, write.value)
if (stopped) {
break
}
assertStructuredAgentResolutionCurrent(operation)
options.adoptConfigOptions(reply.configOptions)
const observed = options.read()
if (observed.current.model !== id) {
continue
}
const learned = observed.models.find((row) => row.id === id)
if (learned) {
models[index] = learned
}
conclusive = true
break
}
} catch {
assertStructuredAgentResolutionCurrent(operation)
conclusive = true
}
if (conclusive) {
attemptedModels.push(id)
}
if (deferred?.demanded().has(id)) {
progress()
} else if (deferred) {
deferred.defer(snapshot, known)
} else {
progress()
}
}
await withTimeout(sweep(), 5_000, undefined)
stopped = true
}
assertStructuredAgentResolutionCurrent(operation)
return { models, attemptedModels }
} finally {
if (!ready) {
+189 -104
View File
@@ -1,9 +1,18 @@
import { readAgentCliVersion } from '../../agent-launch/preflight/agent-cli-version-probe'
import { observeAcpForkSupport } from '../capabilities/acp-fork-support'
import { assertStructuredAgentResolutionCurrent } from '../../native-chat/agent-launch/structured-agent-command-resolution'
import {
assertStructuredAgentResolutionCurrent,
type StructuredAgentResolutionOperation
} from '../../native-chat/agent-launch/structured-agent-command-resolution'
import { homedir } from 'node:os'
import { join } from 'node:path'
import type { AgentCliVersionReader } from '../../agent-launch/preflight/agent-cli-version-probe'
import type { AgentModelCatalogProbe } from '../../native-chat/agent-model-catalog/agent-model-catalog-store'
import type {
AgentModelCatalogProgress,
AgentModelCatalogProbe,
AgentModelCatalogLookup
} from '../../native-chat/agent-model-catalog/agent-model-catalog-discovery'
import { agentModelCatalogInputStamp } from '../../native-chat/agent-model-catalog/agent-model-catalog-input-stamp'
import {
runAgentModelCatalogListing,
runAgentModelCatalogSession
@@ -34,6 +43,7 @@ import { AgentModelCatalogUnavailableError } from '../../native-chat/agent-model
import { usesModelSpecificAcpEfforts } from '../../../shared/agent-session/options/agent-model-catalog-ownership'
export type AcpModelCatalogProbeDeps = AcpLaunchInvocationDeps & {
resolveSavedEnvironment?: () => Promise<Record<string, string>>
probeVersion?: AgentCliVersionReader
connect?: (launch: ProviderProcessLaunch) => AcpModelCatalogConnection
connectSession?: (launch: ProviderProcessLaunch) => AcpModelCatalogSessionConnection
@@ -45,14 +55,17 @@ export function createAcpModelCatalogProbe(
spec: AcpLaunchSpec,
deps: AcpModelCatalogProbeDeps
): AgentModelCatalogProbe {
const probe: AgentModelCatalogProbe = async (accountHome, options) => {
const operation = options?.operation ?? { signal: options?.signal }
const resolve = async (
accountHome: Parameters<AgentModelCatalogProbe>[0],
workspace: string,
lookup: AgentModelCatalogLookup = 'recover',
operation?: StructuredAgentResolutionOperation
) => {
assertStructuredAgentResolutionCurrent(operation)
const discovery = spec.modelDiscovery
if (discovery.kind === 'unavailable') {
throw new Error(discovery.reason)
}
const workspace = options?.workspacePath
// Reject folderless session discovery before spawning even a version probe.
if (!workspace && acpListingNeedsChatFolder(discovery)) {
throw folderlessListing(spec)
@@ -64,121 +77,175 @@ export function createAcpModelCatalogProbe(
accountHome,
{
...deps,
...(lookup !== 'recover'
? {
resolveEnvironment: deps.resolveSavedEnvironment ?? deps.resolveEnvironment,
refreshEnvironmentAfterMiss: undefined
}
: {}),
operation,
resolveWorkingDirectory: () => cwd
}
)
assertStructuredAgentResolutionCurrent(operation)
const release = await checkAcpLaunchVersion(
spec,
{ command, prefixArgs, cwd, env },
acpListingDependsOnRelease(discovery),
deps.probeVersion
)
assertStructuredAgentResolutionCurrent(operation)
if (release.status === 'tooOld') {
throw new Error(`${spec.agent} at ${command} does not run structured chats`)
}
const listing = acpModelListingForRelease(discovery, release.version)
if (listing === null) {
throw new Error(`${spec.agent} at ${command} lists its models only through its chats`)
}
const launch = {
command,
args: [...prefixArgs, ...spec.args({ fullAccess: false, pluginDir: null })],
cwd,
env,
envToDelete
}
const run = { label: spec.agent, signal: options?.signal }
// OMP records every model switch as account usage; starting on the picked model records none.
const launchedModel =
spec.launchesWithModel && usesModelSpecificAcpEfforts(spec.agent)
? options?.selectedModels?.[0]
: undefined
let models: AgentSessionModelOption[]
let attemptedModels: string[] | undefined
if (listing.kind === 'command') {
const stdout = await (deps.runListing ?? runAgentModelCatalogListing)(
{
return {
identity: { command, prefixArgs, cwd, workspace },
inputStamp: JSON.stringify(
await Promise.all(
[
...new Set([cwd, ...(discovery.kind === 'command' && workspace ? [workspace] : [])])
].map((directory) =>
agentModelCatalogInputStamp({ command, prefixArgs, cwd: directory, env })
)
)
),
run: async (
signal?: AbortSignal,
runOperation = operation,
selectedModels?: readonly string[],
publish?: AgentModelCatalogProgress
) => {
const options = { signal, operation: runOperation, selectedModels }
const operation = runOperation
assertStructuredAgentResolutionCurrent(operation)
const release = await checkAcpLaunchVersion(
spec,
{ command, prefixArgs, cwd, env, operation },
acpListingDependsOnRelease(discovery),
deps.probeVersion ?? ((input) => readAgentCliVersion({ ...input, timeoutMs: null }))
)
assertStructuredAgentResolutionCurrent(operation)
if (release.status === 'tooOld') {
throw new Error(`${spec.agent} at ${command} does not run structured chats`)
}
const listing = acpModelListingForRelease(discovery, release.version)
if (listing === null) {
throw new Error(`${spec.agent} at ${command} lists its models only through its chats`)
}
const launch = {
command,
args: [...prefixArgs, ...listing.args],
cwd: workspace ?? cwd,
args: [...prefixArgs, ...spec.args({ fullAccess: false, pluginDir: null })],
cwd,
env,
envToDelete
},
{ site: `${spec.agent}-model-catalog-probe`, signal: options?.signal }
)
models = listing.parse(stdout)
} else if (listing.kind === 'session') {
if (!workspace) {
throw folderlessListing(spec)
}
// Remove disposable session storage after the process closes.
const folderArgs = listing.sessionFolderArgs
const listInSession = async (model: string | undefined) => {
const folder = folderArgs
? await acpRunFolder(join(deps.dataDirectory, 'acp-listing-sessions', spec.agent))
: null
try {
assertStructuredAgentResolutionCurrent(operation)
const args = [
...prefixArgs,
...spec.args({ fullAccess: false, pluginDir: null, model }),
...(folderArgs && folder ? folderArgs(folder.path) : [])
]
return await runAgentModelCatalogSession(
() => (deps.connectSession ?? createAcpAgentConnection)({ ...launch, args }),
(connection, ready) =>
readSessionModels(
spec,
listing,
connection,
workspace,
operation,
options?.selectedModels,
model,
spec.dialect.effortChangesModel ? ready : undefined
),
}
const run = { label: spec.agent, signal: options?.signal, timeoutMs: null }
// OMP records every model switch as account usage; starting on the picked model records none.
const launchedModel =
spec.launchesWithModel && usesModelSpecificAcpEfforts(spec.agent)
? options?.selectedModels?.[0]
: undefined
let models: AgentSessionModelOption[]
let attemptedModels: string[] | undefined
if (listing.kind === 'command') {
const stdout = await (deps.runListing ?? runAgentModelCatalogListing)(
{
command,
args: [...prefixArgs, ...listing.args],
cwd: workspace ?? cwd,
env,
envToDelete
},
{ site: `${spec.agent}-model-catalog-probe`, signal: options?.signal, timeoutMs: null }
)
models = listing.parse(stdout)
} else if (listing.kind === 'session') {
if (!workspace) {
throw folderlessListing(spec)
}
// Remove disposable session storage after the process closes.
const folderArgs = listing.sessionFolderArgs
const listInSession = async (model: string | undefined) => {
const folder = folderArgs
? await acpRunFolder(join(deps.dataDirectory, 'acp-listing-sessions', spec.agent))
: null
try {
assertStructuredAgentResolutionCurrent(operation)
const args = [
...prefixArgs,
...spec.args({ fullAccess: false, pluginDir: null, model }),
...(folderArgs && folder ? folderArgs(folder.path) : [])
]
return await runAgentModelCatalogSession<
AcpModelCatalogSessionConnection,
Awaited<ReturnType<typeof readSessionModels>>
>(
() => (deps.connectSession ?? createAcpAgentConnection)({ ...launch, args }),
(connection, ready) =>
readSessionModels(
spec,
listing,
connection,
workspace,
operation,
options?.selectedModels,
model,
spec.dialect.effortChangesModel ? ready : undefined,
(result) =>
publish?.(
{
...result,
models: [...result.models],
attemptedModels: [...result.attemptedModels],
origin: 'probe',
complete: false
},
true
),
publish?.defer && publish.demanded
? {
defer: (read, known) =>
publish.defer?.(
() => ({ ...read(), origin: 'probe', complete: false }),
known
),
demanded: publish.demanded
}
: undefined
),
run
)
} finally {
await folder?.remove()
}
}
let result: Awaited<ReturnType<typeof listInSession>>
try {
result = await listInSession(launchedModel)
} catch (error) {
// OMP exits before ACP on a model it cannot start (a stale `:level` id); list without it.
if (launchedModel === undefined || !(error instanceof AcpAgentExitedError)) {
throw error
}
assertStructuredAgentResolutionCurrent(operation)
result = { ...(await listInSession(undefined)), attemptedModels: [launchedModel] }
}
models = result.models
attemptedModels = result.attemptedModels
} else {
models = await runAgentModelCatalogSession(
() => (deps.connect ?? createAcpAgentConnection)(launch),
async (connection) => {
const initialized = await connection.initialize()
assertStructuredAgentResolutionCurrent(operation)
observeAcpForkSupport(spec.agent, initialized)
return listing.read(initialized, connection)
},
run
)
} finally {
await folder?.remove()
}
}
let result: Awaited<ReturnType<typeof listInSession>>
try {
result = await listInSession(launchedModel)
} catch (error) {
// OMP exits before ACP on a model it cannot start (a stale `:level` id); list without it.
if (launchedModel === undefined || !(error instanceof AcpAgentExitedError)) {
throw error
if (models.length === 0) {
throw new Error(`${spec.agent} listed no models`)
}
assertStructuredAgentResolutionCurrent(operation)
result = { ...(await listInSession(undefined)), attemptedModels: [launchedModel] }
return { models, origin: 'probe' as const, ...(attemptedModels ? { attemptedModels } : {}) }
}
models = result.models
attemptedModels = result.attemptedModels
} else {
models = await runAgentModelCatalogSession(
() => (deps.connect ?? createAcpAgentConnection)(launch),
async (connection) => {
const initialized = await connection.initialize()
assertStructuredAgentResolutionCurrent(operation)
observeAcpForkSupport(spec.agent, initialized)
return listing.read(initialized, connection)
},
run
)
}
if (models.length === 0) {
throw new Error(`${spec.agent} listed no models`)
}
return { models, origin: 'probe', ...(attemptedModels ? { attemptedModels } : {}) }
}
return async (home, options) => {
const probe: AgentModelCatalogProbe = async (home, options) => {
try {
return await probe(home, options)
return await (
await resolve(home, options?.workspacePath ?? '', 'recover', options?.operation)
).run(options?.signal, options?.operation, options?.selectedModels, options?.publish)
} catch (error) {
if (error instanceof AcpAgentError && error.code === -32000) {
throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' })
@@ -186,6 +253,24 @@ export function createAcpModelCatalogProbe(
throw error
}
}
return Object.assign(probe, {
resolve: async (...args: Parameters<typeof resolve>) => {
const resolved = await resolve(...args)
return {
...resolved,
run: async (...runArgs: Parameters<typeof resolved.run>) => {
try {
return await resolved.run(...runArgs)
} catch (error) {
if (error instanceof AcpAgentError && error.code === -32000) {
throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' })
}
throw error
}
}
}
}
})
}
const folderlessListing = (spec: AcpLaunchSpec) =>
@@ -0,0 +1,94 @@
import type { ProviderProcessLaunch } from '../../provider-process/provider-process-launch'
import { expect, it, vi, onTestFinished } from 'vitest'
import { acpLaunchSpecFor } from '../launch/acp-launch-registry'
import { NewSessionResponseSchema } from '../generated/acp-protocol.generated'
import { createAcpModelCatalogProbe } from './acp-model-catalog-probe'
import type { AcpModelCatalogSessionConnection } from './acp-listing-session-models'
import { AgentModelCatalogStore } from '../../native-chat/agent-model-catalog/agent-model-catalog-store'
import { createAgentModelCatalogService } from '../../native-chat/agent-model-catalog/agent-model-catalog-service'
it('starts the first OMP listing on demand collected before admission, without a second session', async () => {
const base = acpLaunchSpecFor('omp')!
const connect = vi.fn((launch: ProviderProcessLaunch): AcpModelCatalogSessionConnection => {
const modelAt = launch.args.indexOf('--model')
const current = modelAt !== -1 ? launch.args[modelAt + 1] : 'provider/default'
return {
initialize: async () => ({
protocolVersion: 1,
agentCapabilities: { sessionCapabilities: { delete: {} } }
}),
start: async () => ({
kind: 'new',
sessionId: 'listing',
response: NewSessionResponseSchema.parse({
sessionId: 'listing',
configOptions: [
{
id: 'model',
name: 'Model',
category: 'model',
type: 'select',
currentValue: current,
options: [
{ value: 'provider/default', name: 'Default' },
{ value: 'provider/selected', name: 'Selected' }
]
},
{
id: 'thinking',
name: 'Thinking',
category: 'thought_level',
type: 'select',
currentValue: 'high',
options: [{ value: 'high', name: 'High' }]
}
]
})
}),
setConfigOption: async () => {
throw new Error('OMP selects on launch')
},
deleteSession: async () => ({}),
close: async () => {}
}
})
const probe = createAcpModelCatalogProbe(
{
...base,
account: { ...base.account, environment: (_home, env) => env },
modelDiscovery: { kind: 'session', listingNamesConfiguredModel: false }
},
{
resolveEnvironment: async () => ({}),
resolveCommand: () => process.execPath,
probeVersion: async () => ({ version: '18.0.0' }),
homePath: '/account',
dataDirectory: '/unused',
connectSession: connect
}
)
const store = new AgentModelCatalogStore()
const service = createAgentModelCatalogService({
store,
getRecord: () => undefined,
getConversation: () => undefined,
drivesRecord: () => true,
resolveAccountHome: async () => ({ variable: 'PI_CODING_AGENT_DIR', path: '/account' }),
probes: { omp: probe }
})
onTestFinished(() => service.stop())
const result = await service.read({
agent: 'omp',
workspacePath: '/workspace',
selectedModel: 'provider/selected',
waitForListing: true
})
if (result.origin === 'unknown') {
throw new Error('Expected the selected model inventory')
}
expect(result.models?.find((row) => row.id === 'provider/selected')).toMatchObject({
efforts: [{ value: 'high', label: 'High' }]
})
expect(connect).toHaveBeenCalledOnce()
expect(connect.mock.calls[0]?.[0].args).toEqual(['acp', '--model', 'provider/selected'])
})
+11 -2
View File
@@ -1,3 +1,4 @@
import type { StructuredAgentResolutionOperation } from '../../native-chat/agent-launch/structured-agent-command-resolution'
import {
checkAgentCliVersion,
type AgentCliVersionCheck,
@@ -9,7 +10,9 @@ import { noteAcpForkVersion } from '../capabilities/acp-fork-support'
export async function checkAcpLaunchVersion(
spec: AcpLaunchSpec,
launch: Pick<AcpStructuredLaunch, 'command' | 'prefixArgs' | 'cwd' | 'env'>,
launch: Pick<AcpStructuredLaunch, 'command' | 'prefixArgs' | 'cwd' | 'env'> & {
operation?: StructuredAgentResolutionOperation
},
wantVersion: boolean,
read?: AgentCliVersionReader
): Promise<AgentCliVersionCheck> {
@@ -18,7 +21,13 @@ export async function checkAcpLaunchVersion(
return { status: 'unknown', version: null }
}
const result = await checkAgentCliVersion(
{ program: launch.command, prefixArgs: launch.prefixArgs, cwd: launch.cwd, env: launch.env },
{
program: launch.command,
prefixArgs: launch.prefixArgs,
cwd: launch.cwd,
env: launch.env,
operation: launch.operation
},
spec.minimumVersions ?? [],
read
)
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { checkAgentCliVersion } from './agent-cli-version-probe'
import { checkAgentCliVersion, readAgentCliVersion } from './agent-cli-version-probe'
const { runProcess } = vi.hoisted(() => ({ runProcess: vi.fn() }))
vi.mock('@orca/process-host', () => ({ runProcess }))
@@ -38,6 +38,17 @@ describe('checkAgentCliVersion', () => {
)
})
it('propagates owner cancellation instead of certifying a missing CLI', async () => {
const owner = new AbortController()
const closed = Promise.withResolvers<unknown>()
runProcess.mockReturnValue(closed.promise)
const read = checkAgentCliVersion({ ...INPUT, operation: { signal: owner.signal } }, [])
const observed = expect(read).rejects.toThrow('superseded')
owner.abort(new Error('superseded'))
closed.resolve({ code: null, signal: 'SIGTERM', stdout: '', stderr: '', timedOut: false })
await observed
})
it('reads a version printed only on stderr, as Pi 0.73 does', async () => {
prints('', { stderr: '0.73.1\n' })
expect(await checkAgentCliVersion(INPUT, ['0.84.0'])).toEqual({
@@ -47,3 +58,15 @@ describe('checkAgentCliVersion', () => {
})
})
})
it('allows a catalog version prerequisite to complete late without changing normal preflight limits', async () => {
const answer = Promise.withResolvers<unknown>()
runProcess.mockReturnValueOnce(answer.promise)
const catalog = readAgentCliVersion({ ...INPUT, timeoutMs: null })
expect(runProcess.mock.lastCall?.[0].timeoutMs).toBeNull()
answer.resolve({ code: 0, stdout: '1.18.31', stderr: '', timedOut: false })
expect(await catalog).toEqual({ version: '1.18.31' })
runProcess.mockResolvedValueOnce({ code: null, stdout: '', stderr: '', timedOut: true })
expect(await readAgentCliVersion(INPUT)).toEqual({ version: null })
expect(runProcess.mock.lastCall?.[0].timeoutMs).toBe(5_000)
})
@@ -1,3 +1,7 @@
import {
assertStructuredAgentResolutionCurrent,
type StructuredAgentResolutionOperation
} from '../../native-chat/agent-launch/structured-agent-command-resolution'
import { compareAppVersions, parseCliVersion } from '../../../shared/app-version'
import { runProcess } from '@orca/process-host'
import type { ProcessSpec } from '@orca/process-host/process-spec'
@@ -14,6 +18,8 @@ export type AgentCliVersionCheck =
| { status: 'unknown' | 'missing'; version: null }
export type AgentCliVersionInput = Pick<ProcessSpec, 'program' | 'cwd' | 'env'> & {
operation?: StructuredAgentResolutionOperation
timeoutMs?: number | null
prefixArgs?: readonly string[]
}
@@ -60,21 +66,34 @@ export async function readAgentCliVersion(
input: AgentCliVersionInput,
parseVersion: typeof parseCliVersion = parseCliVersion
): Promise<{ version: string | null; started?: false }> {
const { prefixArgs = [], ...launch } = input
const { prefixArgs = [], operation, timeoutMs = 5_000, ...launch } = input
assertStructuredAgentResolutionCurrent(operation)
let result
try {
result = await runProcess({
...launch,
args: [...prefixArgs, '--version'],
terminationBarrier: true,
timeoutMs: 5_000,
signal: operation?.signal,
timeoutMs:
timeoutMs === null
? null
: Math.max(
1,
Math.min(
timeoutMs,
operation?.deadline === undefined ? Infinity : operation.deadline - Date.now()
)
),
maxOutputBytes: 4_096,
killOnOutputLimit: true
})
} catch (error) {
assertStructuredAgentResolutionCurrent(operation)
warnRefused(input.program, `did not start (${String(error)})`)
return { version: null, started: false }
}
assertStructuredAgentResolutionCurrent(operation)
const output = `${result.stdout}\n${result.stderr}`
if (result.timedOut || result.outputTruncated || result.code !== 0) {
const why = result.timedOut
@@ -4,6 +4,8 @@ import { mkdtempSync, rmSync } from 'node:fs'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { AutomationService } from './automation-service'
import { AutomationRunLaunchError } from './launch/automation-launch'
import type { AutomationRun } from '../../shared/automations/history/automation-run'
vi.mock('electron', () => ({ app: { getPath: () => testState.dir } }))
@@ -80,4 +82,82 @@ describe('headless dispatch prompt delivery', () => {
service.stop()
}
)
it('records launch evidence once the agent terminal exists, and keeps it after retention clears the pane', async () => {
const { store, service, run, row, settle } = await heldLaunch('delivery')
await vi.waitFor(() => expect(row()?.terminalPaneKey).toBe(OPENED.terminalPaneKey))
await settle("Aider started, but Orca could not send this run's prompt.")
const launchedAt = row()?.launchedAt
expect(launchedAt).toEqual(expect.any(Number))
expect(row()?.dispatchedAt).toBeNull()
// Dead-PTY retention forgets the terminal but must not forget that the agent launched.
store.updateAutomationRun({
runId: run.id,
status: 'dispatch_failed',
error: row()?.error ?? null,
terminalSessionId: null,
terminalPaneKey: null,
terminalPtyId: null
})
expect(row()).toMatchObject({ terminalPaneKey: null, launchedAt })
service.stop()
})
})
/** Launches one automation per error, each with a launcher that throws that error. */
async function failingLaunches(errors: Error[]) {
const store = await createStore()
store.addRepo({ id: 'r1', path: '/repo', displayName: 'repo', badgeColor: '#fff', addedAt: 1 })
const pending = [...errors]
const service = new AutomationService(store, {
tickMs: 60_000,
allowRemoteHostScheduling: true,
terminalLauncher: {
launch: async () => {
throw pending.shift() ?? new Error('Unexpected launch.')
},
available: () => true
}
})
const rows: (AutomationRun | undefined)[] = []
for (const [index] of errors.entries()) {
const automation = store.createAutomation({
name: `Nightly ${index}`,
prompt: 'Fix the flaky test',
agentId: 'aider',
projectId: 'r1',
workspaceMode: 'new_per_run',
timezone: 'UTC',
rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0',
dtstart: Date.UTC(2026, 4, 12)
})
const run = await service.runNow(automation.id)
await service.runInAutomationLane(automation.id, async () => undefined)
const row = () => store.listAutomationRuns(automation.id).find((entry) => entry.id === run.id)
await vi.waitFor(() => expect(row()?.status).toBe('dispatch_failed'))
rows.push(row())
}
service.stop()
return rows
}
describe('launch evidence when the launcher throws', () => {
beforeEach(() => {
testState.dir = mkdtempSync(join(tmpdir(), 'orca-automation-launch-evidence-'))
})
afterEach(async () => {
await closeTestStores()
rmSync(testState.dir, { recursive: true, force: true })
})
it('records a partial start that opened the agent terminal, but not a launch that opened nothing', async () => {
const [partial, preOpen] = await failingLaunches([
new AutomationRunLaunchError('Prompt paste failed.', OPENED),
new Error('No window to open the terminal in.')
])
expect(partial?.launchedAt).toEqual(expect.any(Number))
// Safety: a failure before any terminal or session opened leaves no launch evidence.
expect(preOpen?.launchedAt).toBeUndefined()
})
})
@@ -75,6 +75,11 @@ export async function runAutomationLaunch(ctx: AutomationLaunchContext): Promise
status: refusal ? 'skipped_unavailable' : 'dispatch_failed',
workspaceId: automation.workspaceId,
...(error instanceof AutomationRunLaunchError ? error.target : {}),
// A partial start produced the agent's terminal or session before failing, so it
// counts as launched at the dispatch-start time.
...(error instanceof AutomationRunLaunchError && openedAgent(error.target)
? { launchedAt: dispatchedAt }
: {}),
...(automation.surface === 'chat' ? { dispatchedAt, precheckResult } : {}),
...(refusal ?? { error: describeDispatchError(error) })
}).run
@@ -82,7 +87,11 @@ export async function runAutomationLaunch(ctx: AutomationLaunchContext): Promise
const launchRunTarget = launchRunTargetOf(launch)
if (launch.delivery) {
// A startup dialog can hold delivery, so link the terminal while the run is still dispatching.
const linked = ctx.transition({ status: 'dispatching', ...launchRunTarget })
const linked = ctx.transition({
status: 'dispatching',
...launchRunTarget,
launchedAt: dispatchedAt
})
if (!linked.applied) {
return linked.run
}
@@ -91,12 +100,18 @@ export async function runAutomationLaunch(ctx: AutomationLaunchContext): Promise
})
const { error } = await launch.delivery
if (error !== null) {
return ctx.transition({ status: 'dispatch_failed', ...launchRunTarget, error }).run
return ctx.transition({
status: 'dispatch_failed',
...launchRunTarget,
launchedAt: dispatchedAt,
error
}).run
}
}
const updated = ctx.transition({
status: 'dispatched',
dispatchedAt,
launchedAt: dispatchedAt,
...launchRunTarget,
// Watched completions omit the precheck result.
...(precheckResult ? { precheckResult } : {}),
@@ -143,6 +158,13 @@ export async function runAutomationLaunch(ctx: AutomationLaunchContext): Promise
return updated.run
}
/** The launch got as far as producing the agent's terminal or chat session. */
function openedAgent(target: AutomationLaunch | undefined): boolean {
return Boolean(
target && (target.terminalPaneKey || target.terminalSessionId || target.chatSessionId)
)
}
function launchRunTargetOf(launch: AutomationLaunch) {
return {
sessionKind: launch.sessionKind ?? 'terminal',
@@ -14,8 +14,8 @@ import type {
} from '../../../shared/browser/runtime/runtime-browser-contracts'
import { BrowserError } from '../browser-error'
import {
parseShellArgs,
stripAgentBrowserTargetArgs
assertAgentBrowserPassthroughStaysOnPage,
parseShellArgs
} from './protocol/agent-browser-passthrough-arguments'
import { AgentBrowserBridgeInteractionCommands } from './commands/agent-browser-bridge-interaction-commands'
import { sendGuestCdpCommand } from '../cdp/debugger/guest-cdp-command'
@@ -272,8 +272,8 @@ export class AgentBrowserBridge extends AgentBrowserBridgeInteractionCommands {
}
async exec(command: string, worktreeId?: string, browserPageId?: string): Promise<unknown> {
// Passthrough arguments must not override Orca's selected page or CDP proxy.
const args = stripAgentBrowserTargetArgs(parseShellArgs(command.trim()))
const args = parseShellArgs(command.trim())
assertAgentBrowserPassthroughStaysOnPage(args)
return this.enqueueTargetedCommand(
worktreeId,
browserPageId,
@@ -0,0 +1,48 @@
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import {
createAgentBrowserProcessEnvironment,
pointAtEmptyAgentBrowserConfig
} from './agent-browser-process-environment'
describe('desktop agent-browser helper environment', () => {
let userDataPath: string | undefined
afterEach(() => {
if (userDataPath) {
rmSync(userDataPath, { recursive: true, force: true })
}
})
it('drops the user agent-browser settings and points config at an empty Orca file', async () => {
userDataPath = mkdtempSync(join(tmpdir(), 'orca-ab-env-'))
const { env } = createAgentBrowserProcessEnvironment({
inheritedEnv: {
PATH: '/usr/bin',
AGENT_BROWSER_PIN_TAB: '1',
agent_browser_auto_connect: '1',
AGENT_BROWSER_CDP: 'ws://127.0.0.1:9222/devtools/browser/user',
AGENT_BROWSER_CONFIG: '/home/user/agent-browser.json',
AGENT_BROWSER_SOCKET_DIR: '/tmp/ab-sockets',
AGENT_BROWSER_NAMESPACE: 'work',
AGENT_BROWSER_IDLE_TIMEOUT_MS: '5000'
},
platform: 'darwin',
userDataPath
})
await pointAtEmptyAgentBrowserConfig(env, userDataPath)
expect(env).toEqual({
PATH: '/usr/bin',
AGENT_BROWSER_SOCKET_DIR: '/tmp/ab-sockets',
AGENT_BROWSER_IDLE_TIMEOUT_MS: '5000',
AGENT_BROWSER_NO_AUTO_DIALOG: '1',
AGENT_BROWSER_CONFIG: join(userDataPath, 'agent-browser-config.json')
})
expect(
JSON.parse(readFileSync(join(userDataPath, 'agent-browser-config.json'), 'utf8'))
).toEqual({})
})
})
@@ -1,6 +1,8 @@
import { createHash } from 'node:crypto'
import { chmodSync, mkdirSync } from 'node:fs'
import { writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { isolateFromUserAgentBrowserSettings } from './agent-browser-user-settings-isolation'
const AGENT_BROWSER_SOCKET_DIRECTORY_PREFIX = 'orca-ab-'
@@ -18,12 +20,15 @@ export function createAgentBrowserProcessEnvironment(options: {
platform: NodeJS.Platform
userDataPath: string
}): AgentBrowserProcessEnvironment {
const env = { ...options.inheritedEnv }
// Commands get Orca's tokened proxy URL per spawn; an inherited target must not reach `close`.
delete env.AGENT_BROWSER_CDP
if (!env.AGENT_BROWSER_IDLE_TIMEOUT_MS?.trim()) {
env.AGENT_BROWSER_IDLE_TIMEOUT_MS = String(AGENT_BROWSER_IDLE_TIMEOUT_MS)
}
// Commands get Orca's tokened proxy URL per spawn; a user's autoConnect, pinTab or extensions
// would refuse it or open tabs behind Orca's tab model.
// A namespace moves sockets under namespaces/<name>/, where session reset and exit waits don't look.
const env = isolateFromUserAgentBrowserSettings(options.inheritedEnv, [
'AGENT_BROWSER_SOCKET_DIR'
])
env.AGENT_BROWSER_IDLE_TIMEOUT_MS =
options.inheritedEnv.AGENT_BROWSER_IDLE_TIMEOUT_MS?.trim() ||
String(AGENT_BROWSER_IDLE_TIMEOUT_MS)
// Its auto-accept dropped alert text; Orca keeps every dialog open until someone answers it.
env.AGENT_BROWSER_NO_AUTO_DIALOG = '1'
if (options.platform === 'win32' || env.AGENT_BROWSER_SOCKET_DIR?.trim()) {
@@ -40,3 +45,14 @@ export function createAgentBrowserProcessEnvironment(options: {
env.AGENT_BROWSER_SOCKET_DIR = socketDirectory
return { env, ownsSocketDirectory: true }
}
/** Until this lands, agent-browser reads the user's ~/.agent-browser/config.json and ./agent-browser.json. */
export async function pointAtEmptyAgentBrowserConfig(
env: NodeJS.ProcessEnv,
userDataPath: string
): Promise<void> {
const path = join(userDataPath, 'agent-browser-config.json')
await writeFile(path, '{}\n', { mode: 0o600 })
// An explicit config skips ~/.agent-browser/config.json and ./agent-browser.json.
env.AGENT_BROWSER_CONFIG = path
}
@@ -0,0 +1,16 @@
/**
* Orca owns its helpers' launch and target, so a user's AGENT_BROWSER_* settings (headed,
* autoConnect, extensions, pinTab...) must not reach them. Windows env keys ignore case.
* Callers also point AGENT_BROWSER_CONFIG at an Orca file so the user's config files are skipped.
*/
export function isolateFromUserAgentBrowserSettings(
inheritedEnv: NodeJS.ProcessEnv,
keepKeys: readonly string[]
): NodeJS.ProcessEnv {
return Object.fromEntries(
Object.entries(inheritedEnv).filter(([key]) => {
const upper = key.toUpperCase()
return !upper.startsWith('AGENT_BROWSER_') || keepKeys.includes(upper)
})
)
}
@@ -190,6 +190,10 @@ async function settle(
: []
)
)
const print = events.find((event) => event.kind === 'print')
if (print) {
effects.blockedPrint = { origin: clip(print.origin) }
}
effects.consoleErrors = firstItems(watch.consoleErrors.map((error) => clip(error)))
return effects
}
@@ -0,0 +1,43 @@
import { describe, expect, it } from 'vitest'
import { assertAgentBrowserPassthroughStaysOnPage } from './agent-browser-passthrough-arguments'
describe('assertAgentBrowserPassthroughStaysOnPage', () => {
it.each([
[['connect', '9222']],
[['tab', 'new']],
[['window', 'new']],
[['close']],
[['set', 'viewport', '800', '600']],
[['set', 'device', 'iPhone 15']],
[['click', '@e1', '--new-tab']],
[['get', 'text', '@e3', '--cdp', 'ws://127.0.0.1:9222']],
[['get', 'url', '--session=other']],
[['snapshot', '--auto-connect']],
[['open', 'https://example.com', '-p', 'browserbase']],
[['fill', '@e1', '--headed']],
[['batch', 'get title', 'connect 9222']],
[['batch', '--bail', 'tab new']],
[['batch', 'c\\onnect 9222']],
[['batch', 'close\\']],
[['inspect']],
[['stream', 'enable', '--port', '9999']],
[['auth', 'login', 'github', '--url', 'https://example.com/login']],
[['batch', 'batch "tab new"']],
[['batch', 'get url --cdp 9222']]
])('refuses %j', (args) => {
expect(() => assertAgentBrowserPassthroughStaysOnPage(args)).toThrow(
expect.objectContaining({ code: 'invalid_argument' })
)
})
it.each([
[['get', 'text', '@e3']],
[['read']],
[['is', 'visible', '@e3']],
[['set', 'media', 'dark']],
[['wait', '--text', 'Done']],
[['batch', '--bail', 'get title', 'snapshot -i']]
])('runs %j', (args) => {
expect(() => assertAgentBrowserPassthroughStaysOnPage(args)).not.toThrow()
})
})
@@ -1,3 +1,5 @@
import { BrowserError } from '../../browser-error'
export function parseShellArgs(input: string): string[] {
const args: string[] = []
let current = ''
@@ -25,18 +27,179 @@ export function parseShellArgs(input: string): string[] {
return args
}
export function stripAgentBrowserTargetArgs(args: string[]): string[] {
const stripped: string[] = []
for (let index = 0; index < args.length; index++) {
const arg = args[index]
if (arg === '--cdp' || arg === '--session') {
index++
continue
/** Splits a command line the way agent-browser splits each `batch` command (backslash escapes, quotes). */
export function splitAgentBrowserCommandLine(input: string): string[] {
const args: string[] = []
let value = ''
let quote: '"' | "'" | null = null
let escaped = false
for (const char of input.trim()) {
if (escaped) {
value += char
escaped = false
} else if (char === '\\' && quote !== "'") {
escaped = true
} else if (quote) {
if (char === quote) {
quote = null
} else {
value += char
}
} else if (char === '"' || char === "'") {
quote = char
} else if (/\s/.test(char)) {
if (value) {
args.push(value)
value = ''
}
} else {
value += char
}
}
// Like agent-browser, drop a trailing lone backslash: `close\` runs `close`.
if (quote) {
throw new BrowserError('invalid_argument', 'Browser command has an unclosed quote.')
}
if (value) {
args.push(value)
}
return args
}
// agent-browser's global options (0.39 through 0.43). It reads them anywhere in argv, even after
// a command, and they choose the browser, session and launch, which Orca owns.
const GLOBAL_OPTIONS = new Set([
'--action-policy',
'--allow-file-access',
'--allowed-domains',
'--annotate',
'--args',
'--auto-connect',
'--ca-cert',
'--capture',
'--cdp',
'--cdp-headers',
'--color-scheme',
'--config',
'--confirm-actions',
'--confirm-interactive',
'--content-boundaries',
'--debug',
'--device',
'--download-path',
'--enable',
'--engine',
'--executable-path',
'--extension',
'--fix',
'--headed',
'--headers',
'--hide-scrollbars',
'--idle-timeout',
'--ignore-https-errors',
'--init-script',
'--input-mode',
'--json',
'--max-output',
'--model',
'--namespace',
'--no-auto-connect',
'--no-auto-dialog',
'--no-ca-cert',
'--no-pin-tab',
'--no-webmcp',
'--offline',
'--pin-tab',
'--profile',
'--provider',
'--proxy',
'--proxy-bypass',
'--quick',
'--quiet',
'--relaunch',
'--restore',
'--restore-check-fn',
'--restore-check-text',
'--restore-check-url',
'--restore-save',
'--screenshot-dir',
'--screenshot-format',
'--screenshot-quality',
'--session',
'--session-name',
'--state',
'--use-system-ca',
'--user-agent',
'--verbose',
'--webgpu',
'-p',
'-q',
'-v'
])
const TAB_HINT = 'Use the typed tab commands (tab list, tab create, tab switch, tab close).'
const VIEWPORT_HINT = 'Use the typed viewport or set device command.'
const NOT_THE_PAGE = 'It acts outside the selected Orca page.'
// Commands that leave Orca's page: connect/tab/window/close retarget or create targets behind
// Orca's tab model; inspect opens the system browser; stream serves an unguarded command relay;
// auth types agent-browser's saved logins into any URL; the rest act on the machine.
const REFUSED_COMMANDS = new Map<string, string>([
['connect', NOT_THE_PAGE],
['inspect', NOT_THE_PAGE],
['stream', NOT_THE_PAGE],
['auth', NOT_THE_PAGE],
['tab', TAB_HINT],
['window', TAB_HINT],
['close', TAB_HINT],
['quit', TAB_HINT],
['exit', TAB_HINT],
['state', NOT_THE_PAGE],
['session', NOT_THE_PAGE],
['install', NOT_THE_PAGE],
['upgrade', NOT_THE_PAGE],
['doctor', NOT_THE_PAGE],
['dashboard', NOT_THE_PAGE],
['profiles', NOT_THE_PAGE],
['plugin', NOT_THE_PAGE],
['plugins', NOT_THE_PAGE],
['mcp', NOT_THE_PAGE],
['chat', NOT_THE_PAGE]
])
function refuse(what: string, hint: string): never {
throw new BrowserError('invalid_argument', `exec does not run ${what}. ${hint}`)
}
/** Refuses raw agent-browser lines that would move the helper off Orca's selected page. */
export function assertAgentBrowserPassthroughStaysOnPage(args: readonly string[]): void {
for (const arg of args) {
const option = arg.split('=')[0]
if (GLOBAL_OPTIONS.has(option)) {
refuse(`agent-browser's global option ${option}`, 'Orca sets global options itself.')
}
}
const [command, subcommand] = args
if (!command) {
return
}
const refusal = REFUSED_COMMANDS.get(command)
if (refusal) {
refuse(`\`${command}\``, refusal)
}
// Both resize the real window around the page (Browser.setContentsSize).
if (command === 'set' && (subcommand === 'viewport' || subcommand === 'device')) {
refuse(`\`set ${subcommand}\``, VIEWPORT_HINT)
}
if (command === 'click' && args.includes('--new-tab')) {
refuse('`click --new-tab`', TAB_HINT)
}
if (command === 'batch') {
// agent-browser runs every operand except `--bail` as its own command line.
for (const step of args.slice(1)) {
if (step !== '--bail') {
assertAgentBrowserPassthroughStaysOnPage(splitAgentBrowserCommandLine(step))
}
}
if (arg.startsWith('--cdp=') || arg.startsWith('--session=')) {
continue
}
stripped.push(arg)
}
return stripped
}
@@ -5,6 +5,7 @@ import type { AgentBrowserCleanupOptions } from './agent-browser-cleanup-options
export abstract class AgentBrowserBridgeShutdown extends AgentBrowserBridgeLifecycle {
async sweepOrphanedSessions(): Promise<string[]> {
await this.agentBrowserConfigReady
return sweepOrphanedAgentBrowserSessions({
binaryPath: this.agentBrowserBin,
env: this.agentBrowserEnv,
@@ -3,7 +3,10 @@ import type { ChildProcess } from 'node:child_process'
import { BrowserError } from '../../browser-error'
import type { BrowserManager } from '../../guest-lifecycle/manager/browser-manager'
import type { BrowserPopupPage } from '../../guest-lifecycle/popups/browser-popup-ownership'
import { createAgentBrowserProcessEnvironment } from '../helper-process/agent-browser-process-environment'
import {
createAgentBrowserProcessEnvironment,
pointAtEmptyAgentBrowserConfig
} from '../helper-process/agent-browser-process-environment'
import { resolveAgentBrowserBinary } from '../helper-process/agent-browser-binary'
import type { QueuedCommand } from '../command-queue/agent-browser-queued-command'
import type { SessionState } from '../session-lifecycle/agent-browser-session-state'
@@ -22,6 +25,8 @@ export abstract class AgentBrowserBridgeState {
protected readonly processingQueues = new Set<string>()
protected readonly agentBrowserBin: string
protected readonly agentBrowserEnv: NodeJS.ProcessEnv
// Startup spawns await it; agent commands come long after it lands.
protected readonly agentBrowserConfigReady: Promise<void>
protected readonly ownsAgentBrowserSocketDirectory: boolean
// Null disables idle-replacement detection when daemon lifetime is unbounded.
protected readonly agentBrowserIdleTimeoutMs: number | null
@@ -38,12 +43,18 @@ export abstract class AgentBrowserBridgeState {
protected readonly options: AgentBrowserBridgeOptions = {}
) {
this.agentBrowserBin = resolveAgentBrowserBinary()
const userDataPath = app.getPath('userData')
const processEnvironment = createAgentBrowserProcessEnvironment({
inheritedEnv: process.env,
platform: process.platform,
userDataPath: app.getPath('userData')
userDataPath
})
this.agentBrowserEnv = processEnvironment.env
// A failed write keeps agent-browser's own lookup; a missing file would fail every command.
this.agentBrowserConfigReady = pointAtEmptyAgentBrowserConfig(
this.agentBrowserEnv,
userDataPath
).catch(() => {})
this.ownsAgentBrowserSocketDirectory = processEnvironment.ownsSocketDirectory
const idleTimeoutMs = Number(this.agentBrowserEnv.AGENT_BROWSER_IDLE_TIMEOUT_MS)
this.agentBrowserIdleTimeoutMs = idleTimeoutMs > 0 ? idleTimeoutMs : null
@@ -5,6 +5,7 @@ import {
isSameExtensionNavigation
} from '../../../../shared/browser/extensions/browser-extension-url'
import { allowPageUnload } from '../../guest-lifecycle/navigation/browser-guest-unload-policy'
import { recordBrowserSessionInput } from '../../guest-lifecycle/input/browser-guest-input-gesture'
import { BrowserExtensionApiError } from '../api-runtime/browser-extension-namespace-registry'
import {
BROWSER_EXTENSION_POPUP_MAX,
@@ -136,6 +137,7 @@ export function openBrowserExtensionPopup(options: BrowserExtensionPopupOptions)
}
current = popup
const contents = window.webContents
recordBrowserSessionInput(contents)
contents.on('will-prevent-unload', allowPageUnload)
// Why measure in the page: preferred-size-changed only reports while the window paints, and its
@@ -188,7 +190,7 @@ export function openBrowserExtensionPopup(options: BrowserExtensionPopupOptions)
void watchSize().then(() => {
if (!window.isDestroyed()) {
presented = true
presentBrowserExtensionWindow(window, true)
presentBrowserExtensionWindow(window, 'focused')
}
})
})
@@ -134,7 +134,7 @@ function openPopupForPage(
openTab: (rawUrl) => {
const resolved = resolveBrowserExtensionRequestedUrl(rawUrl, extension.id)
if (resolved && (resolved.kind !== 'extension' || resolved.extensionId === extension.id)) {
void openBrowserExtensionTab(page.profileId, resolved.url, true).catch(() => {})
void openBrowserExtensionTab(page.profileId, resolved.url, { active: true }).catch(() => {})
}
}
})
@@ -95,7 +95,7 @@ const extensionSession = {
on: vi.fn()
},
serviceWorkers: { on: vi.fn(), getAllRunning: () => ({}), getWorkerFromVersionID: vi.fn() },
protocol: { handle: vi.fn(), unhandle: vi.fn() }
protocol: { handle: vi.fn(), unhandle: vi.fn(), isProtocolHandled: vi.fn(() => true) }
}
const handled: unknown[][] = []
@@ -276,3 +276,29 @@ describe('extension subframe listener retirement', () => {
}
})
})
it('forgets the extension protocol twice without Electron refusing the second unhandle', () => {
const handled = new Set<string>()
const unhandle = vi.fn((scheme: string) => {
if (!handled.delete(scheme)) {
throw new Error(`Failed to unhandle protocol: ${scheme}`)
}
})
const sess = {
...extensionSession,
protocol: {
handle: (scheme: string) => {
handled.add(scheme)
},
isProtocolHandled: (scheme: string) => handled.has(scheme),
unhandle
}
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this session copies the host's stub and replaces only protocol, which the retirement path calls.
const session = sess as unknown as Electron.Session
host.attachBrowserExtensionApiHostToSession(session, 'deleted-profile')
host.forgetBrowserExtensionContexts(session)
expect(() => host.forgetBrowserExtensionContexts(session)).not.toThrow()
expect(unhandle).toHaveBeenCalledTimes(1)
expect(unhandle).toHaveBeenCalledWith('orca-extension-api')
})
@@ -186,7 +186,10 @@ export function attachBrowserExtensionApiHostToSession(
export function forgetBrowserExtensionContexts(sess: Electron.Session): void {
forgetHostSession(sess)
sess.protocol.unhandle(BROWSER_EXTENSION_API_SCHEME)
// Why: profile delete forgets this host twice, and Electron throws if the scheme is already gone.
if (sess.protocol.isProtocolHandled(BROWSER_EXTENSION_API_SCHEME)) {
sess.protocol.unhandle(BROWSER_EXTENSION_API_SCHEME)
}
}
export function getBrowserExtensionProfileIdForSession(sess: Electron.Session): string | null {
@@ -81,7 +81,8 @@ function makeSession(ids: string[]) {
handle: (_scheme: string, handler: ProtocolHandler) => {
handlers.push(handler)
},
unhandle: vi.fn()
unhandle: vi.fn(),
isProtocolHandled: () => true
}
}
}
+10 -1
View File
@@ -21,6 +21,7 @@ import {
import { requireCallerTab } from '../tabs-and-windows/browser-extension-tab-model'
import { getPinnedBrowserExtensionIds } from '../actions/browser-extension-toolbar-pins'
import { requestBrowserExtensionPopup } from '../actions/browser-extension-toolbar'
import { findBrowserExtensionCallerAgentPage } from '../tabs-and-windows/browser-extension-agent-caller'
const tabIdSchema = z.number().int().nonnegative()
const tabDetailsSchema = z.object({ tabId: tabIdSchema.optional() }).loose()
@@ -200,7 +201,15 @@ function createActionNamespace(): BrowserExtensionNamespace {
})),
openPopup: defineBrowserExtensionMethod(
z.tuple([z.object({ windowId: z.number().int() }).loose().optional()]),
(caller) => requestBrowserExtensionPopup(caller.session, caller.extension.id)
(caller) => {
// Why: the popup opens focused on the user's own tab, and an agent's call never takes it over.
if (findBrowserExtensionCallerAgentPage(caller)) {
throw new BrowserExtensionApiError(
'An agent is using this browser profile, so the popup was not opened'
)
}
return requestBrowserExtensionPopup(caller.session, caller.extension.id)
}
)
},
events: ['onClicked', 'onUserSettingsChanged']
@@ -141,7 +141,7 @@ const extensionSession = {
(_filter: unknown, callback: (details: Record<string, unknown>) => void) => callback
)
},
protocol: { handle: vi.fn(), unhandle: vi.fn() },
protocol: { handle: vi.fn(), unhandle: vi.fn(), isProtocolHandled: vi.fn(() => true) },
extensions: Object.assign(new EventEmitter(), {
getExtension: (id: string) => extensions.get(id) ?? null,
getAllExtensions: () => [...extensions.values()]
+6 -1
View File
@@ -1,4 +1,5 @@
import { openBrowserExtensionOptionsPage } from '../tabs-and-windows/browser-extension-options-page'
import { placeBrowserExtensionCallerTab } from '../tabs-and-windows/browser-extension-agent-caller'
import { z } from 'zod'
import {
defineBrowserExtensionMethod,
@@ -10,7 +11,11 @@ export function registerRuntimeNamespace(): void {
registerBrowserExtensionNamespace('runtime', {
methods: {
openOptionsPage: defineBrowserExtensionMethod(z.tuple([]), (caller) =>
openBrowserExtensionOptionsPage(caller.browserProfileId, caller.extension)
openBrowserExtensionOptionsPage(
caller.browserProfileId,
caller.extension,
placeBrowserExtensionCallerTab(caller, true)
)
)
},
events: ['onInstalled', 'onStartup']
+22 -4
View File
@@ -34,6 +34,10 @@ import {
listBrowserExtensionDiscardedTabs,
type BrowserExtensionDiscardedTabEntry
} from '../tabs-and-windows/browser-extension-discarded-tab-model'
import {
findBrowserExtensionCallerAgentPage,
placeBrowserExtensionCallerTab
} from '../tabs-and-windows/browser-extension-agent-caller'
import { captureVisibleTabMethod } from './tabs-capture'
import { tabsPageMethods } from './tabs-page-methods'
import {
@@ -82,7 +86,11 @@ async function createTab(
url: string,
active: boolean
): Promise<chrome.tabs.Tab> {
const tabId = await openBrowserExtensionTab(caller.browserProfileId, url, active)
const tabId = await openBrowserExtensionTab(
caller.browserProfileId,
url,
placeBrowserExtensionCallerTab(caller, active)
)
return describeBrowserExtensionTab(caller.extension, requireCallerTab(caller, tabId))
}
@@ -129,6 +137,17 @@ function queryTabs(
.sort((a, b) => a.windowId - b.windowId || a.index - b.index)
}
/** Highlighting is activating in a one-tab selection (Chrome); an agent's call never switches tabs. */
function mayActivateTab(
caller: BrowserExtensionCaller,
properties: z.infer<typeof updatePropertiesSchema>
): boolean {
return (
Boolean(properties.active || properties.highlighted) &&
!findBrowserExtensionCallerAgentPage(caller)
)
}
/** Activation shows the page, which loads it (onReplaced follows); a URL needs a loaded page. */
async function updateDiscardedTab(
caller: BrowserExtensionCaller,
@@ -138,7 +157,7 @@ async function updateDiscardedTab(
if (properties.url !== undefined) {
throw new BrowserExtensionApiError(`Cannot navigate tab ${entry.tab.tabId} until it loads`)
}
if (properties.active || properties.highlighted) {
if (mayActivateTab(caller, properties)) {
await activateBrowserExtensionDiscardedTab(entry)
}
const current = findBrowserExtensionDiscardedTab(caller.session, entry.tab.tabId)
@@ -229,8 +248,7 @@ export function registerTabsNamespace(): void {
if (properties.muted !== undefined) {
entry.contents.setAudioMuted(properties.muted)
}
// Chrome: highlighting a tab in a one-tab selection model is activating it.
if (properties.active || properties.highlighted) {
if (mayActivateTab(caller, properties)) {
await activateBrowserExtensionTab(entry)
}
return describeBrowserExtensionTab(caller.extension, requireCallerTab(caller, id))
+31 -15
View File
@@ -22,6 +22,7 @@ import {
type BrowserExtensionWindowEntry
} from '../tabs-and-windows/browser-extension-window-model'
import { requireOpenableUrl } from '../tabs-and-windows/browser-extension-requested-navigation'
import { findBrowserExtensionCallerAgentPage } from '../tabs-and-windows/browser-extension-agent-caller'
const windowIdSchema = z.number().int()
const queryOptionsSchema = z
@@ -85,23 +86,34 @@ function requireWindow(
return entry
}
function applyWindowState(
window: Electron.BrowserWindow,
state: z.infer<typeof updateInfoSchema>['state']
): void {
if (state === 'minimized') {
window.minimize()
} else if (state === 'maximized') {
window.maximize()
} else if (state === 'fullscreen') {
window.setFullScreen(true)
} else if (state === 'normal') {
window.setFullScreen(false)
window.unmaximize()
window.restore()
}
}
function applyWindowUpdate(
entry: BrowserExtensionWindowEntry,
info: z.infer<typeof updateInfoSchema>
info: z.infer<typeof updateInfoSchema>,
byAgent: boolean
): void {
const { window } = entry
// Why: extensions may focus Orca's main window but never move, resize or minimize it.
if (!entry.isMain) {
if (info.state === 'minimized') {
window.minimize()
} else if (info.state === 'maximized') {
window.maximize()
} else if (info.state === 'fullscreen') {
window.setFullScreen(true)
} else if (info.state === 'normal') {
window.setFullScreen(false)
window.unmaximize()
window.restore()
// Why not for an agent: restoring, maximizing or fullscreen can show or raise the window.
if (!byAgent || info.state === 'minimized') {
applyWindowState(window, info.state)
}
const bounds = window.getBounds()
window.setBounds({
@@ -114,8 +126,8 @@ function applyWindowUpdate(
if (info.drawAttention !== undefined) {
window.flashFrame(info.drawAttention)
}
if (info.focused) {
presentBrowserExtensionWindow(window, true)
if (info.focused && !byAgent) {
presentBrowserExtensionWindow(window, 'focused')
}
}
@@ -161,7 +173,11 @@ export function registerWindowsNamespace(): void {
top: data?.top,
width: data?.width,
height: data?.height,
focused: data?.focused !== false,
presentation: findBrowserExtensionCallerAgentPage(caller)
? 'whenAppActive'
: data?.focused === false
? 'unfocused'
: 'focused',
state: data?.state
})
return describe(caller, requireWindow(caller, window.id), { populate: true })
@@ -171,7 +187,7 @@ export function registerWindowsNamespace(): void {
z.tuple([windowIdSchema, updateInfoSchema]),
(caller, windowId, info) => {
const entry = requireWindow(caller, windowId)
applyWindowUpdate(entry, info)
applyWindowUpdate(entry, info, findBrowserExtensionCallerAgentPage(caller) !== null)
return describe(caller, entry, undefined)
}
),
@@ -258,6 +258,6 @@ export function openBrowserExtensionOptions(
if (!extension) {
throw new Error('Turn the extension on to open its options')
}
await openBrowserExtensionOptionsPage(browserProfileId, extension)
await openBrowserExtensionOptionsPage(browserProfileId, extension, { active: true })
})
}
@@ -2,7 +2,7 @@ import type { BrowserExtensionCaller } from '../api-runtime/browser-extension-na
import { getHostSessionState } from '../api-runtime/browser-extension-contexts'
import { getBrowserExtensionContextSignal } from '../api-runtime/browser-extension-context-lifetime'
const USER_GESTURE_WINDOW_MS = 5_000
export const BROWSER_EXTENSION_USER_GESTURE_WINDOW_MS = 5_000
const recentGestures = new WeakMap<AbortSignal, number>()
@@ -22,15 +22,21 @@ export function noteBrowserExtensionUserGesture(
recentGestures.set(signal, Date.now())
}
export function hasBrowserExtensionUserGesture(caller: BrowserExtensionCaller): boolean {
if (caller.userGesture === true) {
return true
}
/** When Orca last handed the caller's extension a gesture, if still within the gesture window. */
export function getBrowserExtensionUserGestureAt(caller: BrowserExtensionCaller): number | null {
const state = getHostSessionState(caller.session)
if (!state) {
return false
return null
}
const signal = getBrowserExtensionContextSignal(state, caller.extension.id)
const at = recentGestures.get(signal)
return at !== undefined && !signal.aborted && Date.now() - at <= USER_GESTURE_WINDOW_MS
return at !== undefined &&
!signal.aborted &&
Date.now() - at <= BROWSER_EXTENSION_USER_GESTURE_WINDOW_MS
? at
: null
}
export function hasBrowserExtensionUserGesture(caller: BrowserExtensionCaller): boolean {
return caller.userGesture === true || getBrowserExtensionUserGestureAt(caller) !== null
}
@@ -225,6 +225,10 @@ describe('deleted profile action state', () => {
if (failedUnload) {
mocks.loaded.set(ID, { id: ID })
getBrowserExtensionProfileStateStore().update(profileId, ID, (state) => ({
...state,
workerSignature: 'left-behind'
}))
const unload = vi
.spyOn(mocks.session.extensions, 'removeExtension')
.mockImplementationOnce(() => {
@@ -234,6 +238,7 @@ describe('deleted profile action state', () => {
await expect(loader.detachBrowserExtensionsFromProfileSession(profile)).rejects.toThrow(
'native unload failed'
)
expect(getBrowserExtensionProfileStateStore().readProfile(profileId)).toEqual({})
} finally {
unload.mockRestore()
mocks.loaded.delete(ID)
@@ -158,10 +158,10 @@ export async function detachBrowserExtensionsFromProfileSession(
}
} finally {
forgetBrowserExtensionActionProfile(profile.id)
getBrowserExtensionProfileStateStore().forgetProfile(profile.id)
getBrowserExtensionSyncStorageStore().forgetProfile(profile.id)
getBrowserExtensionDnrStore().forgetProfile(profile.id)
}
getBrowserExtensionProfileStateStore().forgetProfile(profile.id)
getBrowserExtensionSyncStorageStore().forgetProfile(profile.id)
getBrowserExtensionDnrStore().forgetProfile(profile.id)
}
/** The extensions setting turned off: every profile drops its extensions until it turns on. */
@@ -0,0 +1,112 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const browserMocks = vi.hoisted(() => ({
appGetPathMock: vi.fn(() => '/downloads'),
shellOpenExternalMock: vi.fn(),
browserWindowFromWebContentsMock: vi.fn(),
menuBuildFromTemplateMock: vi.fn(),
guestOffMock: vi.fn(),
guestOnMock: vi.fn(),
guestSetBackgroundThrottlingMock: vi.fn(),
guestSetWindowOpenHandlerMock: vi.fn(),
guestOpenDevToolsMock: vi.fn(),
webContentsFromIdMock: vi.fn(),
screenGetCursorScreenPointMock: vi.fn(() => ({ x: 0, y: 0 })),
openPopupWithOriginBarMock: vi.fn()
}))
vi.mock('electron', () => ({
app: { prependListener: vi.fn(), off: vi.fn(), getPath: browserMocks.appGetPathMock },
BrowserWindow: { fromWebContents: browserMocks.browserWindowFromWebContentsMock },
clipboard: { writeText: vi.fn() },
shell: { openExternal: browserMocks.shellOpenExternalMock },
Menu: { buildFromTemplate: browserMocks.menuBuildFromTemplateMock },
screen: { getCursorScreenPoint: browserMocks.screenGetCursorScreenPointMock },
webContents: { fromId: browserMocks.webContentsFromIdMock }
}))
import { browserManager } from '../../guest-lifecycle/manager/browser-manager'
import { runWithActiveRpcCaller } from '../../../runtime/rpc/callers/active-call'
import { withAgentInputInjection } from '../../cdp/input/agent-input-injection'
import {
guestNavigationMethods,
rendererWebContentsId,
resetBrowserManagerMocks,
resetBrowserManagerState
} from '../../guest-lifecycle/manager/browser-manager-test-harness'
import type { BrowserExtensionCaller } from '../api-runtime/browser-extension-namespace-registry'
import { placeBrowserExtensionCallerTab } from './browser-extension-agent-caller'
const workSession = { name: 'work' }
const otherSession = { name: 'other' }
/** Registers a guest page; `click` delivers an activating input to it. */
function registerPage(browserPageId: string, id: number, worktreeId: string) {
const on = vi.fn()
const guest = {
...guestNavigationMethods(),
id,
session: workSession,
isDestroyed: vi.fn(() => false),
isCurrentlyAudible: vi.fn(() => false),
getType: vi.fn(() => 'webview'),
setBackgroundThrottling: vi.fn(),
setWindowOpenHandler: vi.fn(),
setIgnoreMenuShortcuts: vi.fn(),
on,
prependListener: vi.fn(),
off: vi.fn(),
openDevTools: vi.fn()
}
const previous = browserMocks.webContentsFromIdMock.getMockImplementation()
browserMocks.webContentsFromIdMock.mockImplementation((lookup: number) =>
lookup === id ? guest : previous?.(lookup)
)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the manager and input tracker call only the methods stubbed above.
const contents = guest as unknown as Electron.WebContents
browserManager.attachGuestPolicies(contents)
browserManager.registerGuest({
browserPageId,
worktreeId,
webContentsId: id,
rendererWebContentsId
})
const onInput = on.mock.calls.find(([event]) => event === 'input-event')?.[1]
return { contents, click: () => onInput({}, { type: 'mouseUp' }) }
}
function caller(session: typeof workSession, extra: Partial<BrowserExtensionCaller> = {}) {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: placement reads session, webContents and userGesture only.
return { session, browserProfileId: 'work', fromWorker: true, ...extra } as BrowserExtensionCaller
}
beforeEach(() => {
resetBrowserManagerMocks(browserMocks)
resetBrowserManagerState()
browserMocks.webContentsFromIdMock.mockImplementation(() => null)
})
describe('an extension tab set off while an agent drives a page in its profile', () => {
it("opens in the background in the agent's workspace, unless the user set it off", async () => {
const userPage = registerPage('user-page', 11, 'wt-user')
const agentPage = registerPage('agent-page', 12, 'wt-agent')
const command = runWithActiveRpcCaller({ kind: 'local-cli' }, () =>
browserManager.beginAgentCommand('agent-page')
)
const placeFromWorker = () => placeBrowserExtensionCallerTab(caller(workSession), true)
const agentPlacement = { active: false, worktreeId: 'wt-agent' }
expect(placeFromWorker()).toEqual(agentPlacement)
expect(placeBrowserExtensionCallerTab(caller(otherSession), true)).toEqual({ active: true })
expect(
placeBrowserExtensionCallerTab(caller(workSession, { userGesture: true }), true)
).toEqual({ active: true })
userPage.click()
expect(placeFromWorker()).toEqual({ active: true })
await withAgentInputInjection(agentPage.contents, async () => agentPage.click())
expect(placeFromWorker()).toEqual(agentPlacement)
command.end()
})
})
@@ -0,0 +1,56 @@
import { browserManager } from '../../guest-lifecycle/manager/browser-manager'
import { getLastBrowserGuestInputInSession } from '../../guest-lifecycle/input/browser-guest-input-gesture'
import type { BrowserExtensionCaller } from '../api-runtime/browser-extension-namespace-registry'
import {
BROWSER_EXTENSION_USER_GESTURE_WINDOW_MS,
getBrowserExtensionUserGestureAt
} from '../permissions/browser-extension-user-gesture'
import type { openBrowserExtensionTab } from './browser-extension-tab-model'
export type BrowserExtensionTabPlacement = Parameters<typeof openBrowserExtensionTab>[2]
type AgentPage = { worktreeId: string | null }
function pageOfGuest(guestWebContentsId: number): AgentPage {
const tabId = browserManager.getOwningTabIdForGuestWebContentsId(guestWebContentsId)
return { worktreeId: (tabId && browserManager.getWorktreeIdForTab(tabId)) ?? null }
}
/**
* The agent's page this call acts for, or null when the user set it off. An extension reacts to
* its profile's pages (a wallet popup, an OAuth tab) without saying which input started it, so
* the newest recent click, key or Orca-delivered gesture decides, then whether an agent leads.
*/
export function findBrowserExtensionCallerAgentPage(
caller: BrowserExtensionCaller
): AgentPage | null {
const now = Date.now()
const lastInput = getLastBrowserGuestInputInSession(caller.session)
const input =
lastInput && now - lastInput.at <= BROWSER_EXTENSION_USER_GESTURE_WINDOW_MS ? lastInput : null
const gestureAt = getBrowserExtensionUserGestureAt(caller)
if (gestureAt !== null && (!input || gestureAt >= input.at)) {
return null
}
if (input) {
// Agents and remote viewers send injected input; only the desktop user's own may surface.
return input.byAgent ? pageOfGuest(input.guestWebContentsId) : null
}
// Why before userGesture: an agent's click on an extension page also activates that page.
if (caller.webContents && browserManager.isAgentLeadingGuest(caller.webContents.id)) {
return pageOfGuest(caller.webContents.id)
}
return caller.userGesture ? null : browserManager.findAgentPageInSession(caller.session)
}
/** An agent's extension tab opens in the background beside its page, never in the user's view. */
export function placeBrowserExtensionCallerTab(
caller: BrowserExtensionCaller,
active: boolean
): BrowserExtensionTabPlacement {
const agentPage = findBrowserExtensionCallerAgentPage(caller)
if (!agentPage) {
return { active }
}
return { active: false, ...(agentPage.worktreeId ? { worktreeId: agentPage.worktreeId } : {}) }
}
@@ -46,7 +46,9 @@ vi.mock('../../guest-lifecycle/manager/browser-manager', () => ({
getWebContentsIdByTabId: () =>
new Map([...mocks.pageByGuest].map(([id, pageId]) => [pageId, id])),
getTabIdForWebContentsId: (id: number) => mocks.pageByGuest.get(id) ?? null,
getWorktreeIdForTab: () => undefined
getWorktreeIdForTab: () => undefined,
isAgentLeadingGuest: () => false,
findAgentPageInSession: () => null
}
}))
vi.mock('../../../runtime/browser/tabs/browser-tab-renderer-create', () => ({}))
@@ -1,6 +1,7 @@
import { getChromeExtensionOrigin } from '../../../../shared/browser/extensions/browser-extension-url'
import { BrowserExtensionApiError } from '../api-runtime/browser-extension-namespace-registry'
import { openBrowserExtensionTab } from './browser-extension-tab-model'
import type { BrowserExtensionTabPlacement } from './browser-extension-agent-caller'
function optionsPagePath(extension: Electron.Extension): string | null {
const manifest: { options_ui?: { page?: unknown }; options_page?: unknown } =
@@ -12,12 +13,13 @@ function optionsPagePath(extension: Electron.Extension): string | null {
/** Opens the extension's options page as an Orca tab in the given browser profile. */
export async function openBrowserExtensionOptionsPage(
browserProfileId: string,
extension: Electron.Extension
extension: Electron.Extension,
placement: BrowserExtensionTabPlacement
): Promise<void> {
const page = optionsPagePath(extension)
if (!page) {
throw new BrowserExtensionApiError('This extension has no options page')
}
const url = new URL(page.replace(/^\/+/, ''), `${getChromeExtensionOrigin(extension.id)}/`)
await openBrowserExtensionTab(browserProfileId, url.toString(), true)
await openBrowserExtensionTab(browserProfileId, url.toString(), placement)
}
@@ -79,7 +79,7 @@ describe('an extension tab opened while Orca starts', () => {
mocks.pages.set('welcome', { id: 21, session: mocks.workSession, worktreeId: 'wt-a' })
return { browserPageId: 'welcome' }
})
const opening = openBrowserExtensionTab('work', 'https://welcome.test', true)
const opening = openBrowserExtensionTab('work', 'https://welcome.test', { active: true })
await Promise.resolve()
expect(mocks.createTab).not.toHaveBeenCalled()
window.rendererReady()
@@ -87,7 +87,7 @@ describe('an extension tab opened while Orca starts', () => {
// A reload waits for the new document to restore again.
window.reload()
const reopening = openBrowserExtensionTab('work', 'https://welcome.test', true)
const reopening = openBrowserExtensionTab('work', 'https://welcome.test', { active: true })
await Promise.resolve()
expect(mocks.createTab).toHaveBeenCalledTimes(1)
window.rendererReady()
@@ -295,7 +295,7 @@ export function recentWorktreeIdForBrowserProfile(profileId: string): string | u
export async function openBrowserExtensionTab(
browserProfileId: string,
url: string,
active: boolean
placement: { active: boolean; worktreeId?: string }
): Promise<number> {
const profile = getBrowserExtensionProfile(browserProfileId)
// Why wait: extensions open tabs as they load (onInstalled), before Orca's window can take them.
@@ -306,10 +306,10 @@ export async function openBrowserExtensionTab(
const { browserPageId } = await createBrowserTabInRenderer(
{ getAuthoritativeWindow: () => mainWindow },
url,
undefined,
placement.worktreeId,
profile.id,
profile.partition,
active,
placement.active,
undefined,
recentWorktreeIdForBrowserProfile(profile.id)
)
@@ -340,7 +340,7 @@ export async function closeBrowserExtensionTab(entry: BrowserExtensionTabEntry):
export async function activateBrowserExtensionTab(entry: BrowserExtensionTabEntry): Promise<void> {
const popout = getPopoutWindowForWebContents(entry.contents)
if (popout) {
presentBrowserExtensionWindow(popout, true)
presentBrowserExtensionWindow(popout, 'focused')
return
}
const browserPageId = browserManager.getTabIdForWebContentsId(entry.contents.id)
@@ -56,7 +56,7 @@ describe("a profile's extension windows", () => {
const popout = model.createBrowserExtensionPopout(profileB, {
url: 'https://b.test/',
type: 'popup',
focused: false
presentation: 'unfocused'
})
const ids = (sess: Electron.Session) =>
model.listBrowserExtensionWindows(sess).map((entry) => entry.window.id)
@@ -80,7 +80,11 @@ describe("a profile's extension windows", () => {
it("keep each profile's last-focused popout when another profile's popout takes focus", () => {
const main = new BrowserWindow()
model.trackBrowserExtensionMainWindow(main)
const options = { url: 'https://x.test/', type: 'popup' as const, focused: false }
const options = {
url: 'https://x.test/',
type: 'popup' as const,
presentation: 'unfocused' as const
}
const popoutA = model.createBrowserExtensionPopout(profileA, options)
const popoutB = model.createBrowserExtensionPopout(profileB, options)
focus(popoutA)
@@ -4,10 +4,12 @@ import { normalizeBrowserNavigationUrl } from '../../../../shared/browser-url'
import { isSameExtensionNavigation } from '../../../../shared/browser/extensions/browser-extension-url'
import {
isBackgroundLaunch,
showWhenAppActive,
showWindowWithoutStealingFocus
} from '../../../window/activation/foreground-activation-policy'
import { emitBrowserExtensionEvent } from '../api-runtime/browser-extension-api-host'
import { watchBrowserMainWindowReadiness } from '../../guest-lifecycle/browser-main-window-readiness'
import { recordBrowserSessionInput } from '../../guest-lifecycle/input/browser-guest-input-gesture'
/** chrome.windows constants. */
export const WINDOW_ID_NONE = -1
@@ -187,9 +189,19 @@ export function describeBrowserExtensionWindow(
}
}
/** `whenAppActive`: an agent's window, shown without focus once the user is in Orca. */
export type BrowserExtensionWindowPresentation = 'focused' | 'unfocused' | 'whenAppActive'
/** Focus or reveal per Orca's launch policy; automated (background) runs never take the foreground. */
export function presentBrowserExtensionWindow(window: BrowserWindow, focused: boolean): void {
if (focused && !isBackgroundLaunch()) {
export function presentBrowserExtensionWindow(
window: BrowserWindow,
presentation: BrowserExtensionWindowPresentation
): void {
if (presentation === 'whenAppActive') {
showWhenAppActive(window)
return
}
if (presentation === 'focused' && !isBackgroundLaunch()) {
window.show()
window.focus()
return
@@ -204,7 +216,7 @@ export type BrowserExtensionPopoutOptions = {
top?: number
width?: number
height?: number
focused: boolean
presentation: BrowserExtensionWindowPresentation
state?: 'normal' | 'minimized' | 'maximized' | 'fullscreen'
}
@@ -236,6 +248,7 @@ export function createBrowserExtensionPopout(
popoutsByWindowId.delete(windowId)
emitBrowserExtensionEvent({ session: sess }, 'windows', 'onRemoved', [windowId])
})
recordBrowserSessionInput(window.webContents)
window.webContents.setWindowOpenHandler(() => ({ action: 'deny' }))
// Why: the popout gets no Orca guest policies, so it may only move within its extension or to web URLs.
window.webContents.on('will-navigate', (event, url) => {
@@ -245,16 +258,19 @@ export function createBrowserExtensionPopout(
}
})
window.once('ready-to-show', () => {
if (options.state === 'maximized') {
window.maximize()
} else if (options.state === 'fullscreen') {
window.setFullScreen(true)
}
if (options.state === 'minimized') {
window.minimize()
return
}
presentBrowserExtensionWindow(window, options.focused)
// Why not for an agent: maximize can show the window and fullscreen moves the user to a new Space.
if (options.presentation !== 'whenAppActive') {
if (options.state === 'maximized') {
window.maximize()
} else if (options.state === 'fullscreen') {
window.setFullScreen(true)
}
}
presentBrowserExtensionWindow(window, options.presentation)
})
window.loadURL(options.url).catch((error: unknown) => {
console.warn('[browser-extensions] Popout failed to load:', error)

Some files were not shown because too many files have changed in this diff Show More