diff --git a/skill-guides/orca-cli/references/browser.md b/skill-guides/orca-cli/references/browser.md index b07f91ea782..a8435f8a8ce 100644 --- a/skill-guides/orca-cli/references/browser.md +++ b/skill-guides/orca-cli/references/browser.md @@ -1,68 +1,71 @@ # Built-in browser commands -Use a snapshot-interact-re-snapshot loop: +Pin one page, then use a snapshot-interact-re-snapshot loop on it: ```text -ORCA goto --url https://example.com --json -ORCA snapshot --json -ORCA click --element @e3 --json -ORCA snapshot --json +ORCA tab current --json +ORCA goto --page --url https://example.com --json +ORCA snapshot --page --json +ORCA click --page --element @e3 --json +ORCA snapshot --page --json ``` Common commands: ```text -ORCA goto --url --json -ORCA back --json -ORCA reload --json -ORCA snapshot --json -ORCA screenshot --json -ORCA full-screenshot --json -ORCA pdf --json -ORCA click --element --json -ORCA fill --element --value --json -ORCA type --input --json -ORCA select --element --value --json -ORCA check --element --json -ORCA scroll --direction down --amount 1000 --json -ORCA hover --element --json -ORCA focus --element --json -ORCA keypress --key Enter --json -ORCA upload --element --files --json -ORCA wait --text --json -ORCA wait --url --json -ORCA wait --selector --json -ORCA wait --load networkidle --json -ORCA eval --expression --json -ORCA set device --name "iPhone 15" --json -ORCA viewport --width 390 --height 844 --scale 3 --mobile --json +ORCA goto --page --url --json +ORCA back --page --json +ORCA reload --page --json +ORCA snapshot --page --json +ORCA screenshot --page --json +ORCA full-screenshot --page --json +ORCA pdf --page --json +ORCA click --page --element --json +ORCA fill --page --element --value --json +ORCA type --page --input --json +ORCA select --page --element --value --json +ORCA check --page --element --json +ORCA scroll --page --direction down --amount 1000 --json +ORCA hover --page --element --json +ORCA focus --page --element --json +ORCA keypress --page --key Enter --json +ORCA upload --page --element --files --json +ORCA wait --page --text --json +ORCA wait --page --url --json +ORCA wait --page --selector --json +ORCA wait --page --load networkidle --json +ORCA eval --page --expression --json +ORCA set device --page --name "iPhone 15" --json +ORCA viewport --page --width 390 --height 844 --scale 3 --mobile --json ORCA tab list --json ORCA tab create --url --json ORCA tab switch --index --json ORCA tab close --index --json -ORCA cookie get --json -ORCA capture start --json -ORCA console --limit 50 --json -ORCA network --limit 50 --json -ORCA exec --command "help" --json +ORCA cookie get --page --json +ORCA capture start --page --json +ORCA console --page --limit 50 --json +ORCA network --page --limit 50 --json +ORCA exec --page --command "help" --json ``` Browser rules: - Re-snapshot after navigation, tab switches, clicks that change the page, and any `browser_stale_ref`. - Refs like `@e1` are assigned by `snapshot`, scoped to one tab, and invalidated by navigation or tab switch. -- Browser commands default to the current worktree and its active tab. Use `--worktree all` only intentionally. -- For concurrent browser work, run `ORCA tab list --json`, read `tabs[].browserPageId`, and pass `--page ` on later commands. +- Pass `--page ` on every command that acts on one page; `ORCA --help` lists `--page` when it does. `tab list`, `tab current`, `tab create` and `open-url` take no `--page`, and `tab switch`/`tab close` pick a tab by `--index` or `--page`, never both, since `--page` wins. Take the ID from `result.tab.browserPageId` in `ORCA tab current --json` or `result.browserPageId` in `ORCA tab create --json` and keep it for the task; `tab current` shows the routed tab, not the one you pin. When the user means a specific tab, find it by URL or title in `ORCA tab list --json`. +- Without `--page`, a command goes to the worktree's current tab. That stays the agent's tab while it has had a command in the last 20 seconds or so, even after the user picks another (`tab current` reports it too), then moves to the user's pick. +- After `ORCA tab switch`, pass the new page's `browserPageId`. +- Browser commands default to the current worktree. `--page` reaches the page in any worktree unless you also pass `--worktree `. Use `--worktree all` only intentionally. - Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command "tab ..."`, so Orca keeps UI state synchronized. - Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command "set device ..."`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating. - Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts. -- Anything not listed above goes through `ORCA exec --command ""`. -- If `fill` or `type` fails on a custom input, try `ORCA focus --element @e1 --json` then `ORCA inserttext --text "text" --json`. +- Anything not listed above goes through `ORCA exec --page --command ""`. +- If `fill` or `type` fails on a custom input, try `ORCA focus --page --element @e1 --json` then `ORCA inserttext --page --text "text" --json`. - A client-hosted page renders in the paired desktop's browser engine, so every command against it needs that desktop online and returns `browser_host_unavailable` while it is closed, asleep, or disconnected. Server-hosted pages run with no desktop attached; prefer them for long or unattended automation. Common recoveries: - `browser_no_tab`: open a tab with `ORCA tab create --url --json`. -- `browser_stale_ref`: run `ORCA snapshot --json` and retry with fresh refs. +- `browser_stale_ref`: run `ORCA snapshot --page --json` and retry on the same page with fresh refs. - `browser_tab_not_found`: run `ORCA tab list --json` before switching or closing. - `browser_host_unavailable`: the desktop hosting the page is offline. Bring it back, or recreate the page with server placement if the work must outlive the desktop session. diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 46a8ed6a45b..2b649c28076 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -811,70 +811,73 @@ const ORCA_CLI_AUTOMATIONS_REFERENCE_MARKDOWN = [ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [ "# Built-in browser commands", "", - "Use a snapshot-interact-re-snapshot loop:", + "Pin one page, then use a snapshot-interact-re-snapshot loop on it:", "", "```text", - "ORCA goto --url https://example.com --json", - "ORCA snapshot --json", - "ORCA click --element @e3 --json", - "ORCA snapshot --json", + "ORCA tab current --json", + "ORCA goto --page --url https://example.com --json", + "ORCA snapshot --page --json", + "ORCA click --page --element @e3 --json", + "ORCA snapshot --page --json", "```", "", "Common commands:", "", "```text", - "ORCA goto --url --json", - "ORCA back --json", - "ORCA reload --json", - "ORCA snapshot --json", - "ORCA screenshot --json", - "ORCA full-screenshot --json", - "ORCA pdf --json", - "ORCA click --element --json", - "ORCA fill --element --value --json", - "ORCA type --input --json", - "ORCA select --element --value --json", - "ORCA check --element --json", - "ORCA scroll --direction down --amount 1000 --json", - "ORCA hover --element --json", - "ORCA focus --element --json", - "ORCA keypress --key Enter --json", - "ORCA upload --element --files --json", - "ORCA wait --text --json", - "ORCA wait --url --json", - "ORCA wait --selector --json", - "ORCA wait --load networkidle --json", - "ORCA eval --expression --json", - "ORCA set device --name \"iPhone 15\" --json", - "ORCA viewport --width 390 --height 844 --scale 3 --mobile --json", + "ORCA goto --page --url --json", + "ORCA back --page --json", + "ORCA reload --page --json", + "ORCA snapshot --page --json", + "ORCA screenshot --page --json", + "ORCA full-screenshot --page --json", + "ORCA pdf --page --json", + "ORCA click --page --element --json", + "ORCA fill --page --element --value --json", + "ORCA type --page --input --json", + "ORCA select --page --element --value --json", + "ORCA check --page --element --json", + "ORCA scroll --page --direction down --amount 1000 --json", + "ORCA hover --page --element --json", + "ORCA focus --page --element --json", + "ORCA keypress --page --key Enter --json", + "ORCA upload --page --element --files --json", + "ORCA wait --page --text --json", + "ORCA wait --page --url --json", + "ORCA wait --page --selector --json", + "ORCA wait --page --load networkidle --json", + "ORCA eval --page --expression --json", + "ORCA set device --page --name \"iPhone 15\" --json", + "ORCA viewport --page --width 390 --height 844 --scale 3 --mobile --json", "ORCA tab list --json", "ORCA tab create --url --json", "ORCA tab switch --index --json", "ORCA tab close --index --json", - "ORCA cookie get --json", - "ORCA capture start --json", - "ORCA console --limit 50 --json", - "ORCA network --limit 50 --json", - "ORCA exec --command \"help\" --json", + "ORCA cookie get --page --json", + "ORCA capture start --page --json", + "ORCA console --page --limit 50 --json", + "ORCA network --page --limit 50 --json", + "ORCA exec --page --command \"help\" --json", "```", "", "Browser rules:", "", "- Re-snapshot after navigation, tab switches, clicks that change the page, and any `browser_stale_ref`.", "- Refs like `@e1` are assigned by `snapshot`, scoped to one tab, and invalidated by navigation or tab switch.", - "- Browser commands default to the current worktree and its active tab. Use `--worktree all` only intentionally.", - "- For concurrent browser work, run `ORCA tab list --json`, read `tabs[].browserPageId`, and pass `--page ` on later commands.", + "- Pass `--page ` on every command that acts on one page; `ORCA --help` lists `--page` when it does. `tab list`, `tab current`, `tab create` and `open-url` take no `--page`, and `tab switch`/`tab close` pick a tab by `--index` or `--page`, never both, since `--page` wins. Take the ID from `result.tab.browserPageId` in `ORCA tab current --json` or `result.browserPageId` in `ORCA tab create --json` and keep it for the task; `tab current` shows the routed tab, not the one you pin. When the user means a specific tab, find it by URL or title in `ORCA tab list --json`.", + "- Without `--page`, a command goes to the worktree's current tab. That stays the agent's tab while it has had a command in the last 20 seconds or so, even after the user picks another (`tab current` reports it too), then moves to the user's pick.", + "- After `ORCA tab switch`, pass the new page's `browserPageId`.", + "- Browser commands default to the current worktree. `--page` reaches the page in any worktree unless you also pass `--worktree `. Use `--worktree all` only intentionally.", "- Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command \"tab ...\"`, so Orca keeps UI state synchronized.", "- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command \"set device ...\"`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.", "- Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts.", - "- Anything not listed above goes through `ORCA exec --command \"\"`.", - "- If `fill` or `type` fails on a custom input, try `ORCA focus --element @e1 --json` then `ORCA inserttext --text \"text\" --json`.", + "- Anything not listed above goes through `ORCA exec --page --command \"\"`.", + "- If `fill` or `type` fails on a custom input, try `ORCA focus --page --element @e1 --json` then `ORCA inserttext --page --text \"text\" --json`.", "- A client-hosted page renders in the paired desktop's browser engine, so every command against it needs that desktop online and returns `browser_host_unavailable` while it is closed, asleep, or disconnected. Server-hosted pages run with no desktop attached; prefer them for long or unattended automation.", "", "Common recoveries:", "", "- `browser_no_tab`: open a tab with `ORCA tab create --url --json`.", - "- `browser_stale_ref`: run `ORCA snapshot --json` and retry with fresh refs.", + "- `browser_stale_ref`: run `ORCA snapshot --page --json` and retry on the same page with fresh refs.", "- `browser_tab_not_found`: run `ORCA tab list --json` before switching or closing.", "- `browser_host_unavailable`: the desktop hosting the page is offline. Bring it back, or recreate the page with server placement if the work must outlive the desktop session.", "", diff --git a/src/main/browser/agent-browser-bridge-lifecycle.ts b/src/main/browser/agent-browser-bridge-lifecycle.ts index d19093da513..0a5ec29ec4e 100644 --- a/src/main/browser/agent-browser-bridge-lifecycle.ts +++ b/src/main/browser/agent-browser-bridge-lifecycle.ts @@ -62,6 +62,11 @@ export abstract class AgentBrowserBridgeLifecycle extends AgentBrowserBridgeRawP ) { this.activeWebContentsPerWorktree.set(owningWorktreeId, newWebContentsId) } + for (const [key, heldWebContentsId] of this.heldPersonSelections) { + if (heldWebContentsId === oldWebContentsId) { + this.heldPersonSelections.set(key, newWebContentsId) + } + } this.options.onTabsChanged?.(owningWorktreeId ?? undefined) } protected async ensureSession( diff --git a/src/main/browser/agent-browser-bridge-queue.ts b/src/main/browser/agent-browser-bridge-queue.ts index 1a550067ebd..3617b129c80 100644 --- a/src/main/browser/agent-browser-bridge-queue.ts +++ b/src/main/browser/agent-browser-bridge-queue.ts @@ -40,12 +40,9 @@ export abstract class AgentBrowserBridgeQueue extends AgentBrowserBridgeShutdown ) } const [tabId, wcId] = liveEntries[switchedIndex] - this.activeWebContentsId = wcId const owningWorktreeId = worktreeId ?? this.browserManager.getWorktreeIdForTab(tabId) // Page-only switches must also update the owning workspace for later scoped commands. - if (owningWorktreeId) { - this.activeWebContentsPerWorktree.set(owningWorktreeId, wcId) - } + this.pointAgentAt(wcId, owningWorktreeId ?? undefined) this.options.onTabsChanged?.(owningWorktreeId ?? undefined) return { switched: switchedIndex, browserPageId: tabId } }) diff --git a/src/main/browser/agent-browser-bridge-state.ts b/src/main/browser/agent-browser-bridge-state.ts index 0b926d036cb..da3fcdd4079 100644 --- a/src/main/browser/agent-browser-bridge-state.ts +++ b/src/main/browser/agent-browser-bridge-state.ts @@ -13,6 +13,8 @@ export abstract class AgentBrowserBridgeState { // Isolate each worktree's tab selection. protected readonly activeWebContentsPerWorktree = new Map() protected activeWebContentsId: number | null = null + // A person's tab pick waiting for the agent's tab to go idle; the undefined key is the global target. + protected readonly heldPersonSelections = new Map() protected readonly sessions = new Map() protected readonly commandQueues = new Map() protected readonly processingQueues = new Set() diff --git a/src/main/browser/agent-browser-bridge-tab-routing.test.ts b/src/main/browser/agent-browser-bridge-tab-routing.test.ts index e8d0d41f458..5911889c06e 100644 --- a/src/main/browser/agent-browser-bridge-tab-routing.test.ts +++ b/src/main/browser/agent-browser-bridge-tab-routing.test.ts @@ -53,6 +53,7 @@ vi.mock('./cdp-bridge', () => ({ })) import { AgentBrowserBridge } from './agent-browser-bridge' +import { runWithActiveRpcCaller } from '../runtime/rpc/rpc-active-caller' import { answerAgentBrowserCommands, mockBrowserManager, @@ -247,6 +248,71 @@ describe('AgentBrowserBridge', () => { expect(b.tabList('wt-1').tabs).toMatchObject([{ browserPageId: 'tab-a', active: true }]) }) + describe('while an agent drives a tab', () => { + const agentPages = new Set() + const agentOnTabA = (): AgentBrowserBridge => { + const tabs = new Map([ + ['tab-a', 1], + ['tab-b', 2], + ['tab-c', 3] + ]) + const worktrees = new Map([ + ['tab-a', 'wt-1'], + ['tab-b', 'wt-1'], + ['tab-c', 'wt-1'] + ]) + const guests = new Map( + [...tabs].map(([tabId, id]) => [id, mockWebContents(id, `https://${tabId}.com`, tabId)]) + ) + webContentsFromIdMock.mockImplementation((id: number) => guests.get(id) ?? null) + const b = new AgentBrowserBridge( + mockBrowserManager(tabs, worktrees, { + isAgentUsingPage: (id: string) => agentPages.has(id), + beginAgentCommand: (id: string) => { + agentPages.add(id) + return { run: (execute) => execute(), end: () => {} } + } + }) + ) + b.onTabChanged(1, 'wt-1') + agentPages.clear() + agentPages.add('tab-a') + return b + } + + it('keeps unpinned commands on that tab until it goes idle when the user picks another', () => { + const b = agentOnTabA() + + b.onTabChanged(2, 'wt-1') + expect(b.getActivePageId('wt-1')).toBe('tab-a') + + agentPages.clear() + expect(b.getActivePageId('wt-1')).toBe('tab-b') + }) + + it('holds a pick made before the agent switches tabs until the new tab goes idle', async () => { + const b = agentOnTabA() + + b.onTabChanged(3, 'wt-1') + await b.tabSwitch(undefined, 'wt-1', 'tab-b') + expect(b.getActivePageId('wt-1')).toBe('tab-b') + + agentPages.clear() + expect(b.getActivePageId('wt-1')).toBe('tab-c') + }) + + it.each([ + [{ kind: 'paired-device', deviceId: 'phone' } as const, 'tab-a'], + [{ kind: 'local-cli' } as const, 'tab-b'] + ])('a tab created by %o routes unpinned commands to %s', (caller, expected) => { + const b = agentOnTabA() + + runWithActiveRpcCaller(caller, () => b.setActiveTab(2, 'wt-1')) + + expect(b.getActivePageId('wt-1')).toBe(expected) + }) + }) + it('updates the owning worktree active tab when switching by browser page id', async () => { const tabs = new Map([ ['tab-a', 1], diff --git a/src/main/browser/agent-browser-bridge-tabs.ts b/src/main/browser/agent-browser-bridge-tabs.ts index 16ffbe33113..047b7fd978b 100644 --- a/src/main/browser/agent-browser-bridge-tabs.ts +++ b/src/main/browser/agent-browser-bridge-tabs.ts @@ -2,16 +2,76 @@ import type { BrowserTabInfo, BrowserTabListResult } from '../../shared/runtime- import { BrowserError } from './cdp-bridge' import type { ResolvedBrowserCommandTarget } from './agent-browser-bridge-types' import { AgentBrowserBridgeState } from './agent-browser-bridge-state' +import { isAgentRpcCaller } from '../runtime/rpc/rpc-caller-identity' +import { getActiveRpcCaller } from '../runtime/rpc/rpc-active-caller' export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState { setActiveTab(webContentsId: number, worktreeId?: string): void { - this.activeWebContentsId = webContentsId - if (worktreeId) { - this.activeWebContentsPerWorktree.set(worktreeId, webContentsId) + if (isAgentRpcCaller(getActiveRpcCaller())) { + this.pointAgentAt(webContentsId, worktreeId) + } else { + this.selectForPerson(webContentsId, worktreeId) } this.options.onTabsChanged?.(worktreeId) } + protected pointAgentAt(webContentsId: number, worktreeId?: string): void { + for (const key of routingKeys(worktreeId)) { + this.setRoutedWebContentsId(key, webContentsId) + } + // Mark the new tab now, or a person's pick before the agent's next command would take it. + const browserPageId = this.resolveTabIdSafe(webContentsId) + if (browserPageId) { + this.browserManager.beginAgentCommand(browserPageId).end() + } + } + + // A person's pick never pulls an agent off the tab it is driving; it applies once that tab goes idle. + private selectForPerson(webContentsId: number, worktreeId?: string): void { + for (const key of routingKeys(worktreeId)) { + const current = this.routedWebContentsId(key) + if (current !== null && current !== webContentsId && this.isAgentUsingTab(current)) { + this.heldPersonSelections.set(key, webContentsId) + } else { + this.setRoutedWebContentsId(key, webContentsId) + this.heldPersonSelections.delete(key) + } + } + } + + protected applyHeldPersonSelections(worktreeId?: string): void { + for (const key of routingKeys(worktreeId)) { + const held = this.heldPersonSelections.get(key) + const current = this.routedWebContentsId(key) + if (held === undefined || (current !== null && this.isAgentUsingTab(current))) { + continue + } + this.heldPersonSelections.delete(key) + if (this.getWebContents(held)) { + this.setRoutedWebContentsId(key, held) + } + } + } + + private isAgentUsingTab(webContentsId: number): boolean { + const browserPageId = this.resolveTabIdSafe(webContentsId) + return browserPageId !== null && this.browserManager.isAgentUsingPage(browserPageId) + } + + private routedWebContentsId(key: string | undefined): number | null { + return key === undefined + ? this.activeWebContentsId + : (this.activeWebContentsPerWorktree.get(key) ?? null) + } + + private setRoutedWebContentsId(key: string | undefined, webContentsId: number): void { + if (key === undefined) { + this.activeWebContentsId = webContentsId + } else { + this.activeWebContentsPerWorktree.set(key, webContentsId) + } + } + protected selectFallbackActiveWebContents( worktreeId: string, excludedWebContentsId?: number @@ -53,10 +113,7 @@ export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState { } } onTabChanged(webContentsId: number, worktreeId?: string): void { - this.activeWebContentsId = webContentsId - if (worktreeId) { - this.activeWebContentsPerWorktree.set(worktreeId, webContentsId) - } + this.selectForPerson(webContentsId, worktreeId) this.options.onTabsChanged?.(worktreeId) } getRegisteredTabs(worktreeId?: string): Map { @@ -82,6 +139,7 @@ export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState { } tabList(worktreeId?: string): BrowserTabListResult { + this.applyHeldPersonSelections(worktreeId) const tabs = this.getRegisteredTabs(worktreeId) // Match command routing without changing active-tab selection during discovery. let activeWcId = @@ -162,6 +220,7 @@ export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState { } protected resolveActiveTab(worktreeId?: string): ResolvedBrowserCommandTarget { + this.applyHeldPersonSelections(worktreeId) const tabs = this.getRegisteredTabs(worktreeId) if (tabs.size === 0) { @@ -233,3 +292,7 @@ export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState { return this.resolveActiveTab(onlyWorktreeId) } } + +function routingKeys(worktreeId: string | undefined): (string | undefined)[] { + return worktreeId ? [undefined, worktreeId] : [undefined] +} diff --git a/src/main/browser/agent-browser-bridge-test-harness.ts b/src/main/browser/agent-browser-bridge-test-harness.ts index b1f91098095..1761b48890e 100644 --- a/src/main/browser/agent-browser-bridge-test-harness.ts +++ b/src/main/browser/agent-browser-bridge-test-harness.ts @@ -42,6 +42,7 @@ export function mockBrowserManager( end: () => {} }), showAgentPointer: async () => {}, + isAgentUsingPage: () => false, ...overrides } as unknown as BrowserManager } diff --git a/src/main/browser/browser-agent-activity.ts b/src/main/browser/browser-agent-activity.ts index 5a681c917d3..7ee49ed8870 100644 --- a/src/main/browser/browser-agent-activity.ts +++ b/src/main/browser/browser-agent-activity.ts @@ -105,6 +105,10 @@ export class BrowserAgentActivity { } } + isActive(browserPageId: string): boolean { + return this.pages.has(browserPageId) + } + end(browserPageId: string): void { const page = this.pages.get(browserPageId) if (!page) { diff --git a/src/main/browser/browser-manager-visibility.ts b/src/main/browser/browser-manager-visibility.ts index 951592284c2..837f4195b19 100644 --- a/src/main/browser/browser-manager-visibility.ts +++ b/src/main/browser/browser-manager-visibility.ts @@ -22,6 +22,11 @@ export abstract class BrowserManagerVisibility extends BrowserManagerState { return this.agentActivity.beginCommand(browserPageId) } + /** The page shows "An agent is using this page": a command is queued or one settled within the idle window. */ + isAgentUsingPage(browserPageId: string): boolean { + return this.agentActivity.isActive(browserPageId) + } + // Show the agent's pointer landing before the input it precedes reaches the page. showAgentPointer(guest: Electron.WebContents, input: AgentPointerInput): Promise { return this.agentActivity.showPointer(guest, input)