Files
archived-orca/src/main/browser/agent-browser/command-queue/agent-browser-bridge-queue.ts
T
Kelvin Amoaba ada69cba8c fix(browser): stop agent clicks from opening the system file picker over your work (#41299)
* fix(browser): stop agent clicks from opening the system file picker

An agent click on a file input opened the macOS Open panel, which brought
Orca's window forward over the app the user was in. Page actions now catch
the picker the way upload does and tell the agent to use upload.

* fix(browser): enable Page, cancel the picker, and clean up outside host scope

* fix(browser): limit picker blocking to input delivery and bound its setup

* perf(browser): skip picker blocking for actions without user activation

* fix(browser): bound picker interception cleanup on a hung page

* fix(browser): bound the picker interception setup as a whole
2026-11-12 00:18:39 +00:00

330 lines
13 KiB
TypeScript

import type { BrowserTabSwitchResult } from '../../../../shared/browser/runtime/runtime-browser-contracts'
import { BrowserError } from '../../browser-error'
import { AgentBrowserBridgeActionEffects } from './agent-browser-bridge-action-effects'
import { ORCA_TAB_SESSION_PREFIX } from '../session-lifecycle/agent-browser-session-name'
import type { EnqueueTargetedCommandOptions } from './agent-browser-queued-command'
import type { ResolvedBrowserCommandTarget } from '../session-routing/agent-browser-command-target'
import { runWithRemoteCliPageAdmission } from '../../routing/input-ownership/remote-cli-page-admission'
import { runWithActionEffects } from '../observation/agent-browser-action-effects'
import { getRequestedPageUrl, runExpectingPageUrl } from '../../cdp/input/browser-expected-page-url'
import { AGENT_BROWSER_CLEANUP_TIMEOUT_MS } from '../session-lifecycle/agent-browser-cleanup-options'
import { releaseHeldPageInput } from '../../cdp/input/guest-direct-key-input'
import { withFileChooserBlocked } from '../../cdp/input/guest-file-chooser'
import { onBrowserJavaScriptDialogOpened } from '../../guest-lifecycle/prompts/browser-javascript-dialogs'
import {
assertNoJavaScriptDialog,
dialogAnswerHint,
quoteDialogMessage
} from './agent-browser-dialog-errors'
import {
captureRemoteCliBrowserRevocation,
captureRemoteCliBrowserScope,
runOutsideRemoteCliBrowserScope
} from '../../../runtime/desktop-browser-bridge/remote-cli-browser-scope'
import { captureActiveRpcCall, getActiveRpcCaller } from '../../../runtime/rpc/rpc-active-caller'
export abstract class AgentBrowserBridgeQueue extends AgentBrowserBridgeActionEffects {
// Serialize tab switches behind commands targeting the current tab.
async tabSwitch(
index: number | undefined,
worktreeId?: string,
browserPageId?: string
): Promise<BrowserTabSwitchResult> {
return this.enqueueCommand(worktreeId, async () => {
const tabs = this.getRegisteredTabs(worktreeId)
// The tab list may change while this switch waits in the queue.
const liveEntries = [...tabs.entries()].filter(([, wcId]) => this.getWebContents(wcId))
let switchedIndex = index ?? -1
let resolvedPageId = browserPageId
if (resolvedPageId) {
switchedIndex = liveEntries.findIndex(([tabId]) => tabId === resolvedPageId)
}
if (switchedIndex < 0 || switchedIndex >= liveEntries.length) {
const targetLabel =
resolvedPageId != null ? `Browser page ${resolvedPageId}` : `Tab index ${index}`
throw new BrowserError(
'browser_tab_not_found',
`${targetLabel} out of range (0-${liveEntries.length - 1})`
)
}
const [tabId, wcId] = liveEntries[switchedIndex]
const owningWorktreeId = worktreeId ?? this.browserManager.getWorktreeIdForTab(tabId)
// Page-only switches must also update the owning workspace for later scoped commands.
this.pointAgentAt(wcId, owningWorktreeId ?? undefined)
this.options.onTabsChanged?.(owningWorktreeId ?? undefined)
return { switched: switchedIndex, browserPageId: tabId }
})
}
protected async enqueueCommand<T>(
worktreeId: string | undefined,
execute: (sessionName: string) => Promise<T>
): Promise<T> {
return this.enqueueTargetedCommand(worktreeId, undefined, execute)
}
/** A command that acts on the page; an agent's gets the effects it caused added to its result. */
protected enqueuePageAction<T>(
worktreeId: string | undefined,
browserPageId: string | undefined,
execute: (sessionName: string, target: ResolvedBrowserCommandTarget) => Promise<T>,
options: EnqueueTargetedCommandOptions = {}
): Promise<T> {
return this.enqueueTargetedCommand(worktreeId, browserPageId, execute, {
...options,
reportEffects: options.reportEffects !== false
})
}
/** Navigation, pointer moves, wheel and scrolling deliver no user activation, so they open no file picker to block. */
protected enqueueInertAction<T>(
worktreeId: string | undefined,
browserPageId: string | undefined,
execute: (sessionName: string, target: ResolvedBrowserCommandTarget) => Promise<T>,
options: EnqueueTargetedCommandOptions = {}
): Promise<T> {
return this.enqueuePageAction(worktreeId, browserPageId, execute, {
...options,
skipFileChooserBlock: true
})
}
protected async enqueueTargetedCommand<T>(
worktreeId: string | undefined,
browserPageId: string | undefined,
execute: (sessionName: string, target: ResolvedBrowserCommandTarget) => Promise<T>,
options: EnqueueTargetedCommandOptions = {}
): Promise<T> {
this.assertCommandAdmission()
const assertNotRevoked = captureRemoteCliBrowserRevocation()
const origin = captureRemoteCliBrowserScope()
assertNotRevoked()
const { browserPageId: pageId, webContentsId } = this.resolveCommandTarget(
worktreeId,
browserPageId,
options.requireScopedTarget
)
if (!browserPageId && options.elements) {
this.assertRefsFromPage(pageId, options.elements)
}
// Fail before queueing: the turn ahead may be a command waiting on this very dialog.
assertNoJavaScriptDialog(webContentsId, pageId, assertNotRevoked)
const sessionName = `${ORCA_TAB_SESSION_PREFIX}${pageId}`
const agentCommand = this.browserManager.beginAgentCommand(pageId)
// Queued commands run under the caller ahead of them, so read this one's caller now
// and re-enter its call, or what it marks would count as that other chat's.
const caller = getActiveRpcCaller()
const inThisCall = captureActiveRpcCall()
// Decided at enqueue: a queued command runs under the caller of the command ahead of it.
const reportEffects = options.reportEffects === true && agentCommand.agent
// Page actions deliver agent or remote input; a desktop picker would activate Orca over the user's work.
const blockFileChooser = options.reportEffects === true && options.skipFileChooserBlock !== true
const expectedUrl = getRequestedPageUrl()
// A dialog blocks the page until it is answered. A command still waiting never runs; one
// already running answers its caller now and keeps its turn until it ends, so nothing overtakes it.
let running = false
let interrupted = false
// A started command ends the lease when it finishes; a dropped one when it settles.
let started = false
let stopWatching = (): void => {}
return new Promise<T>((resolve, reject) => {
let queue = this.commandQueues.get(sessionName)
if (!queue) {
queue = []
this.commandQueues.set(sessionName, queue)
}
const pageQueue = queue
const entry = {
execute: () => {
assertNotRevoked()
const run = (): Promise<T> =>
this.executeQueuedCommand(
worktreeId,
pageId,
(session, target) => {
const gate = (): void => assertNotRevoked(target.webContentsId)
gate()
if (interrupted) {
throw new BrowserError(
'browser_dialog_opened',
'A dialog interrupted this command'
)
}
assertNoJavaScriptDialog(target.webContentsId, pageId, gate)
if (target.webContentsId !== webContentsId) {
// The page got a replacement guest while this command waited.
const stopWatchingQueued = stopWatching
const stopWatchingTarget = watch(target.webContentsId)
stopWatching = () => {
stopWatchingQueued()
stopWatchingTarget()
}
}
running = true
// The effects baseline can take seconds, so the check sits inside it, next to the action.
const wc = blockFileChooser ? this.getWebContents(target.webContentsId) : null
// Only the input itself, so the user's own pickers work during the effects wait.
const deliver = (): Promise<T> =>
wc
? withFileChooserBlocked(wc, () => execute(session, target), {
agent: agentCommand.agent
})
: execute(session, target)
const guarded = (): Promise<T> =>
expectedUrl === undefined
? deliver()
: runExpectingPageUrl(
expectedUrl,
target.browserPageId,
() => this.livePageUrls(target),
deliver
)
const act = (): Promise<T> =>
reportEffects
? runWithActionEffects(
this.actionEffectsHost(worktreeId, pageId, session),
guarded
)
: guarded()
return origin.bridged
? this.runHostCommand(session, target.webContentsId, gate, origin, act)
: act()
},
options
)
return inThisCall(() =>
origin.run(() => {
started = true
return this.browserManager.runPageCommand(pageId, caller, () => agentCommand.run(run))
})
)
},
resolve: resolve as (value: unknown) => void,
reject
}
const watch = (guestId: number): (() => void) =>
onBrowserJavaScriptDialogOpened(guestId, (dialog) => {
const waitingAt = pageQueue.indexOf(entry)
if (waitingAt !== -1) {
pageQueue.splice(waitingAt, 1)
}
interrupted = true
try {
assertNotRevoked(guestId)
} catch (error) {
reject(error)
return
}
const opened = `The page opened a JavaScript ${dialog.type}: ${quoteDialogMessage(dialog)}.`
const effect = running
? "The command may already have taken effect, so don't repeat it."
: 'The command did not run.'
reject(
new BrowserError(
'browser_dialog_opened',
`${opened} ${effect} ${dialogAnswerHint(dialog, pageId)}.`
)
)
})
pageQueue.push(entry)
stopWatching = watch(webContentsId)
this.processQueue(sessionName)
}).finally(() => {
stopWatching()
if (!started) {
agentCommand.end()
}
})
}
protected async executeQueuedCommand<T>(
worktreeId: string | undefined,
browserPageId: string,
execute: (sessionName: string, target: ResolvedBrowserCommandTarget) => Promise<T>,
options: EnqueueTargetedCommandOptions
): Promise<T> {
this.assertCommandAdmission()
const sessionName = `${ORCA_TAB_SESSION_PREFIX}${browserPageId}`
// A queued page may acquire a replacement guest before execution.
const target = this.resolveCommandTarget(worktreeId, browserPageId)
const releaseGuest = this.browserManager.holdGuestActive(target.webContentsId)
try {
if (options.ensureSession !== false) {
await this.ensureSession(sessionName, browserPageId, target.webContentsId)
}
return await execute(sessionName, target)
} finally {
releaseGuest()
}
}
/** Revocation must stop every later step of a host command. */
private runHostCommand<T>(
sessionName: string,
webContentsId: number,
gate: () => void,
{ revoked, hostRevoked }: { revoked?: AbortSignal; hostRevoked?: AbortSignal },
execute: () => Promise<T>
): Promise<T> {
return runWithRemoteCliPageAdmission(webContentsId, gate, execute, {
revoked,
hostRevoked,
inPageTurn: (work) => this.enqueueDesktopPageWork(sessionName, work),
retire: () =>
this.destroySession(sessionName, {
closeTimeoutMs: AGENT_BROWSER_CLEANUP_TIMEOUT_MS,
keepQueuedCommands: true
}),
releaseHeld: (release) => {
const wc = this.getWebContents(webContentsId)
return wc
? releaseHeldPageInput(wc, release, () =>
this.browserManager.guardUserFocusForAgentInput(webContentsId)
)
: Promise.resolve()
}
})
}
/** Serialize desktop cleanup with commands on the same page. */
private enqueueDesktopPageWork(sessionName: string, work: () => Promise<void>): Promise<unknown> {
return new Promise((resolve, reject) => {
let queue = this.commandQueues.get(sessionName)
if (!queue) {
queue = []
this.commandQueues.set(sessionName, queue)
}
queue.push({
execute: () => runOutsideRemoteCliBrowserScope(work),
resolve,
reject
})
void this.processQueue(sessionName)
})
}
protected async processQueue(sessionName: string): Promise<void> {
if (this.processingQueues.has(sessionName)) {
return
}
this.processingQueues.add(sessionName)
const queue = this.commandQueues.get(sessionName)
while (queue && queue.length > 0) {
const cmd = queue.shift()!
try {
const result = await cmd.execute()
cmd.resolve(result)
} catch (error) {
cmd.reject(error)
}
}
if (queue && queue.length === 0 && this.commandQueues.get(sessionName) === queue) {
this.commandQueues.delete(sessionName)
}
this.processingQueues.delete(sessionName)
}
}