From fa5c319c49a42a6b8abbaa7bc557ea7df19d8b2e Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:57:36 -0800 Subject: [PATCH] fix(browser): keep guest input bound to its webContents --- .../agent-browser-bridge-key-commands.ts | 10 +-- .../browser/cdp/input/cdp-dom-focus-replay.ts | 24 +++---- .../cdp/input/cdp-page-input-commands.ts | 15 ++-- .../cdp/input/guest-direct-key-input.ts | 68 ++++++------------ .../cdp/input/guest-focused-frame-route.ts | 4 +- .../browser/cdp/input/guest-key-event-plan.ts | 6 +- ...rowser-agent-input-focus-isolation.spec.ts | 70 ++++++++++++++++++- 7 files changed, 113 insertions(+), 84 deletions(-) diff --git a/src/main/browser/agent-browser/commands/agent-browser-bridge-key-commands.ts b/src/main/browser/agent-browser/commands/agent-browser-bridge-key-commands.ts index d313e934626..c0d26c0e020 100644 --- a/src/main/browser/agent-browser/commands/agent-browser-bridge-key-commands.ts +++ b/src/main/browser/agent-browser/commands/agent-browser-bridge-key-commands.ts @@ -91,13 +91,9 @@ export abstract class AgentBrowserBridgeKeyCommands extends AgentBrowserBridgeCa if (typeof step === 'string') { // Text no key types (emoji, controls) goes in like a paste, between the keys around it. await flush() - const route = await routePageTextInsertion(wc, { text: step }, send, guardFocus) - try { - assertExpectedPageUrlForStep() - await route.deliver({ text: step }) - } finally { - route.release() - } + const route = await routePageTextInsertion(wc, send) + assertExpectedPageUrlForStep() + await route.deliver({ text: step }) continue } const event = { diff --git a/src/main/browser/cdp/input/cdp-dom-focus-replay.ts b/src/main/browser/cdp/input/cdp-dom-focus-replay.ts index 31ffed7c213..a049e459779 100644 --- a/src/main/browser/cdp/input/cdp-dom-focus-replay.ts +++ b/src/main/browser/cdp/input/cdp-dom-focus-replay.ts @@ -4,7 +4,7 @@ import type { CdpClientResponseWriter } from '../transport/cdp-client-response-w import type { CdpDebuggerChannel } from '../debugger/cdp-debugger-channel' import type { TextInsertionRoute } from './guest-direct-key-input' -/** Replay DOM.focus before insertion because native focusing can blur the target. */ +/** Replay DOM.focus before insertion so pipelined inserts use the resolved element. */ export class CdpDomFocusReplay { private pendingDomFocusBySession = new Map< string | undefined, @@ -72,7 +72,7 @@ export class CdpDomFocusReplay { return pendingFocus } - /** Choose the route before replay: native fallback focus can blur the target. */ + /** Choose a page-bound route before replaying the selected element. */ async insertText( client: WebSocket, clientId: number, @@ -97,18 +97,14 @@ export class CdpDomFocusReplay { this.responder.sendError(clientId, err instanceof Error ? err.message : String(err), client) return } - try { - await this.replayAndDeliver( - client, - clientId, - params, - effectiveSessionId, - pendingFocusParams, - route - ) - } finally { - route.release() - } + await this.replayAndDeliver( + client, + clientId, + params, + effectiveSessionId, + pendingFocusParams, + route + ) } private async replayAndDeliver( diff --git a/src/main/browser/cdp/input/cdp-page-input-commands.ts b/src/main/browser/cdp/input/cdp-page-input-commands.ts index 372d5c21991..632ada08ec3 100644 --- a/src/main/browser/cdp/input/cdp-page-input-commands.ts +++ b/src/main/browser/cdp/input/cdp-page-input-commands.ts @@ -12,6 +12,7 @@ import { import { dispatchPageKeyEvent, isEmbeddedGuest, + pageInputRouteUnavailableError, routePageTextInsertion } from './guest-direct-key-input' import type { GuestSessionCdpSender } from './guest-frame-input-sessions' @@ -84,10 +85,9 @@ export class CdpPageInputCommands { 'Input.insertText', insertParams, sessionId - ), - release: () => {} + ) }) - : routePageTextInsertion(this.webContents, params, this.sendRoot, this.guardFocus) + : routePageTextInsertion(this.webContents, this.sendRoot) ) ) return true @@ -122,17 +122,16 @@ export class CdpPageInputCommands { withAgentInputInjection(this.webContents, () => this.debuggerChannel.sendDebuggerCommand(method, params, sessionId) ) - const routedByFocus = !sessionId && isFocusRoutedCdpInput(method) - if (!routedByFocus && !isFocusTakingCdpInput(method, params)) { + if (!sessionId && isFocusRoutedCdpInput(method)) { + throw pageInputRouteUnavailableError() + } + if (!isFocusTakingCdpInput(method, params)) { return await send() } // Emulate page focus while the pane guards and restores user focus. await enablePageFocusEmulation(this.sendRoot) const release = await this.guardFocus() try { - if (routedByFocus) { - this.webContents.focus() - } return await send() } finally { release() diff --git a/src/main/browser/cdp/input/guest-direct-key-input.ts b/src/main/browser/cdp/input/guest-direct-key-input.ts index 01ca2355541..85f43698be7 100644 --- a/src/main/browser/cdp/input/guest-direct-key-input.ts +++ b/src/main/browser/cdp/input/guest-direct-key-input.ts @@ -1,6 +1,7 @@ import { assertExpectedPageUrlForStep } from './browser-expected-page-url' import type { WebContents } from 'electron' -import { enablePageFocusEmulation, type AgentInputFocusGuard } from './agent-input-focus-guard' +import type { AgentInputFocusGuard } from './agent-input-focus-guard' +import { BrowserError } from '../../browser-error' import { withAgentInputInjection } from './agent-input-injection' import { resolveFocusedFrameRoute } from './guest-focused-frame-route' import { @@ -8,7 +9,7 @@ import { type GuestSessionCdpSender, sendFrameSessionInput } from './guest-frame-input-sessions' -import { planCdpKeyEvent, planInsertText, type GuestEditCommand } from './guest-key-event-plan' +import { planCdpKeyEvent, type GuestEditCommand } from './guest-key-event-plan' import { sendGuestCdpCommand } from '../debugger/guest-cdp-command' import { acquireElectronDebugger } from '../debugger/electron-debugger-lease' import { normalizeCdpPointerButton } from './cdp-pointer-targets' @@ -68,16 +69,11 @@ function runEditCommand(webContents: WebContents, command: GuestEditCommand): vo const NO_FOCUS_GUARD: AgentInputFocusGuard = async () => () => {} -// Restore user focus after CDP fallback; emulation keeps the page from seeing blur. -async function focusGuestForCdpInput( - webContents: WebContents, - send: GuestSessionCdpSender, - guardFocus: AgentInputFocusGuard -): Promise<() => void> { - await enablePageFocusEmulation(send) - const release = await guardFocus() - webContents.focus() - return release +export function pageInputRouteUnavailableError(): BrowserError { + return new BrowserError( + 'browser_input_target_unavailable', + 'Input could not be routed directly to the browser page, so it was not sent.' + ) } // Cross-site frame sessions route CDP input without borrowing user focus. @@ -93,7 +89,7 @@ async function sendToFrameSession( ) } -/** Prefer guest widget or frame-session events; borrow focus for embedded-guest CDP fallback. */ +/** Guest input must target its widget or frame session without borrowing window focus. */ export async function dispatchPageKeyEvent( webContents: WebContents, params: Record, @@ -190,17 +186,7 @@ export async function dispatchPageKeyEvents( } return {} } - const release = await focusGuestForCdpInput(webContents, send, guardFocus) - try { - for (const params of remaining) { - result = await withAgentInputInjection(webContents, () => - send('Input.dispatchKeyEvent', params) - ) - } - return result - } finally { - release() - } + throw pageInputRouteUnavailableError() } /** Release input held by a revoked host, routing keys to the focused frame. */ @@ -235,49 +221,39 @@ export async function releaseHeldPageInput( export type TextInsertionRoute = { deliver: (params: Record) => Promise - // Release borrowed focus even if delivery is abandoned. - release: () => void } -/** Route before DOM.focus replay: CDP fallback can blur the selected element. */ +/** Native insertion targets the guest even when another tab or the host has focus. */ export async function routePageTextInsertion( webContents: WebContents, - params: Record, - send: GuestSessionCdpSender, - guardFocus: AgentInputFocusGuard = NO_FOCUS_GUARD + send: GuestSessionCdpSender ): Promise { const sendText = (insertParams: Record): Promise => send('Input.insertText', insertParams) if (!isEmbeddedGuest(webContents)) { - webContents.focus() - return { deliver: sendText, release: () => {} } + return { deliver: sendText } } const route = await resolveFocusedFrameRoute(webContents, send) if (route.kind === 'session') { const { sessionId } = route return { deliver: (insertParams) => - sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams), - release: () => {} + sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams) } } - const events = planInsertText(params.text) - if (events && route.kind === 'widget') { + if (route.kind === 'widget') { return { - deliver: () => + deliver: (insertParams) => withAgentInputInjection(webContents, async () => { - for (const event of events) { - webContents.sendInputEvent(event) + assertExpectedPageUrlForStep() + if (typeof insertParams.text !== 'string') { + throw new BrowserError('invalid_argument', 'Browser text must be a string') } + await webContents.insertText(insertParams.text) await waitForPageToHandleInput(send) return {} - }), - release: () => {} + }) } } - const release = await focusGuestForCdpInput(webContents, send, guardFocus) - return { - deliver: (insertParams) => withAgentInputInjection(webContents, () => sendText(insertParams)), - release - } + throw pageInputRouteUnavailableError() } diff --git a/src/main/browser/cdp/input/guest-focused-frame-route.ts b/src/main/browser/cdp/input/guest-focused-frame-route.ts index 6d957cf5c1a..34e6f515a45 100644 --- a/src/main/browser/cdp/input/guest-focused-frame-route.ts +++ b/src/main/browser/cdp/input/guest-focused-frame-route.ts @@ -16,7 +16,7 @@ export type FocusedFrameRoute = | { kind: 'session'; sessionId: string } | { kind: 'unknown' } -// Deeper nesting uses the focus-borrowing fallback. +// Refuse input if deeper nesting prevents a direct route. const MAX_FRAME_DEPTH = 6 const PROBE_OBJECT_GROUP = 'orca-focused-frame-probe' const PROBE_WORLD_NAME = 'orca-focused-frame-probe' @@ -91,7 +91,7 @@ function routeFor(at: ProbeSite): FocusedFrameRoute { return at.sessionId ? { kind: 'session', sessionId: at.sessionId } : { kind: 'widget' } } -/** Follow focus through same-site contexts and cross-site sessions; unknown routes borrow focus. */ +/** Follow focus through same-site contexts and cross-site sessions. */ export async function resolveFocusedFrameRoute( webContents: WebContents, send: GuestSessionCdpSender diff --git a/src/main/browser/cdp/input/guest-key-event-plan.ts b/src/main/browser/cdp/input/guest-key-event-plan.ts index 44d976d057c..28284e133b9 100644 --- a/src/main/browser/cdp/input/guest-key-event-plan.ts +++ b/src/main/browser/cdp/input/guest-key-event-plan.ts @@ -79,7 +79,7 @@ function electronModifiers(mask: unknown): ElectronModifier[] { return CDP_MODIFIER_NAMES.filter(([bit]) => (bits & bit) !== 0).map(([, name]) => name) } -// Blink drops control characters from char events; use the fallback to preserve them. +// Blink drops control characters from char events. function typeableText(text: unknown): text is string { return ( typeof text === 'string' && @@ -146,7 +146,3 @@ export function planCdpKeyEvent(params: Record): GuestKeyPlan | : [] return { events: [event, ...enterChar], commands } } - -export function planInsertText(text: unknown): GuestKeyboardEvent[] | null { - return typeableText(text) ? charEvents(text, []) : null -} diff --git a/tests/e2e/browser-agent-input-focus-isolation.spec.ts b/tests/e2e/browser-agent-input-focus-isolation.spec.ts index 8c026f3ed60..3062aa347dc 100644 --- a/tests/e2e/browser-agent-input-focus-isolation.spec.ts +++ b/tests/e2e/browser-agent-input-focus-isolation.spec.ts @@ -3,14 +3,80 @@ * Runs hidden; browser-agent-input-focus-isolation-native.spec.ts repeats it in a visible window. */ -import { test } from './helpers/orca-app' +import { test, expect } from './helpers/orca-app' import { + evalInPage, + openBackgroundProbeTab, + readProperty, + rpc, runAgentClicksScenario, runAgentCrossSiteFrameScenario, runAgentKeysScenario, - runAgentShortcutsScenario + runAgentShortcutsScenario, + startProbeServer } from './helpers/browser-agent-focus-scenarios' import { runAgentClickDuringUserCompositionScenario } from './helpers/browser-agent-composition-scenario' +import { getActiveTabId } from './helpers/store' + +test('page-bound input stays out of chat drafts across hidden tabs and stale targets', async ({ + orcaPage +}) => { + const server = await startProbeServer() + try { + const first = await openBackgroundProbeTab(orcaPage, server.url) + const second = await openBackgroundProbeTab(orcaPage, server.url) + await orcaPage.evaluate(() => { + const draft = document.createElement('textarea') + draft.id = 'browser-target-chat-draft' + draft.value = 'Unsent human draft' + document.body.append(draft) + draft.focus() + }) + const draft = orcaPage.locator('#browser-target-chat-draft') + await rpc(orcaPage, 'browser.click', { page: first, element: '#notes' }) + await draft.focus() + await rpc(orcaPage, 'browser.tabSwitch', { page: second }) + await evalInPage(orcaPage, second, 'document.getElementById("notes").focus()') + + await Promise.all([ + rpc(orcaPage, 'browser.keyboardInsertText', { page: first, text: 'First line\nSecond ๐Ÿ™' }), + rpc(orcaPage, 'browser.keyboardInsertText', { page: second, text: 'Other\ntab' }), + draft.focus() + ]) + await rpc(orcaPage, 'browser.type', { page: first, input: '!' }) + await rpc(orcaPage, 'browser.keypress', { page: second, key: 'End' }) + await rpc(orcaPage, 'browser.type', { page: second, input: '?' }) + expect(await evalInPage(orcaPage, first, 'document.getElementById("notes").value')).toBe( + 'First line\nSecond ๐Ÿ™!' + ) + expect(await evalInPage(orcaPage, second, 'document.getElementById("notes").value')).toBe( + 'Other\ntab?' + ) + await expect(draft).toHaveValue('Unsent human draft') + + await rpc(orcaPage, 'browser.tabClose', { page: first }) + const terminalTab = await getActiveTabId(orcaPage) + for (const command of [ + { method: 'browser.keyboardInsertText', params: { page: first, text: 'Stale insert' } }, + { method: 'browser.type', params: { page: 'missing-browser-page', input: 'Missing type' } }, + { method: 'browser.click', params: { page: terminalTab, element: 'textarea' } }, + { method: 'browser.keypress', params: { page: first, key: 'Enter' } } + ]) { + const response = await orcaPage.evaluate( + (request) => window.api.runtime.call(request), + command + ) + expect(readProperty(response, 'ok')).toBe(false) + expect(readProperty(readProperty(response, 'error'), 'code')).toBe('browser_tab_not_found') + } + await expect(draft).toHaveValue('Unsent human draft') + expect(await evalInPage(orcaPage, second, 'document.getElementById("notes").value')).toBe( + 'Other\ntab?' + ) + } finally { + await server.close() + } +}) test('agent keys and text reach the page without leaving the userโ€™s terminal', async ({ orcaPage