fix(browser): keep guest input bound to its webContents

This commit is contained in:
Jinjing
2026-11-11 19:20:32 -08:00
parent 437d078e18
commit fa5c319c49
7 changed files with 113 additions and 84 deletions
@@ -91,13 +91,9 @@ export abstract class AgentBrowserBridgeKeyCommands extends AgentBrowserBridgeCa
if (typeof step === 'string') {
// Text no key types (emoji, controls) goes in like a paste, between the keys around it.
await flush()
const route = await routePageTextInsertion(wc, { text: step }, send, guardFocus)
try {
assertExpectedPageUrlForStep()
await route.deliver({ text: step })
} finally {
route.release()
}
const route = await routePageTextInsertion(wc, send)
assertExpectedPageUrlForStep()
await route.deliver({ text: step })
continue
}
const event = {
@@ -4,7 +4,7 @@ import type { CdpClientResponseWriter } from '../transport/cdp-client-response-w
import type { CdpDebuggerChannel } from '../debugger/cdp-debugger-channel'
import type { TextInsertionRoute } from './guest-direct-key-input'
/** Replay DOM.focus before insertion because native focusing can blur the target. */
/** Replay DOM.focus before insertion so pipelined inserts use the resolved element. */
export class CdpDomFocusReplay {
private pendingDomFocusBySession = new Map<
string | undefined,
@@ -72,7 +72,7 @@ export class CdpDomFocusReplay {
return pendingFocus
}
/** Choose the route before replay: native fallback focus can blur the target. */
/** Choose a page-bound route before replaying the selected element. */
async insertText(
client: WebSocket,
clientId: number,
@@ -97,18 +97,14 @@ export class CdpDomFocusReplay {
this.responder.sendError(clientId, err instanceof Error ? err.message : String(err), client)
return
}
try {
await this.replayAndDeliver(
client,
clientId,
params,
effectiveSessionId,
pendingFocusParams,
route
)
} finally {
route.release()
}
await this.replayAndDeliver(
client,
clientId,
params,
effectiveSessionId,
pendingFocusParams,
route
)
}
private async replayAndDeliver(
@@ -12,6 +12,7 @@ import {
import {
dispatchPageKeyEvent,
isEmbeddedGuest,
pageInputRouteUnavailableError,
routePageTextInsertion
} from './guest-direct-key-input'
import type { GuestSessionCdpSender } from './guest-frame-input-sessions'
@@ -84,10 +85,9 @@ export class CdpPageInputCommands {
'Input.insertText',
insertParams,
sessionId
),
release: () => {}
)
})
: routePageTextInsertion(this.webContents, params, this.sendRoot, this.guardFocus)
: routePageTextInsertion(this.webContents, this.sendRoot)
)
)
return true
@@ -122,17 +122,16 @@ export class CdpPageInputCommands {
withAgentInputInjection(this.webContents, () =>
this.debuggerChannel.sendDebuggerCommand(method, params, sessionId)
)
const routedByFocus = !sessionId && isFocusRoutedCdpInput(method)
if (!routedByFocus && !isFocusTakingCdpInput(method, params)) {
if (!sessionId && isFocusRoutedCdpInput(method)) {
throw pageInputRouteUnavailableError()
}
if (!isFocusTakingCdpInput(method, params)) {
return await send()
}
// Emulate page focus while the pane guards and restores user focus.
await enablePageFocusEmulation(this.sendRoot)
const release = await this.guardFocus()
try {
if (routedByFocus) {
this.webContents.focus()
}
return await send()
} finally {
release()
@@ -1,6 +1,7 @@
import { assertExpectedPageUrlForStep } from './browser-expected-page-url'
import type { WebContents } from 'electron'
import { enablePageFocusEmulation, type AgentInputFocusGuard } from './agent-input-focus-guard'
import type { AgentInputFocusGuard } from './agent-input-focus-guard'
import { BrowserError } from '../../browser-error'
import { withAgentInputInjection } from './agent-input-injection'
import { resolveFocusedFrameRoute } from './guest-focused-frame-route'
import {
@@ -8,7 +9,7 @@ import {
type GuestSessionCdpSender,
sendFrameSessionInput
} from './guest-frame-input-sessions'
import { planCdpKeyEvent, planInsertText, type GuestEditCommand } from './guest-key-event-plan'
import { planCdpKeyEvent, type GuestEditCommand } from './guest-key-event-plan'
import { sendGuestCdpCommand } from '../debugger/guest-cdp-command'
import { acquireElectronDebugger } from '../debugger/electron-debugger-lease'
import { normalizeCdpPointerButton } from './cdp-pointer-targets'
@@ -68,16 +69,11 @@ function runEditCommand(webContents: WebContents, command: GuestEditCommand): vo
const NO_FOCUS_GUARD: AgentInputFocusGuard = async () => () => {}
// Restore user focus after CDP fallback; emulation keeps the page from seeing blur.
async function focusGuestForCdpInput(
webContents: WebContents,
send: GuestSessionCdpSender,
guardFocus: AgentInputFocusGuard
): Promise<() => void> {
await enablePageFocusEmulation(send)
const release = await guardFocus()
webContents.focus()
return release
export function pageInputRouteUnavailableError(): BrowserError {
return new BrowserError(
'browser_input_target_unavailable',
'Input could not be routed directly to the browser page, so it was not sent.'
)
}
// Cross-site frame sessions route CDP input without borrowing user focus.
@@ -93,7 +89,7 @@ async function sendToFrameSession(
)
}
/** Prefer guest widget or frame-session events; borrow focus for embedded-guest CDP fallback. */
/** Guest input must target its widget or frame session without borrowing window focus. */
export async function dispatchPageKeyEvent(
webContents: WebContents,
params: Record<string, unknown>,
@@ -190,17 +186,7 @@ export async function dispatchPageKeyEvents(
}
return {}
}
const release = await focusGuestForCdpInput(webContents, send, guardFocus)
try {
for (const params of remaining) {
result = await withAgentInputInjection(webContents, () =>
send('Input.dispatchKeyEvent', params)
)
}
return result
} finally {
release()
}
throw pageInputRouteUnavailableError()
}
/** Release input held by a revoked host, routing keys to the focused frame. */
@@ -235,49 +221,39 @@ export async function releaseHeldPageInput(
export type TextInsertionRoute = {
deliver: (params: Record<string, unknown>) => Promise<unknown>
// Release borrowed focus even if delivery is abandoned.
release: () => void
}
/** Route before DOM.focus replay: CDP fallback can blur the selected element. */
/** Native insertion targets the guest even when another tab or the host has focus. */
export async function routePageTextInsertion(
webContents: WebContents,
params: Record<string, unknown>,
send: GuestSessionCdpSender,
guardFocus: AgentInputFocusGuard = NO_FOCUS_GUARD
send: GuestSessionCdpSender
): Promise<TextInsertionRoute> {
const sendText = (insertParams: Record<string, unknown>): Promise<unknown> =>
send('Input.insertText', insertParams)
if (!isEmbeddedGuest(webContents)) {
webContents.focus()
return { deliver: sendText, release: () => {} }
return { deliver: sendText }
}
const route = await resolveFocusedFrameRoute(webContents, send)
if (route.kind === 'session') {
const { sessionId } = route
return {
deliver: (insertParams) =>
sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams),
release: () => {}
sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams)
}
}
const events = planInsertText(params.text)
if (events && route.kind === 'widget') {
if (route.kind === 'widget') {
return {
deliver: () =>
deliver: (insertParams) =>
withAgentInputInjection(webContents, async () => {
for (const event of events) {
webContents.sendInputEvent(event)
assertExpectedPageUrlForStep()
if (typeof insertParams.text !== 'string') {
throw new BrowserError('invalid_argument', 'Browser text must be a string')
}
await webContents.insertText(insertParams.text)
await waitForPageToHandleInput(send)
return {}
}),
release: () => {}
})
}
}
const release = await focusGuestForCdpInput(webContents, send, guardFocus)
return {
deliver: (insertParams) => withAgentInputInjection(webContents, () => sendText(insertParams)),
release
}
throw pageInputRouteUnavailableError()
}
@@ -16,7 +16,7 @@ export type FocusedFrameRoute =
| { kind: 'session'; sessionId: string }
| { kind: 'unknown' }
// Deeper nesting uses the focus-borrowing fallback.
// Refuse input if deeper nesting prevents a direct route.
const MAX_FRAME_DEPTH = 6
const PROBE_OBJECT_GROUP = 'orca-focused-frame-probe'
const PROBE_WORLD_NAME = 'orca-focused-frame-probe'
@@ -91,7 +91,7 @@ function routeFor(at: ProbeSite): FocusedFrameRoute {
return at.sessionId ? { kind: 'session', sessionId: at.sessionId } : { kind: 'widget' }
}
/** Follow focus through same-site contexts and cross-site sessions; unknown routes borrow focus. */
/** Follow focus through same-site contexts and cross-site sessions. */
export async function resolveFocusedFrameRoute(
webContents: WebContents,
send: GuestSessionCdpSender
@@ -79,7 +79,7 @@ function electronModifiers(mask: unknown): ElectronModifier[] {
return CDP_MODIFIER_NAMES.filter(([bit]) => (bits & bit) !== 0).map(([, name]) => name)
}
// Blink drops control characters from char events; use the fallback to preserve them.
// Blink drops control characters from char events.
function typeableText(text: unknown): text is string {
return (
typeof text === 'string' &&
@@ -146,7 +146,3 @@ export function planCdpKeyEvent(params: Record<string, unknown>): GuestKeyPlan |
: []
return { events: [event, ...enterChar], commands }
}
export function planInsertText(text: unknown): GuestKeyboardEvent[] | null {
return typeableText(text) ? charEvents(text, []) : null
}
@@ -3,14 +3,80 @@
* Runs hidden; browser-agent-input-focus-isolation-native.spec.ts repeats it in a visible window.
*/
import { test } from './helpers/orca-app'
import { test, expect } from './helpers/orca-app'
import {
evalInPage,
openBackgroundProbeTab,
readProperty,
rpc,
runAgentClicksScenario,
runAgentCrossSiteFrameScenario,
runAgentKeysScenario,
runAgentShortcutsScenario
runAgentShortcutsScenario,
startProbeServer
} from './helpers/browser-agent-focus-scenarios'
import { runAgentClickDuringUserCompositionScenario } from './helpers/browser-agent-composition-scenario'
import { getActiveTabId } from './helpers/store'
test('page-bound input stays out of chat drafts across hidden tabs and stale targets', async ({
orcaPage
}) => {
const server = await startProbeServer()
try {
const first = await openBackgroundProbeTab(orcaPage, server.url)
const second = await openBackgroundProbeTab(orcaPage, server.url)
await orcaPage.evaluate(() => {
const draft = document.createElement('textarea')
draft.id = 'browser-target-chat-draft'
draft.value = 'Unsent human draft'
document.body.append(draft)
draft.focus()
})
const draft = orcaPage.locator('#browser-target-chat-draft')
await rpc(orcaPage, 'browser.click', { page: first, element: '#notes' })
await draft.focus()
await rpc(orcaPage, 'browser.tabSwitch', { page: second })
await evalInPage(orcaPage, second, 'document.getElementById("notes").focus()')
await Promise.all([
rpc(orcaPage, 'browser.keyboardInsertText', { page: first, text: 'First line\nSecond 🙏' }),
rpc(orcaPage, 'browser.keyboardInsertText', { page: second, text: 'Other\ntab' }),
draft.focus()
])
await rpc(orcaPage, 'browser.type', { page: first, input: '!' })
await rpc(orcaPage, 'browser.keypress', { page: second, key: 'End' })
await rpc(orcaPage, 'browser.type', { page: second, input: '?' })
expect(await evalInPage(orcaPage, first, 'document.getElementById("notes").value')).toBe(
'First line\nSecond 🙏!'
)
expect(await evalInPage(orcaPage, second, 'document.getElementById("notes").value')).toBe(
'Other\ntab?'
)
await expect(draft).toHaveValue('Unsent human draft')
await rpc(orcaPage, 'browser.tabClose', { page: first })
const terminalTab = await getActiveTabId(orcaPage)
for (const command of [
{ method: 'browser.keyboardInsertText', params: { page: first, text: 'Stale insert' } },
{ method: 'browser.type', params: { page: 'missing-browser-page', input: 'Missing type' } },
{ method: 'browser.click', params: { page: terminalTab, element: 'textarea' } },
{ method: 'browser.keypress', params: { page: first, key: 'Enter' } }
]) {
const response = await orcaPage.evaluate(
(request) => window.api.runtime.call(request),
command
)
expect(readProperty(response, 'ok')).toBe(false)
expect(readProperty(readProperty(response, 'error'), 'code')).toBe('browser_tab_not_found')
}
await expect(draft).toHaveValue('Unsent human draft')
expect(await evalInPage(orcaPage, second, 'document.getElementById("notes").value')).toBe(
'Other\ntab?'
)
} finally {
await server.close()
}
})
test('agent keys and text reach the page without leaving the user’s terminal', async ({
orcaPage