mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 18:37:24 +00:00
fix(browser): keep guest input bound to its webContents
This commit is contained in:
@@ -91,13 +91,9 @@ export abstract class AgentBrowserBridgeKeyCommands extends AgentBrowserBridgeCa
|
||||
if (typeof step === 'string') {
|
||||
// Text no key types (emoji, controls) goes in like a paste, between the keys around it.
|
||||
await flush()
|
||||
const route = await routePageTextInsertion(wc, { text: step }, send, guardFocus)
|
||||
try {
|
||||
assertExpectedPageUrlForStep()
|
||||
await route.deliver({ text: step })
|
||||
} finally {
|
||||
route.release()
|
||||
}
|
||||
const route = await routePageTextInsertion(wc, send)
|
||||
assertExpectedPageUrlForStep()
|
||||
await route.deliver({ text: step })
|
||||
continue
|
||||
}
|
||||
const event = {
|
||||
|
||||
@@ -4,7 +4,7 @@ import type { CdpClientResponseWriter } from '../transport/cdp-client-response-w
|
||||
import type { CdpDebuggerChannel } from '../debugger/cdp-debugger-channel'
|
||||
import type { TextInsertionRoute } from './guest-direct-key-input'
|
||||
|
||||
/** Replay DOM.focus before insertion because native focusing can blur the target. */
|
||||
/** Replay DOM.focus before insertion so pipelined inserts use the resolved element. */
|
||||
export class CdpDomFocusReplay {
|
||||
private pendingDomFocusBySession = new Map<
|
||||
string | undefined,
|
||||
@@ -72,7 +72,7 @@ export class CdpDomFocusReplay {
|
||||
return pendingFocus
|
||||
}
|
||||
|
||||
/** Choose the route before replay: native fallback focus can blur the target. */
|
||||
/** Choose a page-bound route before replaying the selected element. */
|
||||
async insertText(
|
||||
client: WebSocket,
|
||||
clientId: number,
|
||||
@@ -97,18 +97,14 @@ export class CdpDomFocusReplay {
|
||||
this.responder.sendError(clientId, err instanceof Error ? err.message : String(err), client)
|
||||
return
|
||||
}
|
||||
try {
|
||||
await this.replayAndDeliver(
|
||||
client,
|
||||
clientId,
|
||||
params,
|
||||
effectiveSessionId,
|
||||
pendingFocusParams,
|
||||
route
|
||||
)
|
||||
} finally {
|
||||
route.release()
|
||||
}
|
||||
await this.replayAndDeliver(
|
||||
client,
|
||||
clientId,
|
||||
params,
|
||||
effectiveSessionId,
|
||||
pendingFocusParams,
|
||||
route
|
||||
)
|
||||
}
|
||||
|
||||
private async replayAndDeliver(
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
import {
|
||||
dispatchPageKeyEvent,
|
||||
isEmbeddedGuest,
|
||||
pageInputRouteUnavailableError,
|
||||
routePageTextInsertion
|
||||
} from './guest-direct-key-input'
|
||||
import type { GuestSessionCdpSender } from './guest-frame-input-sessions'
|
||||
@@ -84,10 +85,9 @@ export class CdpPageInputCommands {
|
||||
'Input.insertText',
|
||||
insertParams,
|
||||
sessionId
|
||||
),
|
||||
release: () => {}
|
||||
)
|
||||
})
|
||||
: routePageTextInsertion(this.webContents, params, this.sendRoot, this.guardFocus)
|
||||
: routePageTextInsertion(this.webContents, this.sendRoot)
|
||||
)
|
||||
)
|
||||
return true
|
||||
@@ -122,17 +122,16 @@ export class CdpPageInputCommands {
|
||||
withAgentInputInjection(this.webContents, () =>
|
||||
this.debuggerChannel.sendDebuggerCommand(method, params, sessionId)
|
||||
)
|
||||
const routedByFocus = !sessionId && isFocusRoutedCdpInput(method)
|
||||
if (!routedByFocus && !isFocusTakingCdpInput(method, params)) {
|
||||
if (!sessionId && isFocusRoutedCdpInput(method)) {
|
||||
throw pageInputRouteUnavailableError()
|
||||
}
|
||||
if (!isFocusTakingCdpInput(method, params)) {
|
||||
return await send()
|
||||
}
|
||||
// Emulate page focus while the pane guards and restores user focus.
|
||||
await enablePageFocusEmulation(this.sendRoot)
|
||||
const release = await this.guardFocus()
|
||||
try {
|
||||
if (routedByFocus) {
|
||||
this.webContents.focus()
|
||||
}
|
||||
return await send()
|
||||
} finally {
|
||||
release()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { assertExpectedPageUrlForStep } from './browser-expected-page-url'
|
||||
import type { WebContents } from 'electron'
|
||||
import { enablePageFocusEmulation, type AgentInputFocusGuard } from './agent-input-focus-guard'
|
||||
import type { AgentInputFocusGuard } from './agent-input-focus-guard'
|
||||
import { BrowserError } from '../../browser-error'
|
||||
import { withAgentInputInjection } from './agent-input-injection'
|
||||
import { resolveFocusedFrameRoute } from './guest-focused-frame-route'
|
||||
import {
|
||||
@@ -8,7 +9,7 @@ import {
|
||||
type GuestSessionCdpSender,
|
||||
sendFrameSessionInput
|
||||
} from './guest-frame-input-sessions'
|
||||
import { planCdpKeyEvent, planInsertText, type GuestEditCommand } from './guest-key-event-plan'
|
||||
import { planCdpKeyEvent, type GuestEditCommand } from './guest-key-event-plan'
|
||||
import { sendGuestCdpCommand } from '../debugger/guest-cdp-command'
|
||||
import { acquireElectronDebugger } from '../debugger/electron-debugger-lease'
|
||||
import { normalizeCdpPointerButton } from './cdp-pointer-targets'
|
||||
@@ -68,16 +69,11 @@ function runEditCommand(webContents: WebContents, command: GuestEditCommand): vo
|
||||
|
||||
const NO_FOCUS_GUARD: AgentInputFocusGuard = async () => () => {}
|
||||
|
||||
// Restore user focus after CDP fallback; emulation keeps the page from seeing blur.
|
||||
async function focusGuestForCdpInput(
|
||||
webContents: WebContents,
|
||||
send: GuestSessionCdpSender,
|
||||
guardFocus: AgentInputFocusGuard
|
||||
): Promise<() => void> {
|
||||
await enablePageFocusEmulation(send)
|
||||
const release = await guardFocus()
|
||||
webContents.focus()
|
||||
return release
|
||||
export function pageInputRouteUnavailableError(): BrowserError {
|
||||
return new BrowserError(
|
||||
'browser_input_target_unavailable',
|
||||
'Input could not be routed directly to the browser page, so it was not sent.'
|
||||
)
|
||||
}
|
||||
|
||||
// Cross-site frame sessions route CDP input without borrowing user focus.
|
||||
@@ -93,7 +89,7 @@ async function sendToFrameSession(
|
||||
)
|
||||
}
|
||||
|
||||
/** Prefer guest widget or frame-session events; borrow focus for embedded-guest CDP fallback. */
|
||||
/** Guest input must target its widget or frame session without borrowing window focus. */
|
||||
export async function dispatchPageKeyEvent(
|
||||
webContents: WebContents,
|
||||
params: Record<string, unknown>,
|
||||
@@ -190,17 +186,7 @@ export async function dispatchPageKeyEvents(
|
||||
}
|
||||
return {}
|
||||
}
|
||||
const release = await focusGuestForCdpInput(webContents, send, guardFocus)
|
||||
try {
|
||||
for (const params of remaining) {
|
||||
result = await withAgentInputInjection(webContents, () =>
|
||||
send('Input.dispatchKeyEvent', params)
|
||||
)
|
||||
}
|
||||
return result
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
throw pageInputRouteUnavailableError()
|
||||
}
|
||||
|
||||
/** Release input held by a revoked host, routing keys to the focused frame. */
|
||||
@@ -235,49 +221,39 @@ export async function releaseHeldPageInput(
|
||||
|
||||
export type TextInsertionRoute = {
|
||||
deliver: (params: Record<string, unknown>) => Promise<unknown>
|
||||
// Release borrowed focus even if delivery is abandoned.
|
||||
release: () => void
|
||||
}
|
||||
|
||||
/** Route before DOM.focus replay: CDP fallback can blur the selected element. */
|
||||
/** Native insertion targets the guest even when another tab or the host has focus. */
|
||||
export async function routePageTextInsertion(
|
||||
webContents: WebContents,
|
||||
params: Record<string, unknown>,
|
||||
send: GuestSessionCdpSender,
|
||||
guardFocus: AgentInputFocusGuard = NO_FOCUS_GUARD
|
||||
send: GuestSessionCdpSender
|
||||
): Promise<TextInsertionRoute> {
|
||||
const sendText = (insertParams: Record<string, unknown>): Promise<unknown> =>
|
||||
send('Input.insertText', insertParams)
|
||||
if (!isEmbeddedGuest(webContents)) {
|
||||
webContents.focus()
|
||||
return { deliver: sendText, release: () => {} }
|
||||
return { deliver: sendText }
|
||||
}
|
||||
const route = await resolveFocusedFrameRoute(webContents, send)
|
||||
if (route.kind === 'session') {
|
||||
const { sessionId } = route
|
||||
return {
|
||||
deliver: (insertParams) =>
|
||||
sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams),
|
||||
release: () => {}
|
||||
sendToFrameSession(webContents, send, sessionId, 'Input.insertText', insertParams)
|
||||
}
|
||||
}
|
||||
const events = planInsertText(params.text)
|
||||
if (events && route.kind === 'widget') {
|
||||
if (route.kind === 'widget') {
|
||||
return {
|
||||
deliver: () =>
|
||||
deliver: (insertParams) =>
|
||||
withAgentInputInjection(webContents, async () => {
|
||||
for (const event of events) {
|
||||
webContents.sendInputEvent(event)
|
||||
assertExpectedPageUrlForStep()
|
||||
if (typeof insertParams.text !== 'string') {
|
||||
throw new BrowserError('invalid_argument', 'Browser text must be a string')
|
||||
}
|
||||
await webContents.insertText(insertParams.text)
|
||||
await waitForPageToHandleInput(send)
|
||||
return {}
|
||||
}),
|
||||
release: () => {}
|
||||
})
|
||||
}
|
||||
}
|
||||
const release = await focusGuestForCdpInput(webContents, send, guardFocus)
|
||||
return {
|
||||
deliver: (insertParams) => withAgentInputInjection(webContents, () => sendText(insertParams)),
|
||||
release
|
||||
}
|
||||
throw pageInputRouteUnavailableError()
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ export type FocusedFrameRoute =
|
||||
| { kind: 'session'; sessionId: string }
|
||||
| { kind: 'unknown' }
|
||||
|
||||
// Deeper nesting uses the focus-borrowing fallback.
|
||||
// Refuse input if deeper nesting prevents a direct route.
|
||||
const MAX_FRAME_DEPTH = 6
|
||||
const PROBE_OBJECT_GROUP = 'orca-focused-frame-probe'
|
||||
const PROBE_WORLD_NAME = 'orca-focused-frame-probe'
|
||||
@@ -91,7 +91,7 @@ function routeFor(at: ProbeSite): FocusedFrameRoute {
|
||||
return at.sessionId ? { kind: 'session', sessionId: at.sessionId } : { kind: 'widget' }
|
||||
}
|
||||
|
||||
/** Follow focus through same-site contexts and cross-site sessions; unknown routes borrow focus. */
|
||||
/** Follow focus through same-site contexts and cross-site sessions. */
|
||||
export async function resolveFocusedFrameRoute(
|
||||
webContents: WebContents,
|
||||
send: GuestSessionCdpSender
|
||||
|
||||
@@ -79,7 +79,7 @@ function electronModifiers(mask: unknown): ElectronModifier[] {
|
||||
return CDP_MODIFIER_NAMES.filter(([bit]) => (bits & bit) !== 0).map(([, name]) => name)
|
||||
}
|
||||
|
||||
// Blink drops control characters from char events; use the fallback to preserve them.
|
||||
// Blink drops control characters from char events.
|
||||
function typeableText(text: unknown): text is string {
|
||||
return (
|
||||
typeof text === 'string' &&
|
||||
@@ -146,7 +146,3 @@ export function planCdpKeyEvent(params: Record<string, unknown>): GuestKeyPlan |
|
||||
: []
|
||||
return { events: [event, ...enterChar], commands }
|
||||
}
|
||||
|
||||
export function planInsertText(text: unknown): GuestKeyboardEvent[] | null {
|
||||
return typeableText(text) ? charEvents(text, []) : null
|
||||
}
|
||||
|
||||
@@ -3,14 +3,80 @@
|
||||
* Runs hidden; browser-agent-input-focus-isolation-native.spec.ts repeats it in a visible window.
|
||||
*/
|
||||
|
||||
import { test } from './helpers/orca-app'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
import {
|
||||
evalInPage,
|
||||
openBackgroundProbeTab,
|
||||
readProperty,
|
||||
rpc,
|
||||
runAgentClicksScenario,
|
||||
runAgentCrossSiteFrameScenario,
|
||||
runAgentKeysScenario,
|
||||
runAgentShortcutsScenario
|
||||
runAgentShortcutsScenario,
|
||||
startProbeServer
|
||||
} from './helpers/browser-agent-focus-scenarios'
|
||||
import { runAgentClickDuringUserCompositionScenario } from './helpers/browser-agent-composition-scenario'
|
||||
import { getActiveTabId } from './helpers/store'
|
||||
|
||||
test('page-bound input stays out of chat drafts across hidden tabs and stale targets', async ({
|
||||
orcaPage
|
||||
}) => {
|
||||
const server = await startProbeServer()
|
||||
try {
|
||||
const first = await openBackgroundProbeTab(orcaPage, server.url)
|
||||
const second = await openBackgroundProbeTab(orcaPage, server.url)
|
||||
await orcaPage.evaluate(() => {
|
||||
const draft = document.createElement('textarea')
|
||||
draft.id = 'browser-target-chat-draft'
|
||||
draft.value = 'Unsent human draft'
|
||||
document.body.append(draft)
|
||||
draft.focus()
|
||||
})
|
||||
const draft = orcaPage.locator('#browser-target-chat-draft')
|
||||
await rpc(orcaPage, 'browser.click', { page: first, element: '#notes' })
|
||||
await draft.focus()
|
||||
await rpc(orcaPage, 'browser.tabSwitch', { page: second })
|
||||
await evalInPage(orcaPage, second, 'document.getElementById("notes").focus()')
|
||||
|
||||
await Promise.all([
|
||||
rpc(orcaPage, 'browser.keyboardInsertText', { page: first, text: 'First line\nSecond 🙏' }),
|
||||
rpc(orcaPage, 'browser.keyboardInsertText', { page: second, text: 'Other\ntab' }),
|
||||
draft.focus()
|
||||
])
|
||||
await rpc(orcaPage, 'browser.type', { page: first, input: '!' })
|
||||
await rpc(orcaPage, 'browser.keypress', { page: second, key: 'End' })
|
||||
await rpc(orcaPage, 'browser.type', { page: second, input: '?' })
|
||||
expect(await evalInPage(orcaPage, first, 'document.getElementById("notes").value')).toBe(
|
||||
'First line\nSecond 🙏!'
|
||||
)
|
||||
expect(await evalInPage(orcaPage, second, 'document.getElementById("notes").value')).toBe(
|
||||
'Other\ntab?'
|
||||
)
|
||||
await expect(draft).toHaveValue('Unsent human draft')
|
||||
|
||||
await rpc(orcaPage, 'browser.tabClose', { page: first })
|
||||
const terminalTab = await getActiveTabId(orcaPage)
|
||||
for (const command of [
|
||||
{ method: 'browser.keyboardInsertText', params: { page: first, text: 'Stale insert' } },
|
||||
{ method: 'browser.type', params: { page: 'missing-browser-page', input: 'Missing type' } },
|
||||
{ method: 'browser.click', params: { page: terminalTab, element: 'textarea' } },
|
||||
{ method: 'browser.keypress', params: { page: first, key: 'Enter' } }
|
||||
]) {
|
||||
const response = await orcaPage.evaluate(
|
||||
(request) => window.api.runtime.call(request),
|
||||
command
|
||||
)
|
||||
expect(readProperty(response, 'ok')).toBe(false)
|
||||
expect(readProperty(readProperty(response, 'error'), 'code')).toBe('browser_tab_not_found')
|
||||
}
|
||||
await expect(draft).toHaveValue('Unsent human draft')
|
||||
expect(await evalInPage(orcaPage, second, 'document.getElementById("notes").value')).toBe(
|
||||
'Other\ntab?'
|
||||
)
|
||||
} finally {
|
||||
await server.close()
|
||||
}
|
||||
})
|
||||
|
||||
test('agent keys and text reach the page without leaving the user’s terminal', async ({
|
||||
orcaPage
|
||||
|
||||
Reference in New Issue
Block a user