fix(browser): refuse agent clicks while the user is mid-IME composition (#30358)

docs/reference/agent-browser-focus-isolation.md

@@ -75,17 +75,23 @@ popup and copies every frame. It stays an experiment, not the fix.
    If the pane does not answer, the input is refused with `browser_focus_guard_unavailable`
    rather than sent. Agent Tab keys take the same guard, because Tab from the page's last field
    moves focus out of the guest into Orca's window.
+   While the user's IME composition is open the pane answers `composing` and main refuses the
+   input, because the blur would commit the unfinished text (into a terminal, that sends it to
+   the shell). It is refused rather than held, so a press cannot outlive its agent or land in the
+   user's next composition. Direct commands report `browser_user_composing`; commands run through
+   the agent-browser helper, like `click`, report `browser_error` with the same retry message.
 4. **Agent keys are marked as the agent's.** While agent input is injected, the guest's Orca
    shortcut and grab handlers ignore keys and `setIgnoreMenuShortcuts` keeps unhandled keys
    from Orca's app menu, so an agent's Cmd/Ctrl+T, W or Ctrl+Tab stays in the page. Measured:
    synthetic input never fired the guest's `before-input-event`; the mark is the guarantee.
 
 Known gaps: the user's element is blurred for the length of one agent press, so keys typed in that
-window reach the page and an in-progress IME composition is lost. Edit commands (`selectAll`,
-`paste`, ...) run even when the page cancels the key. Focus emulation stays on while the debugger
-is attached, so a page the user really leaves sees no blur, and it covers the main frame only, so
-a menu inside a cross-site frame can still close. A pane that misses the 150 ms ready window
-refuses that input. Only offscreen rendering removes the first gap.
+window reach the page, and a composition started between the pane's answer and the press is still
+committed. Edit commands (`selectAll`, `paste`, ...) run even when the page cancels the key. Focus
+emulation stays on while the debugger is attached, so a page the user really leaves sees no blur,
+and it covers the main frame only, so a menu inside a cross-site frame can still close. A pane
+that misses the 150 ms ready window refuses that input. Only offscreen rendering removes the first
+two gaps.
 
 ## Tracker
 
@@ -97,3 +103,4 @@ refuses that input. Only offscreen rendering removes the first gap.
 | Regression spec: user typing in a terminal while an agent clicks, types and presses keys | stack 2/3, 3/3 | in review |
 | Visible-window spec (`@headful`, isolated display only): native focus stays on Orca's window | stack 3/3 | in review |
 | Main-session keys and text for a focused cross-site iframe go to its own session, no focus borrow | follow-up PR | in review |
+| Agent presses are refused while the user's IME composition is open (#28700) | follow-up PR | in review |

src/main/browser/agent-input-focus-guard.test.ts

@@ -101,6 +101,17 @@ describe('guardPaneFocusForAgentInput', () => {
     await vi.advanceTimersByTimeAsync(150)
     expect(settled).toBe('refused')
   })
+  it('refuses the input at once and ends the guard while the user composes', async () => {
+    const { sent, renderer } = fakeRenderer(7, false)
+
+    const guarded = guardPaneFocusForAgentInput(renderer, 'page-1')
+    const [listener] = ipcListeners.get(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL) ?? []
+    listener({ sender: { id: 7 } }, { token: sent[0].data.token, composing: true })
+
+    await expect(guarded).rejects.toMatchObject({ code: 'browser_user_composing' })
+    expect(sent.map(({ data }) => data.active)).toEqual([true, false])
+    expect(ipcListeners.get(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL)).toEqual([])
+  })
 })
 
 describe('isFocusTakingCdpInput', () => {

src/main/browser/agent-input-focus-guard.ts

@@ -24,16 +24,25 @@ export function focusGuardUnavailableError(): BrowserError {
   )
 }
 
+// Why: blurring the user's open IME composition commits its unfinished text (#28700). Refused
+// rather than held: a held press could outlive its agent or land in the user's next composition.
+function userComposingError(): BrowserError {
+  return new BrowserError(
+    'browser_user_composing',
+    'The user is typing with an input method in Orca, so the input was not sent to keep their unfinished text. Retry in a moment.'
+  )
+}
+
 export async function guardPaneFocusForAgentInput(
   renderer: WebContents,
   browserPageId: string
 ): Promise<() => void> {
   const token = randomUUID()
-  const answered = await new Promise<boolean>((resolve) => {
-    const finish = (ready: boolean): void => {
+  const answer = await new Promise<'ready' | 'unavailable' | 'composing'>((resolve) => {
+    const finish = (outcome: 'ready' | 'unavailable' | 'composing'): void => {
       clearTimeout(timer)
       ipcMain.removeListener(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL, onReady)
-      resolve(ready)
+      resolve(outcome)
     }
     const onReady = (event: Electron.IpcMainEvent, reply: unknown): void => {
       if (
@@ -43,10 +52,10 @@ export async function guardPaneFocusForAgentInput(
         'token' in reply &&
         reply.token === token
       ) {
-        finish(true)
+        finish('composing' in reply && reply.composing === true ? 'composing' : 'ready')
       }
     }
-    const timer = setTimeout(() => finish(false), PANE_READY_TIMEOUT_MS)
+    const timer = setTimeout(() => finish('unavailable'), PANE_READY_TIMEOUT_MS)
     ipcMain.on(AGENT_INPUT_FOCUS_GUARD_READY_CHANNEL, onReady)
     renderer.send(AGENT_INPUT_FOCUS_GUARD_CHANNEL, { browserPageId, token, active: true })
   })
@@ -60,10 +69,10 @@ export async function guardPaneFocusForAgentInput(
       renderer.send(AGENT_INPUT_FOCUS_GUARD_CHANNEL, { browserPageId, token, active: false })
     }
   }
-  if (!answered) {
+  if (answer !== 'ready') {
     // Why: input sent without a pane holding the user's element would take their focus for good.
     release()
-    throw focusGuardUnavailableError()
+    throw answer === 'composing' ? userComposingError() : focusGuardUnavailableError()
   }
   return release
 }

src/preload/api/browser-api.ts

@@ -127,7 +127,7 @@ export type BrowserApi = {
   onAgentInputFocusGuard: (
     callback: (data: { browserPageId: string; token: string; active: boolean }) => void
   ) => () => void
-  agentInputFocusGuardReady: (token: string) => void
+  agentInputFocusGuardReady: (token: string, composing: boolean) => void
   onPaneFocus: (
     callback: (data: { worktreeId: string | null; browserPageId: string }) => void
   ) => () => void

src/preload/api/browser-bridge-page-interaction-and-sessions.ts

@@ -82,8 +82,8 @@ export const browserPageInteractionAndSessionsApi = {
     ipcRenderer.on('browser:agentInputFocusGuard', listener)
     return () => ipcRenderer.removeListener('browser:agentInputFocusGuard', listener)
   },
-  agentInputFocusGuardReady: (token: string): void => {
-    ipcRenderer.send('browser:agentInputFocusGuardReady', { token })
+  agentInputFocusGuardReady: (token: string, composing: boolean): void => {
+    ipcRenderer.send('browser:agentInputFocusGuardReady', { token, composing })
   },
   onPaneFocus: (
     callback: (data: { worktreeId: string | null; browserPageId: string }) => void

src/renderer/src/components/browser-pane/host-guest/agent-input-focus-guard.test.ts

@@ -4,7 +4,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import {
   beginAgentInputFocusGuard,
   clearAgentInputFocusGuards,
-  endAgentInputFocusGuard
+  endAgentInputFocusGuard,
+  watchUserComposition
 } from './agent-input-focus-guard'
 import { webviewRegistry } from './webview-registry'
 
@@ -24,12 +25,24 @@ function focusTerminal(): HTMLTextAreaElement {
   return terminal
 }
 
+function compose(
+  element: HTMLElement,
+  type: 'compositionstart' | 'compositionupdate' | 'compositionend',
+  data = ''
+): void {
+  element.dispatchEvent(new CompositionEvent(type, { bubbles: true, data }))
+}
+
 describe('agent input focus guard', () => {
+  let unwatchUserComposition: () => void
+
   beforeEach(() => {
     vi.useFakeTimers()
+    unwatchUserComposition = watchUserComposition()
   })
 
   afterEach(() => {
+    unwatchUserComposition()
     clearAgentInputFocusGuards()
     vi.runAllTimers()
     vi.useRealTimers()
@@ -137,4 +150,61 @@ describe('agent input focus guard', () => {
 
     expect(document.activeElement).toBe(terminal)
   })
+
+  it('refuses a press while the user composes and leaves their focus alone', () => {
+    const webview = mountPage('page-1')
+    const terminal = focusTerminal()
+    compose(terminal, 'compositionstart')
+
+    expect(beginAgentInputFocusGuard('page-1', 'input-1')).toBe('composing')
+    endAgentInputFocusGuard('input-1')
+    webview.focus()
+    vi.advanceTimersByTime(30_000)
+
+    expect(document.activeElement).toBe(webview)
+  })
+
+  it('treats a nonempty update as an open composition and an empty one as not ending it', () => {
+    mountPage('page-1')
+    const terminal = focusTerminal()
+
+    compose(terminal, 'compositionupdate', 'zhong')
+    expect(beginAgentInputFocusGuard('page-1', 'input-1')).toBe('composing')
+    compose(terminal, 'compositionupdate', '')
+    expect(beginAgentInputFocusGuard('page-1', 'input-2')).toBe('composing')
+    compose(terminal, 'compositionend')
+    expect(beginAgentInputFocusGuard('page-1', 'input-3')).toBe('ready')
+  })
+
+  it('keeps a Cangjie composition open through its own input and Backspace', () => {
+    mountPage('page-1')
+    const terminal = focusTerminal()
+    const composingInput = (inputType?: string): InputEvent =>
+      new InputEvent('input', { bubbles: true, isComposing: true, inputType })
+
+    // Recorded: compose 尸 and 口, then Backspace leaves 尸 unfinished.
+    compose(terminal, 'compositionstart')
+    compose(terminal, 'compositionupdate', '尸')
+    terminal.dispatchEvent(composingInput())
+    compose(terminal, 'compositionupdate', '尸口')
+    terminal.dispatchEvent(composingInput())
+    compose(terminal, 'compositionupdate', '尸')
+    terminal.dispatchEvent(composingInput('deleteContentBackward'))
+
+    expect(beginAgentInputFocusGuard('page-1', 'input-1')).toBe('composing')
+  })
+
+  it('guards a press once the composition ended by typing or the user left it', () => {
+    mountPage('page-1')
+    const terminal = focusTerminal()
+
+    compose(terminal, 'compositionstart')
+    terminal.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText' }))
+    expect(beginAgentInputFocusGuard('page-1', 'input-1')).toBe('ready')
+    endAgentInputFocusGuard('input-1')
+
+    compose(terminal, 'compositionstart')
+    terminal.blur()
+    expect(beginAgentInputFocusGuard('page-1', 'input-2')).toBe('ready')
+  })
 })

src/renderer/src/components/browser-pane/host-guest/agent-input-focus-guard.ts

@@ -11,6 +11,9 @@ const LATE_FOCUS_RECHECK_MS = 50
 const ABANDONED_GUARD_MS = 30_000
 
 const releasesByToken = new Map<string, () => void>()
+// Why: blurring an element mid-composition commits the unfinished text, which a terminal sends to
+// the shell, so agent presses are refused while the user composes (#28700).
+let composingElement: EventTarget | null = null
 // Why: while any agent input or its recheck is in flight, document.activeElement may be a page
 // an agent focused, not the user's choice; overlapping guards share the element seen first.
 let inFlight = 0
@@ -23,7 +26,65 @@ function settle(): void {
   }
 }
 
-export function beginAgentInputFocusGuard(browserPageId: string, token: string): void {
+function isUserComposing(): boolean {
+  return composingElement !== null && composingElement === document.activeElement
+}
+
+/** Tracks the window's open IME composition, by the terminal tracker's rules; returns the unsubscribe. */
+export function watchUserComposition(): () => void {
+  if (typeof document === 'undefined') {
+    return () => {}
+  }
+  const onStart = (event: Event): void => {
+    composingElement = event.target
+  }
+  // Why: some IMEs send a nonempty update without a new compositionstart; Sogou/fcitx send empty
+  // ones while the candidate popup is still open, so those must not end it.
+  const onUpdate = (event: Event): void => {
+    if (event instanceof CompositionEvent && event.data !== '') {
+      composingElement = event.target
+    }
+  }
+  const onEnd = (): void => {
+    composingElement = null
+  }
+  // Why: typing outside a composition proves none is open, so a missed compositionend cannot stick.
+  // Composition-owned input, like Cangjie's Backspace over one of several radicals, keeps it open.
+  const onInput = (event: Event): void => {
+    if (!(event instanceof InputEvent && event.isComposing)) {
+      composingElement = null
+    }
+  }
+  const listeners: [string, (event: Event) => void][] = [
+    ['compositionstart', onStart],
+    ['compositionupdate', onUpdate],
+    ['compositionend', onEnd],
+    ['input', onInput]
+  ]
+  for (const [type, listener] of listeners) {
+    document.addEventListener(type, listener, true)
+  }
+  return () => {
+    for (const [type, listener] of listeners) {
+      document.removeEventListener(type, listener, true)
+    }
+    composingElement = null
+  }
+}
+
+/** Refuses the input while the user composes; otherwise holds their focus until the input ends. */
+export function beginAgentInputFocusGuard(
+  browserPageId: string,
+  token: string
+): 'ready' | 'composing' {
+  if (isUserComposing()) {
+    return 'composing'
+  }
+  guardUserFocus(browserPageId, token)
+  return 'ready'
+}
+
+function guardUserFocus(browserPageId: string, token: string): void {
   const webview = webviewRegistry.get(browserPageId)
   if (!webview) {
     return

src/renderer/src/hooks/ipc-events/browser-state-ipc-bridge.ts

@@ -11,7 +11,8 @@ import { acquireBrowserAutomationBootstrapLease } from './browser-automation-boo
 import {
   beginAgentInputFocusGuard,
   clearAgentInputFocusGuards,
-  endAgentInputFocusGuard
+  endAgentInputFocusGuard,
+  watchUserComposition
 } from '@/components/browser-pane/host-guest/agent-input-focus-guard'
 
 /**
@@ -104,16 +105,18 @@ export function registerBrowserStateIpcBridge(
   const unsubscribeAgentInputFocusGuard = window.api.browser.onAgentInputFocusGuard?.(
     ({ browserPageId, token, active }) => {
       if (active) {
-        beginAgentInputFocusGuard(browserPageId, token)
-        window.api.browser.agentInputFocusGuardReady(token)
+        const answer = beginAgentInputFocusGuard(browserPageId, token)
+        window.api.browser.agentInputFocusGuardReady(token, answer === 'composing')
       } else {
         endAgentInputFocusGuard(token)
       }
     }
   )
   if (unsubscribeAgentInputFocusGuard) {
+    const unwatchUserComposition = watchUserComposition()
     unsubs.push(() => {
       unsubscribeAgentInputFocusGuard()
+      unwatchUserComposition()
       clearAgentInputFocusGuards()
     })
   }

src/renderer/src/hooks/useIpcEvents-lifecycle.test.ts

@@ -161,7 +161,15 @@ describe('useIpcEvents App-lifetime lifecycle', () => {
     ).toBe(true)
     const activeSubscriptionCount = storeSubscriptions.filter((item) => item.active).length
     expect(activeSubscriptionCount).toBeGreaterThan(0)
-    expect(addDocumentListener.mock.calls.map(([type]) => type)).toEqual(['focusin', 'pointerdown'])
+    const documentListenerTypes = [
+      'focusin',
+      'pointerdown',
+      'compositionstart',
+      'compositionupdate',
+      'compositionend',
+      'input'
+    ]
+    expect(addDocumentListener.mock.calls.map(([type]) => type)).toEqual(documentListenerTypes)
 
     firstCleanup()
     expect(runtimeSubscriptions.every((unsubscribe) => unsubscribe.mock.calls.length === 1)).toBe(
@@ -185,10 +193,8 @@ describe('useIpcEvents App-lifetime lifecycle', () => {
     ).toBe(true)
     expect(storeSubscriptions.filter((item) => item.active)).toHaveLength(activeSubscriptionCount)
     expect(addDocumentListener.mock.calls.map(([type]) => type)).toEqual([
-      'focusin',
-      'pointerdown',
-      'focusin',
-      'pointerdown'
+      ...documentListenerTypes,
+      ...documentListenerTypes
     ])
 
     secondCleanup()

tests/e2e/browser-agent-input-focus-isolation.spec.ts

@@ -10,6 +10,7 @@ import {
   runAgentKeysScenario,
   runAgentShortcutsScenario
 } from './helpers/browser-agent-focus-scenarios'
+import { runAgentClickDuringUserCompositionScenario } from './helpers/browser-agent-composition-scenario'
 
 test('agent keys and text reach the page without leaving the user’s terminal', async ({
   orcaPage
@@ -38,3 +39,10 @@ test('agent keys and text reach a focused cross-site frame without moving the us
   test.setTimeout(120_000)
   await runAgentCrossSiteFrameScenario(orcaPage)
 })
+
+test('an agent click is refused while the user composes with an IME in the terminal', async ({
+  orcaPage
+}) => {
+  test.setTimeout(120_000)
+  await runAgentClickDuringUserCompositionScenario(orcaPage)
+})

tests/e2e/helpers/browser-agent-composition-scenario.ts

@@ -0,0 +1,77 @@
+/**
+ * An agent press while the user is mid-IME-composition in the terminal (#28700).
+ * Blurring the terminal then would commit the unfinished text to the shell, so the press is refused.
+ * See docs/reference/agent-browser-focus-isolation.md.
+ */
+
+import type { Page } from '@stablyai/playwright-test'
+import { expect } from './orca-app'
+import {
+  evalInPage,
+  openBackgroundProbeTab,
+  rpc,
+  startProbeServer
+} from './browser-agent-focus-scenarios'
+import { getTerminalContent } from './terminal-pane-identity'
+
+async function readCompositionLog(page: Page): Promise<string[]> {
+  return page.evaluate(() => {
+    const parsed: unknown = JSON.parse(document.documentElement.dataset.compositionProbeLog ?? '[]')
+    return Array.isArray(parsed)
+      ? parsed.filter((entry): entry is string => typeof entry === 'string')
+      : []
+  })
+}
+
+export async function runAgentClickDuringUserCompositionScenario(page: Page): Promise<void> {
+  const server = await startProbeServer()
+  try {
+    const pageId = await openBackgroundProbeTab(page, server.url)
+    await page.evaluate(() => {
+      const textarea = document.querySelector<HTMLTextAreaElement>('.xterm-helper-textarea')
+      if (!textarea) {
+        throw new Error('No terminal input to focus')
+      }
+      textarea.focus()
+      const log: string[] = []
+      const record = (entry: string): void => {
+        log.push(entry)
+        document.documentElement.dataset.compositionProbeLog = JSON.stringify(log)
+      }
+      for (const type of ['compositionend', 'blur']) {
+        textarea.addEventListener(type, (event) =>
+          record(event instanceof CompositionEvent ? `${type}:${event.data}` : type)
+        )
+      }
+    })
+    // Why: CDP enters Chromium where a real input method does: pinyin typed, no candidate picked.
+    const cdp = await page.context().newCDPSession(page)
+    await cdp.send('Input.imeSetComposition', { text: 'zhong', selectionStart: 5, selectionEnd: 5 })
+
+    const refused: unknown = await page.evaluate(
+      (id) =>
+        window.api.runtime.call({
+          method: 'browser.click',
+          params: { page: id, element: '#target' }
+        }),
+      pageId
+    )
+    expect(JSON.stringify(refused)).toContain('typing with an input method')
+    expect(await evalInPage(page, pageId, 'window.__clicks')).toBe('0')
+    expect(await readCompositionLog(page)).toEqual([])
+
+    await cdp.send('Input.insertText', { text: '中' })
+    await rpc(page, 'browser.click', { page: pageId, element: '#target' })
+    await expect.poll(() => evalInPage(page, pageId, 'window.__clicks')).toBe('1')
+    expect((await readCompositionLog(page))[0]).toBe('compositionend:中')
+    await expect
+      .poll(() => page.evaluate(() => document.activeElement?.className ?? ''))
+      .toContain('xterm-helper-textarea')
+    // Why: a non-UTF-8 CI shell may not echo 中; a later ASCII key shows what reached it before.
+    await page.keyboard.type('#28700')
+    await expect.poll(() => getTerminalContent(page)).toContain('#28700')
+    expect(await getTerminalContent(page)).not.toContain('zhong')
+  } finally {
+    await server.close()
+  }
+}

tests/e2e/helpers/browser-agent-focus-scenarios.ts

@@ -76,7 +76,7 @@ const FRAME_HTML = `<!doctype html>
   </body>
 </html>`
 
-async function startProbeServer(): Promise<{ url: string; close: () => Promise<void> }> {
+export async function startProbeServer(): Promise<{ url: string; close: () => Promise<void> }> {
   const server: Server = createServer((request, response) => {
     response.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' })
     const address = server.address()
@@ -110,7 +110,11 @@ function readProperty(value: unknown, key: string): unknown {
     : undefined
 }
 
-async function rpc(page: Page, method: string, params: Record<string, unknown>): Promise<unknown> {
+export async function rpc(
+  page: Page,
+  method: string,
+  params: Record<string, unknown>
+): Promise<unknown> {
   const response: unknown = await page.evaluate(
     ({ targetMethod, targetParams }) =>
       window.api.runtime.call({ method: targetMethod, params: targetParams }),
@@ -165,7 +169,7 @@ export async function readHost(page: Page): Promise<{ log: string[]; terminalFoc
 }
 
 /** Opens the probe in a browser tab, then puts the terminal back in front of it. */
-async function openBackgroundProbeTab(page: Page, url: string): Promise<string> {
+export async function openBackgroundProbeTab(page: Page, url: string): Promise<string> {
   await waitForActiveWorktree(page)
   await ensureTerminalVisible(page)
   const worktreeId = await getActiveWorktreeId(page)