feat(dictation): dictate into the focused field of a page in the browser pane (#40191)

* feat(dictation): dictate into the focused field of a page in the browser pane

* test(dictation): cover dictation into a focused browser page

* fix(dictation): type page dictation through the frame that owns the field

* fix(dictation): space page rich-text dictation within its block and offer editors the beforeinput

* test(dictation): simplify the page insertion test mock

* fix(dictation): copy page dictation to the clipboard when an editor blocks the input

src/main/browser/dictation/browser-guest-dictation-text.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it, vi } from 'vitest'
+import { insertDictationTextIntoGuest } from './browser-guest-dictation-text'
+
+function guestWithFocusedField(field: unknown) {
+  const executeJavaScript = vi
+    .fn<(code: string) => Promise<unknown>>()
+    .mockResolvedValueOnce(field)
+    .mockResolvedValueOnce(true)
+  const frame = { executeJavaScript }
+  return { guest: { focusedFrame: frame, mainFrame: frame }, executeJavaScript }
+}
+
+describe('insertDictationTextIntoGuest', () => {
+  it('types the text into the focused field, spaced from the words around the caret', async () => {
+    const { guest, executeJavaScript } = guestWithFocusedField({ before: 'Hello', after: 'again' })
+
+    await expect(insertDictationTextIntoGuest(guest, 'there')).resolves.toBe(true)
+
+    expect(executeJavaScript).toHaveBeenCalledTimes(2)
+    expect(executeJavaScript.mock.calls[1][0]).toContain('" there "')
+  })
+
+  it('reports nothing landed when the page has no editable field focused', async () => {
+    const { guest, executeJavaScript } = guestWithFocusedField(null)
+
+    await expect(insertDictationTextIntoGuest(guest, 'there')).resolves.toBe(false)
+
+    expect(executeJavaScript).toHaveBeenCalledTimes(1)
+  })
+})

src/main/browser/dictation/browser-guest-dictation-text.ts

@@ -0,0 +1,92 @@
+import {
+  DICTATION_TEXT_BLOCK_SELECTOR,
+  dictationSeparators
+} from '../../../shared/dictation-transcript-spacing'
+
+// The focused text field of this frame's document, looking through open shadow roots, or null.
+const FOCUSED_FIELD_EXPRESSION = `(() => {
+  let field = document.activeElement
+  while (field?.shadowRoot?.activeElement) field = field.shadowRoot.activeElement
+  const typedInto = field instanceof HTMLTextAreaElement ||
+    (field instanceof HTMLInputElement && ['text', 'search', 'url', 'tel', 'email'].includes(field.type))
+  if (typedInto) return field.readOnly || field.disabled ? null : field
+  return field instanceof HTMLElement && field.isContentEditable ? field : null
+})()`
+
+// Text around the caret of the focused field, so dictated words are spaced from their neighbours.
+const READ_FOCUSED_FIELD_SCRIPT = `(() => {
+  const limit = 200
+  const field = ${FOCUSED_FIELD_EXPRESSION}
+  if (!field) return null
+  if (!field.isContentEditable) {
+    const start = field.selectionStart ?? field.value.length
+    const end = field.selectionEnd ?? start
+    return {
+      before: field.value.slice(Math.max(0, start - limit), start),
+      after: field.value.slice(end, end + limit)
+    }
+  }
+  const root = field.getRootNode()
+  const selection = root.getSelection ? root.getSelection() : document.getSelection()
+  if (!selection || selection.rangeCount === 0) return { before: '', after: '' }
+  const caret = selection.getRangeAt(0)
+  const blockOf = (node) => {
+    const block = (node instanceof Element ? node : node.parentElement)?.closest(${JSON.stringify(DICTATION_TEXT_BLOCK_SELECTOR)})
+    return block && field.contains(block) ? block : field
+  }
+  const before = document.createRange()
+  before.selectNodeContents(blockOf(caret.startContainer))
+  before.setEnd(caret.startContainer, caret.startOffset)
+  const after = document.createRange()
+  after.selectNodeContents(blockOf(caret.endContainer))
+  after.setStart(caret.endContainer, caret.endOffset)
+  return { before: before.toString().slice(-limit), after: after.toString().slice(0, limit) }
+})()`
+
+// execCommand types into the frame that owns the field; webContents.insertText only reaches the top frame.
+// Rich-text editors get a cancelable beforeinput first; a cancel counts as landed only if the field's text then changes.
+const typeIntoFocusedFieldScript = (text: string): string => `(() => {
+  const text = ${JSON.stringify(text)}
+  const field = ${FOCUSED_FIELD_EXPRESSION}
+  if (!field) return false
+  if (field.isContentEditable) {
+    const previous = field.textContent
+    const beforeInput = new InputEvent('beforeinput', { bubbles: true, cancelable: true, inputType: 'insertText', data: text })
+    if (!field.dispatchEvent(beforeInput)) {
+      return new Promise((resolve) => setTimeout(() => resolve(field.textContent !== previous), 50))
+    }
+  }
+  return document.execCommand('insertText', false, text)
+})()`
+
+type FocusedFieldText = { before: string; after: string }
+type ScriptFrame = { executeJavaScript(code: string): Promise<unknown> }
+type DictationGuest = { focusedFrame: ScriptFrame | null; mainFrame: ScriptFrame }
+
+function isFocusedFieldText(value: unknown): value is FocusedFieldText {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'before' in value &&
+    'after' in value &&
+    typeof value.before === 'string' &&
+    typeof value.after === 'string'
+  )
+}
+
+/** Types dictated text into the page's focused field, spaced from its neighbours; false when no field is focused. */
+export async function insertDictationTextIntoGuest(
+  guest: DictationGuest,
+  text: string
+): Promise<boolean> {
+  const frame = guest.focusedFrame ?? guest.mainFrame
+  const field = await frame.executeJavaScript(READ_FOCUSED_FIELD_SCRIPT)
+  if (!isFocusedFieldText(field)) {
+    return false
+  }
+  const { leading, trailing } = dictationSeparators(field.before, text, field.after)
+  const typed = await frame.executeJavaScript(
+    typeIntoFocusedFieldScript(`${leading}${text}${trailing}`)
+  )
+  return typed === true
+}

src/main/browser/guest-lifecycle/input/shortcuts/browser-guest-dictation-shortcut.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('electron', async () => {
+  const { EventEmitter } = await import('node:events')
+  return {
+    BrowserWindow: class {
+      webContents = new (class extends EventEmitter {
+        send = vi.fn()
+      })()
+    },
+    screen: { getCursorScreenPoint: vi.fn(() => ({ x: 0, y: 0 })) },
+    webContents: { fromId: vi.fn() }
+  }
+})
+
+import { BrowserWindow } from 'electron'
+import { setupGuestShortcutForwarding } from './browser-guest-shortcut-forwarding'
+
+function pressDictationInPage(isComposing: boolean) {
+  const guest = new BrowserWindow().webContents
+  const renderer = new BrowserWindow().webContents
+  const cleanup = setupGuestShortcutForwarding({
+    browserTabId: 'tab-1',
+    guest,
+    resolveRenderer: () => renderer,
+    shouldForwardDictationShortcut: () => true,
+    getKeybindings: () => ({})
+  })
+  const isMac = process.platform === 'darwin'
+  const event: Electron.Event = { preventDefault: vi.fn(), defaultPrevented: false }
+  const input: Electron.Input = {
+    type: 'keyDown',
+    code: 'KeyE',
+    key: 'e',
+    meta: isMac,
+    control: !isMac,
+    alt: false,
+    shift: false,
+    isAutoRepeat: false,
+    isComposing,
+    location: 0,
+    modifiers: []
+  }
+  guest.emit('before-input-event', event, input)
+  cleanup()
+  return { prevented: vi.mocked(event.preventDefault).mock.calls.length > 0, renderer }
+}
+
+describe('dictation shortcut from a focused browser page', () => {
+  it('leaves the chord to the page while an IME composition owns it', () => {
+    const composing = pressDictationInPage(true)
+    expect(composing.prevented).toBe(false)
+    expect(composing.renderer.send).not.toHaveBeenCalledWith('ui:dictationKeyDown')
+
+    const typed = pressDictationInPage(false)
+    expect(typed.prevented).toBe(true)
+    expect(typed.renderer.send).toHaveBeenCalledWith('ui:dictationKeyDown')
+  })
+})

src/main/browser/guest-lifecycle/input/shortcuts/browser-guest-shortcut-dispatch.ts

@@ -10,10 +10,14 @@ import type { BrowserFindTarget } from '../../../../../shared/browser-find-sourc
 import type { BrowserPageTabShortcut } from '../../../../../shared/browser-page-command-target'
 import type { ResolveRenderer } from '../../browser-guest-renderer-target'
 import { matchesGuestTabShortcut } from './browser-guest-tab-shortcut'
+import { textInputOwnsComposition } from '../../../../../shared/text-input-shortcut-policy'
 
 export type ShouldForwardDictationShortcut = () => boolean
 export type IsMobileEmulatorEnabled = () => boolean
-export type GuestShortcutInput = WindowShortcutInput & { isAutoRepeat?: boolean }
+export type GuestShortcutInput = WindowShortcutInput & {
+  isAutoRepeat?: boolean
+  isComposing?: boolean
+}
 
 export type GuestShortcutForwardContext = {
   browserTabId: string
@@ -47,6 +51,11 @@ export function forwardGuestShortcutInput(
   } = ctx
   const page = ctx.browserPage !== false
   const keybindings = getKeybindings?.()
+  // An IME composition keeps the key, as in the main window.
+  const forwardsDictation =
+    action?.type === 'dictationKeyDown' &&
+    !textInputOwnsComposition(input) &&
+    shouldForwardDictationShortcut?.() === true
   if (action?.type === 'zoom') {
     if (!page) {
       // Why: a plugin frame has no page zoom; Orca zooms what it would with focus outside a pane.
@@ -59,7 +68,7 @@ export function forwardGuestShortcutInput(
   }
   if (input.isAutoRepeat) {
     if (
-      (action?.type === 'dictationKeyDown' && shouldForwardDictationShortcut?.()) ||
+      forwardsDictation ||
       action?.type === 'deleteCurrentWorkspace' ||
       // Why: Orca claims a held tab chord on every repeat but acts once, so repeats never reach the page.
       matchesGuestTabShortcut(
@@ -204,7 +213,7 @@ export function forwardGuestShortcutInput(
       renderer.send('ui:jumpToTabIndex', action.index)
     }
   } else if (action?.type === 'dictationKeyDown') {
-    if (!shouldForwardDictationShortcut?.()) {
+    if (!forwardsDictation) {
       return false
     }
     renderer.send('ui:dictationKeyDown')

src/main/ipc/browser/browser-dictation-ipc.ts

@@ -0,0 +1,16 @@
+import { ipcMain } from 'electron'
+import { insertDictationTextIntoGuest } from '../../browser/dictation/browser-guest-dictation-text'
+import { pageGuest } from './browser-guest-view-ipc'
+
+export function registerBrowserDictationHandlers(): void {
+  ipcMain.removeHandler('browser:insertDictationText')
+  ipcMain.handle(
+    'browser:insertDictationText',
+    (event, args: { browserPageId: string; text: unknown }): Promise<boolean> | false => {
+      const guest = pageGuest(event, args?.browserPageId)
+      return guest && typeof args.text === 'string' && args.text
+        ? insertDictationTextIntoGuest(guest, args.text).catch(() => false)
+        : false
+    }
+  )
+}

src/main/ipc/browser/browser-guest-view-ipc.ts

@@ -32,6 +32,21 @@ function isPagePoint(value: unknown): value is { x: number; y: number } {
   )
 }
 
+// Page/session operations exclude document previews.
+const pageGuestId = (event: IpcMainInvokeEvent, browserPageId: unknown): number | null =>
+  isTrustedBrowserRenderer(event.sender) && typeof browserPageId === 'string'
+    ? browserManager.getGuestWebContentsId(browserPageId)
+    : null
+
+export function pageGuest(
+  event: IpcMainInvokeEvent,
+  browserPageId: unknown
+): Electron.WebContents | null {
+  const guestId = pageGuestId(event, browserPageId)
+  const guest = guestId === null ? undefined : webContents.fromId(guestId)
+  return guest && !guest.isDestroyed() ? guest : null
+}
+
 export function registerBrowserGuestViewHandlers(): void {
   ipcMain.removeHandler('browser:openDevTools')
   ipcMain.removeHandler('browser:closeDevTools')
@@ -49,21 +64,6 @@ export function registerBrowserGuestViewHandlers(): void {
   ipcMain.removeHandler('browser:setPageSiteColorScheme')
   ipcMain.removeHandler('browser:setPageAudioMuted')
 
-  // Page/session operations exclude document previews.
-  const pageGuestId = (event: IpcMainInvokeEvent, browserPageId: unknown): number | null =>
-    isTrustedBrowserRenderer(event.sender) && typeof browserPageId === 'string'
-      ? browserManager.getGuestWebContentsId(browserPageId)
-      : null
-
-  const pageGuest = (
-    event: IpcMainInvokeEvent,
-    browserPageId: unknown
-  ): Electron.WebContents | null => {
-    const guestId = pageGuestId(event, browserPageId)
-    const guest = guestId === null ? undefined : webContents.fromId(guestId)
-    return guest && !guest.isDestroyed() ? guest : null
-  }
-
   ipcMain.handle(
     'browser:getPageSiteDisplay',
     (event, args: { browserPageId: string }): BrowserPageSiteDisplay | null => {

src/main/ipc/browser/browser.ts

@@ -11,6 +11,7 @@ import {
   resolveTabRegistrationWaiters
 } from './browser-tab-registration-wait'
 import { registerBrowserGuestViewHandlers } from './browser-guest-view-ipc'
+import { registerBrowserDictationHandlers } from './browser-dictation-ipc'
 import {
   disposeGrabModeStateForPage,
   registerBrowserGrabHandlers,
@@ -250,6 +251,7 @@ export function registerBrowserHandlers(store: BrowserPasswordsSettingsSource):
   })
 
   registerBrowserGuestViewHandlers()
+  registerBrowserDictationHandlers()
 
   registerBrowserGrabHandlers()
 

src/main/window/createMainWindow-terminal-focus-shortcuts.test.ts

@@ -15,6 +15,7 @@ vi.mock('../browser/guest-lifecycle/manager/browser-manager', async () =>
 )
 
 import { createMainWindow } from './createMainWindow'
+import { browserManager } from '../browser/guest-lifecycle/manager/browser-manager'
 import { ipcMain } from 'electron'
 import { resetExpectedTeardownStateForTest } from '../crash-reporting/expected-teardown-state'
 import { browserWindowMock, resetMainWindowMocks } from './createMainWindow-test-harness'
@@ -69,6 +70,10 @@ describe('createMainWindow', () => {
       getSettings: () => ({ windowBackgroundBlur: false, voice }) as never,
       updateUI: vi.fn()
     } as never)
+    // A focused browser page forwards only keydown, so it hands the chord over unless Hold needs the release.
+    const pageForwardsDictation = vi.mocked(browserManager.setDictationShortcutForwardingPredicate)
+      .mock.lastCall?.[0]
+    expect(pageForwardsDictation?.()).toBe(false)
 
     const isDarwin = process.platform === 'darwin'
     const dictationInput = {
@@ -91,6 +96,7 @@ describe('createMainWindow', () => {
 
     // #18691: with no model the renderer still needs the chord to show its setup toast.
     voice.enabled = true
+    expect(pageForwardsDictation?.()).toBe(true)
     const noModelPreventDefault = vi.fn()
     windowHandlers['before-input-event']({ preventDefault: noModelPreventDefault }, dictationInput)
     expect(noModelPreventDefault).toHaveBeenCalledTimes(1)
@@ -99,6 +105,7 @@ describe('createMainWindow', () => {
     webContents.send.mockClear()
     voice.sttModel = 'test-model'
     voice.dictationMode = 'hold'
+    expect(pageForwardsDictation?.()).toBe(false)
     const holdPreventDefault = vi.fn()
     windowHandlers['before-input-event'](
       { preventDefault: holdPreventDefault } as never,
@@ -108,6 +115,7 @@ describe('createMainWindow', () => {
     expect(webContents.send).not.toHaveBeenCalledWith('ui:dictationKeyDown')
 
     voice.dictationMode = 'auto'
+    expect(pageForwardsDictation?.()).toBe(true)
     const autoPreventDefault = vi.fn()
     windowHandlers['before-input-event']({ preventDefault: autoPreventDefault }, dictationInput)
     expect(autoPreventDefault).not.toHaveBeenCalled()

src/main/window/createMainWindow.ts

@@ -100,8 +100,9 @@ export function createMainWindow(
 
   const settings = store?.getSettings()
   browserManager.setDictationShortcutForwardingPredicate(() => {
-    // Webview guests have no safe transcript target; yield the dictation shortcut to the page.
-    return false
+    const voice = store?.getSettings().voice
+    // Guests forward only keydown, so Hold's release can't reach the renderer; the page keeps the key then.
+    return voice?.enabled === true && voice.dictationMode !== 'hold'
   })
   const windowMaterial = resolveWindowMaterial(
     settings,

src/preload/api/browser/browser-api.ts

@@ -130,6 +130,8 @@ export type BrowserApi = {
     browserPageId: string
     point: { x: number; y: number }
   }) => Promise<boolean>
+  /** Types dictated text into the page's focused field; false when no field is focused. */
+  insertDictationText: (args: { browserPageId: string; text: string }) => Promise<boolean>
   setViewportOverride: (args: {
     browserPageId: string
     override: BrowserViewportOverride | null

src/preload/api/browser/browser-bridge-guest-registration-and-downloads.ts

@@ -102,6 +102,8 @@ export const browserGuestRegistrationAndDownloadsApi = {
     browserPageId: string
     point: { x: number; y: number }
   }): Promise<boolean> => ipcRenderer.invoke('browser:copyImageAt', args),
+  insertDictationText: (args: { browserPageId: string; text: string }): Promise<boolean> =>
+    ipcRenderer.invoke('browser:insertDictationText', args),
   setViewportOverride: (args: {
     browserPageId: string
     override: BrowserViewportOverride | null

src/renderer/src/components/dictation/dictation-insertion-target.test.ts

@@ -6,7 +6,7 @@ import {
   TEXT_CONTROL_PASTE_DIRECT_MAX_BYTES,
   TEXT_CONTROL_PASTE_MAX_BYTES
 } from '@/lib/text-control-paste'
-import { insertText } from './dictation-insertion-target'
+import { captureInsertionTarget, insertText } from './dictation-insertion-target'
 
 type DocumentWithExecCommand = Document & {
   execCommand?: (commandId: string, showUi?: boolean, value?: string) => boolean
@@ -18,6 +18,7 @@ afterEach(() => {
   document.body.replaceChildren()
   vi.useRealTimers()
   vi.restoreAllMocks()
+  vi.unstubAllGlobals()
   Object.defineProperty(document, 'execCommand', {
     configurable: true,
     value: originalExecCommand
@@ -45,6 +46,24 @@ function installExecCommandMock(
 }
 
 describe('dictation insertion target', () => {
+  it('sends text dictated into a focused browser page to that page', async () => {
+    const insertDictationText = vi.fn(async () => true)
+    vi.stubGlobal('api', { browser: { insertDictationText } })
+    const viewport = document.createElement('div')
+    viewport.dataset.browserPageViewportId = 'page-1'
+    const guest = document.createElement('webview')
+    guest.tabIndex = 0
+    viewport.appendChild(guest)
+    document.body.appendChild(viewport)
+    guest.focus()
+
+    const target = captureInsertionTarget()
+    expect(target).toEqual({ kind: 'browser', browserPageId: 'page-1' })
+    await expect(insertText('hello', target!)).resolves.toBe(true)
+
+    expect(insertDictationText).toHaveBeenCalledWith({ browserPageId: 'page-1', text: 'hello' })
+  })
+
   it('cancels chunked contenteditable dictation when the target disconnects', async () => {
     vi.useFakeTimers()
     const editor = appendContentEditable()

src/renderer/src/components/dictation/dictation-insertion-target.ts

@@ -8,7 +8,11 @@ import {
   measureTextControlPasteByteLengthWithYield,
   pasteTextIntoTextControl
 } from '@/lib/text-control-paste'
-import { dictationSeparators } from '../../../../shared/dictation-transcript-spacing'
+import {
+  DICTATION_TEXT_BLOCK_SELECTOR,
+  dictationSeparators
+} from '../../../../shared/dictation-transcript-spacing'
+import { focusedBrowserPageId } from '@/hooks/resolve-zoom-target'
 import { DICTATION_FOCUS_INPUT_EVENT } from './dictation-control-events'
 
 /** Each returns whether the text landed. */
@@ -18,6 +22,7 @@ type TerminalInsert = (text: string, paneId: number) => Promise<boolean>
 export type DictationInsertionTarget =
   | { kind: 'composer'; insert: ComposerInsert; root: HTMLElement }
   | { kind: 'terminal'; tabId: string; paneId: number; pane: HTMLElement }
+  | { kind: 'browser'; browserPageId: string }
   | { kind: 'text'; element: HTMLInputElement | HTMLTextAreaElement }
   | { kind: 'contentEditable'; element: HTMLElement }
 
@@ -81,6 +86,15 @@ function readFocusedInsertionTarget(): DictationInsertionTarget | null {
     return null
   }
 
+  if (activeElement.tagName === 'WEBVIEW') {
+    const browserPageId =
+      focusedBrowserPageId(activeElement) ??
+      activeElement
+        .closest('[data-browser-client-page-id]')
+        ?.getAttribute('data-browser-client-page-id')
+    return browserPageId ? { kind: 'browser', browserPageId } : null
+  }
+
   if (activeElement instanceof HTMLInputElement || activeElement instanceof HTMLTextAreaElement) {
     return { kind: 'text', element: activeElement }
   }
@@ -103,9 +117,6 @@ function readSelectionRangeIn(element: HTMLElement): Range | null {
   return range && element.contains(range.commonAncestorContainer) ? range : null
 }
 
-const CONTENT_EDITABLE_BLOCK_SELECTOR =
-  'p, li, h1, h2, h3, h4, h5, h6, pre, blockquote, td, th, div'
-
 /** The selection's neighbours within its own block, so a new paragraph's start has none. */
 function readContentEditableNeighbours(element: HTMLElement): { before: string; after: string } {
   const range = readSelectionRangeIn(element)
@@ -114,7 +125,7 @@ function readContentEditableNeighbours(element: HTMLElement): { before: string;
   }
   const blockOf = (node: Node): Node => {
     const block = (node instanceof Element ? node : node.parentElement)?.closest(
-      CONTENT_EDITABLE_BLOCK_SELECTOR
+      DICTATION_TEXT_BLOCK_SELECTOR
     )
     return block && element.contains(block) ? block : element
   }
@@ -143,6 +154,13 @@ export async function insertText(
     const insert = terminalTargets.get(target.tabId)
     return insert ? insert(spaced(text, previousText, ''), target.paneId) : false
   }
+  if (target.kind === 'browser') {
+    // The page spaces the text from its own field.
+    return (
+      window.api.browser?.insertDictationText({ browserPageId: target.browserPageId, text }) ??
+      false
+    )
+  }
   if (target.kind === 'text') {
     const element = target.element
     if (!element.isConnected) {

src/shared/dictation-transcript-spacing.ts

@@ -1,3 +1,7 @@
+/** Blocks that bound a rich-text caret's neighbours, so a new paragraph's start has none. */
+export const DICTATION_TEXT_BLOCK_SELECTOR =
+  'p, li, h1, h2, h3, h4, h5, h6, pre, blockquote, td, th, div'
+
 const CJK_BOUNDARY_CHAR_RE =
   /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]$/u
 const NO_SPACE_BEFORE_CHAR_RE = /^[,.;:!?%。，、！？；：）)\]}”’»]$/u