mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
fix(browser): press keys through a US-layout CDP key table instead of a subprocess per keystroke (#15310)
Typing in the remote browser pane spawned an agent-browser process per keystroke -- ~160ms each, so a 17-character password took seconds -- and some keys arrived half-formed: F-keys, Insert and ContextMenu dispatched windowsVirtualKeyCode 0, Shift+1 typed '1' instead of '!', and non-ASCII printables reported success while typing nothing at all. keypress now resolves the key name through a US-layout table and dispatches the Input.dispatchKeyEvent pair over the electron debugger, the same transport mouseClick already uses. Two fallbacks keep the old behavior reachable: - a single printable BMP character outside the table dispatches in process as an IME-style event (keyCode 229 with the character as text, the shape composed input already has when it reaches pages) - anything else -- media keys, surrogate pairs, unrecognized names -- goes to the helper exactly as before, and only that path pays for creating the helper session Virtual key codes come from the table, never from the character's own char code: charCodeAt puts '&' on 38 (VK_UP) and '.' on 46 (VK_DELETE), which Blink runs as caret commands that swallow the character. Dispatch failures normalize the way evaluate's already do -- a gone page becomes browser_tab_not_found, anything else browser_error -- because attach and sendCommand reject with plain Errors that the RPC layer would report as runtime_error, and the pane only reclaims a dead page when it sees a browser_* code. Result shape is unchanged and no wire, schema or RPC surface moves, so mixed-version client/server pairs see no difference. Pages can observe the fidelity fixes: Shift+a now types 'A', Shift+1 now types '!', Alt+<char> no longer carries text, and editing keys arrive as rawKeyDown. Each matches what a real US keyboard produces. Verified against the shipped agent-browser 0.27 binary on the same browser: every difference is a fix, nothing regressed. macOS editing shortcuts (Cmd+A) do not fire through either path -- Blink runs those off the native responder chain and neither sends CDP `commands` -- so that gap is unchanged, not introduced. Co-authored-by: Neil <neil@stably.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Neil
Claude Fable 5
parent
e944e76537
commit
a28cd9eae5
@@ -12,6 +12,8 @@ import type {
|
||||
} from '../../shared/runtime-types'
|
||||
import { BrowserError } from './cdp-bridge'
|
||||
import { WAIT_PROCESS_TIMEOUT_GRACE_MS } from './agent-browser-bridge-types'
|
||||
import { acquireElectronDebugger } from './electron-debugger-lease'
|
||||
import { parseCdpKeyEvent, imeFallbackKeyEvent } from './cdp-keyboard-us-layout'
|
||||
import { AgentBrowserBridgeCaptureCommands } from './agent-browser-bridge-capture-commands'
|
||||
|
||||
export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowserBridgeCaptureCommands {
|
||||
@@ -170,9 +172,70 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
|
||||
worktreeId?: string,
|
||||
browserPageId?: string
|
||||
): Promise<BrowserKeypressResult> {
|
||||
return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
|
||||
return (await this.execAgentBrowser(sessionName, ['press', key])) as BrowserKeypressResult
|
||||
})
|
||||
return this.enqueueTargetedCommand(
|
||||
worktreeId,
|
||||
browserPageId,
|
||||
async (sessionName, target) => {
|
||||
const parsed = parseCdpKeyEvent(key) ?? imeFallbackKeyEvent(key)
|
||||
if (!parsed) {
|
||||
// Why: a key name the table cannot express must not dispatch keyCode 0 and
|
||||
// report success — route it to the helper, creating its session only now so
|
||||
// the direct path never pays for it.
|
||||
await this.ensureSession(sessionName, target.browserPageId, target.webContentsId)
|
||||
return (await this.execAgentBrowser(sessionName, ['press', key])) as BrowserKeypressResult
|
||||
}
|
||||
const wc = this.getWebContents(target.webContentsId)
|
||||
if (!wc || wc.isDestroyed()) {
|
||||
throw new BrowserError(
|
||||
'browser_tab_not_found',
|
||||
`Browser page ${target.browserPageId} is no longer available`
|
||||
)
|
||||
}
|
||||
const event = {
|
||||
windowsVirtualKeyCode: parsed.keyCode,
|
||||
nativeVirtualKeyCode: parsed.keyCode,
|
||||
key: parsed.key,
|
||||
code: parsed.code,
|
||||
modifiers: parsed.modifiers,
|
||||
location: parsed.location
|
||||
}
|
||||
let releaseDebugger = (): void => {}
|
||||
try {
|
||||
releaseDebugger = acquireElectronDebugger(wc).release
|
||||
await wc.debugger.sendCommand('Input.dispatchKeyEvent', {
|
||||
// Why: rawKeyDown is the no-character form; sending keyDown without text
|
||||
// makes Blink synthesize an empty input for editing keys.
|
||||
type: parsed.text === null ? 'rawKeyDown' : 'keyDown',
|
||||
...event,
|
||||
...(parsed.text === null ? {} : { text: parsed.text, unmodifiedText: parsed.text })
|
||||
})
|
||||
await wc.debugger.sendCommand('Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
...event,
|
||||
// Why: the self bit is keydown-only -- Blink reports shiftKey false on the Shift keyup.
|
||||
modifiers: parsed.modifiers & ~parsed.selfModifier
|
||||
})
|
||||
return { pressed: key }
|
||||
} catch (error) {
|
||||
// Why: attach/dispatch reject with plain Errors, which the RPC layer would report as
|
||||
// runtime_error — the helper path this replaced always produced a browser_* code, and
|
||||
// the pane only reclaims a dead page when it sees one.
|
||||
if (error instanceof BrowserError) {
|
||||
throw error
|
||||
}
|
||||
if (!this.getWebContents(target.webContentsId)) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
throw new BrowserError(
|
||||
'browser_error',
|
||||
`Failed to press ${key} in browser page ${target.browserPageId}: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
} finally {
|
||||
releaseDebugger()
|
||||
}
|
||||
},
|
||||
{ ensureSession: false }
|
||||
)
|
||||
}
|
||||
|
||||
async pdf(worktreeId?: string, browserPageId?: string): Promise<BrowserPdfResult> {
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
|
||||
const { execFileMock, webContentsFromIdMock, existsSyncMock, readFileSyncMock, stdinWrites } =
|
||||
vi.hoisted(() => {
|
||||
const stdinWrites: string[] = []
|
||||
return {
|
||||
execFileMock: vi.fn(),
|
||||
webContentsFromIdMock: vi.fn(),
|
||||
existsSyncMock: vi.fn(() => false),
|
||||
readFileSyncMock: vi.fn(() => Buffer.from('')),
|
||||
stdinWrites
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('child_process', () => ({ execFile: execFileMock }))
|
||||
vi.mock('fs', () => ({
|
||||
existsSync: existsSyncMock,
|
||||
readFileSync: readFileSyncMock,
|
||||
accessSync: vi.fn(),
|
||||
chmodSync: vi.fn(),
|
||||
constants: { X_OK: 1 }
|
||||
}))
|
||||
vi.mock('os', () => ({ platform: () => 'darwin', arch: () => 'arm64' }))
|
||||
vi.mock('electron', () => {
|
||||
return {
|
||||
app: { getPath: vi.fn(() => '/app'), getAppPath: vi.fn(() => '/project'), isPackaged: false },
|
||||
webContents: { fromId: webContentsFromIdMock }
|
||||
}
|
||||
})
|
||||
const { CdpWsProxyMock } = vi.hoisted(() => {
|
||||
const instances: unknown[] = []
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const MockClass = vi.fn().mockImplementation(function (this: any, _wc: unknown) {
|
||||
this._wc = _wc
|
||||
this.start = vi.fn(async () => 'ws://127.0.0.1:9222')
|
||||
this.stop = vi.fn(async () => {})
|
||||
this.getPort = vi.fn(() => 9222)
|
||||
instances.push(this)
|
||||
})
|
||||
return { CdpWsProxyMock: Object.assign(MockClass, { instances }) }
|
||||
})
|
||||
|
||||
vi.mock('./cdp-ws-proxy', () => ({
|
||||
CdpWsProxy: CdpWsProxyMock
|
||||
}))
|
||||
|
||||
import { AgentBrowserBridge } from './agent-browser-bridge'
|
||||
import {
|
||||
createSucceedWith,
|
||||
mockBrowserManager,
|
||||
mockWebContents,
|
||||
overrideBridgeWebContentsLookup,
|
||||
resetAgentBrowserBridgeMocks,
|
||||
type MockWebContents
|
||||
} from './agent-browser-bridge-test-harness'
|
||||
|
||||
overrideBridgeWebContentsLookup(AgentBrowserBridge.prototype, webContentsFromIdMock)
|
||||
|
||||
const succeedWith = createSucceedWith(execFileMock, stdinWrites)
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null
|
||||
}
|
||||
|
||||
function isStringArray(value: unknown): value is string[] {
|
||||
return Array.isArray(value) && value.every((entry) => typeof entry === 'string')
|
||||
}
|
||||
|
||||
function keyEventCalls(wc: MockWebContents): Record<string, unknown>[] {
|
||||
return wc.debugger.sendCommand.mock.calls
|
||||
.filter(([method]) => method === 'Input.dispatchKeyEvent')
|
||||
.map(([, params]) => params)
|
||||
.filter(isRecord)
|
||||
}
|
||||
|
||||
describe('AgentBrowserBridge keypress input', () => {
|
||||
let bridge: AgentBrowserBridge
|
||||
let wc: MockWebContents
|
||||
|
||||
beforeEach(() => {
|
||||
resetAgentBrowserBridgeMocks({
|
||||
webContentsFromIdMock,
|
||||
existsSyncMock,
|
||||
readFileSyncMock,
|
||||
stdinWrites,
|
||||
cdpWsProxyInstances: CdpWsProxyMock.instances
|
||||
})
|
||||
bridge = new AgentBrowserBridge(mockBrowserManager())
|
||||
bridge.setActiveTab(100)
|
||||
wc = mockWebContents(100)
|
||||
wc.debugger.sendCommand.mockResolvedValue({})
|
||||
webContentsFromIdMock.mockImplementation((id: number) => (id === 100 ? wc : null))
|
||||
})
|
||||
|
||||
it('dispatches a printable key over CDP without spawning agent-browser', async () => {
|
||||
await expect(bridge.keypress('a', undefined, 'tab-1')).resolves.toEqual({ pressed: 'a' })
|
||||
|
||||
expect(execFileMock).not.toHaveBeenCalled()
|
||||
expect(CdpWsProxyMock.instances).toHaveLength(0)
|
||||
// Why: exactly two CDP calls, so the dispatch pair is the whole interaction.
|
||||
expect(wc.debugger.sendCommand.mock.calls).toHaveLength(2)
|
||||
expect(keyEventCalls(wc)).toEqual([
|
||||
{
|
||||
type: 'keyDown',
|
||||
windowsVirtualKeyCode: 65,
|
||||
nativeVirtualKeyCode: 65,
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
modifiers: 0,
|
||||
location: 0,
|
||||
text: 'a',
|
||||
unmodifiedText: 'a'
|
||||
},
|
||||
{
|
||||
type: 'keyUp',
|
||||
windowsVirtualKeyCode: 65,
|
||||
nativeVirtualKeyCode: 65,
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
modifiers: 0,
|
||||
location: 0
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('types & as shifted 7 instead of colliding with the ArrowUp virtual key code', async () => {
|
||||
await bridge.keypress('&', undefined, 'tab-1')
|
||||
|
||||
expect(keyEventCalls(wc)[0]).toMatchObject({
|
||||
type: 'keyDown',
|
||||
windowsVirtualKeyCode: 55,
|
||||
modifiers: 8,
|
||||
text: '&'
|
||||
})
|
||||
})
|
||||
|
||||
it('dispatches editing and navigation keys as rawKeyDown with no text', async () => {
|
||||
await bridge.keypress('ArrowDown', undefined, 'tab-1')
|
||||
|
||||
expect(keyEventCalls(wc)[0]).toMatchObject({
|
||||
type: 'rawKeyDown',
|
||||
windowsVirtualKeyCode: 40,
|
||||
key: 'ArrowDown'
|
||||
})
|
||||
expect(keyEventCalls(wc)[0]).not.toHaveProperty('text')
|
||||
})
|
||||
|
||||
it('carries modifier masks for shortcuts', async () => {
|
||||
await expect(bridge.keypress('Ctrl+Shift+K', undefined, 'tab-1')).resolves.toEqual({
|
||||
pressed: 'Ctrl+Shift+K'
|
||||
})
|
||||
|
||||
expect(keyEventCalls(wc)[0]).toMatchObject({
|
||||
type: 'rawKeyDown',
|
||||
windowsVirtualKeyCode: 75,
|
||||
modifiers: 10
|
||||
})
|
||||
})
|
||||
|
||||
it('reports the modifier bit on a bare Shift keydown but not on its keyup', async () => {
|
||||
await bridge.keypress('Shift', undefined, 'tab-1')
|
||||
|
||||
expect(keyEventCalls(wc)[0]).toMatchObject({
|
||||
type: 'rawKeyDown',
|
||||
windowsVirtualKeyCode: 16,
|
||||
code: 'ShiftLeft',
|
||||
modifiers: 8,
|
||||
location: 1
|
||||
})
|
||||
expect(keyEventCalls(wc)[1]).toMatchObject({ type: 'keyUp', modifiers: 0, location: 1 })
|
||||
})
|
||||
|
||||
it('keeps held modifiers on the keyup of a non-modifier shortcut key', async () => {
|
||||
await bridge.keypress('Ctrl+Shift+K', undefined, 'tab-1')
|
||||
|
||||
expect(keyEventCalls(wc)[1]).toMatchObject({ type: 'keyUp', modifiers: 10 })
|
||||
})
|
||||
|
||||
it('presses Enter with its carriage-return text so fields submit', async () => {
|
||||
await bridge.keypress('Enter', undefined, 'tab-1')
|
||||
|
||||
expect(keyEventCalls(wc)[0]).toMatchObject({
|
||||
type: 'keyDown',
|
||||
windowsVirtualKeyCode: 13,
|
||||
text: '\r'
|
||||
})
|
||||
})
|
||||
|
||||
it('dispatches a non-US printable character as an IME-style event in process', async () => {
|
||||
await expect(bridge.keypress('é', undefined, 'tab-1')).resolves.toEqual({ pressed: 'é' })
|
||||
|
||||
expect(execFileMock).not.toHaveBeenCalled()
|
||||
expect(keyEventCalls(wc)[0]).toMatchObject({
|
||||
type: 'keyDown',
|
||||
windowsVirtualKeyCode: 229,
|
||||
key: 'é',
|
||||
code: '',
|
||||
text: 'é',
|
||||
unmodifiedText: 'é'
|
||||
})
|
||||
expect(keyEventCalls(wc)[1]).toMatchObject({ type: 'keyUp', windowsVirtualKeyCode: 229 })
|
||||
})
|
||||
|
||||
it('keeps the helper for a surrogate-pair character', async () => {
|
||||
succeedWith({ pressed: '👍' })
|
||||
|
||||
await expect(bridge.keypress('👍', undefined, 'tab-1')).resolves.toEqual({ pressed: '👍' })
|
||||
|
||||
expect(keyEventCalls(wc)).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('falls back to agent-browser for a key name the table cannot express', async () => {
|
||||
succeedWith({ pressed: 'MediaPlayPause' })
|
||||
|
||||
await expect(bridge.keypress('MediaPlayPause', undefined, 'tab-1')).resolves.toEqual({
|
||||
pressed: 'MediaPlayPause'
|
||||
})
|
||||
|
||||
expect(keyEventCalls(wc)).toHaveLength(0)
|
||||
const pressCall = execFileMock.mock.calls
|
||||
.map(([, commandArgs]) => commandArgs)
|
||||
.filter(isStringArray)
|
||||
.find((commandArgs) => commandArgs.includes('press'))
|
||||
expect(pressCall).toBeDefined()
|
||||
const args = pressCall ?? []
|
||||
expect(args[args.indexOf('press') + 1]).toBe('MediaPlayPause')
|
||||
})
|
||||
|
||||
it('rejects with tab not found when the page is gone', async () => {
|
||||
webContentsFromIdMock.mockReturnValue(null)
|
||||
|
||||
await expect(bridge.keypress('a', undefined, 'tab-1')).rejects.toMatchObject({
|
||||
code: 'browser_tab_not_found'
|
||||
})
|
||||
})
|
||||
|
||||
// Why: one keypress looks the page up three times — the queued target, the
|
||||
// automation-visibility refresh, then the dispatch guard. Serving the first N keeps the
|
||||
// later ones on the guard; the trailing assertions fail loudly if that count ever moves.
|
||||
function killPageAfterLookups(lookups: number): () => number {
|
||||
let remaining = lookups
|
||||
webContentsFromIdMock.mockImplementation((id: number) => {
|
||||
if (id !== 100 || remaining === 0) {
|
||||
return null
|
||||
}
|
||||
remaining -= 1
|
||||
return wc
|
||||
})
|
||||
return () => remaining
|
||||
}
|
||||
|
||||
it('rejects with tab not found when the page dies after its target is resolved', async () => {
|
||||
const remaining = killPageAfterLookups(2)
|
||||
|
||||
await expect(bridge.keypress('a', undefined, 'tab-1')).rejects.toMatchObject({
|
||||
code: 'browser_tab_not_found'
|
||||
})
|
||||
expect(remaining()).toBe(0)
|
||||
expect(keyEventCalls(wc)).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('rejects with tab not found when the page dies mid-dispatch', async () => {
|
||||
const remaining = killPageAfterLookups(3)
|
||||
wc.debugger.sendCommand.mockRejectedValue(new Error('Inspected target navigated or closed'))
|
||||
|
||||
await expect(bridge.keypress('a', undefined, 'tab-1')).rejects.toMatchObject({
|
||||
code: 'browser_tab_not_found'
|
||||
})
|
||||
expect(remaining()).toBe(0)
|
||||
})
|
||||
|
||||
it('reports a dispatch failure on a live page as a browser error', async () => {
|
||||
wc.debugger.sendCommand.mockRejectedValue(new Error('Debugger is not attached to the target'))
|
||||
|
||||
await expect(bridge.keypress('a', undefined, 'tab-1')).rejects.toMatchObject({
|
||||
code: 'browser_error',
|
||||
message: expect.stringContaining('Debugger is not attached to the target')
|
||||
})
|
||||
})
|
||||
|
||||
it('reports a debugger attach failure as a browser error', async () => {
|
||||
wc.debugger.isAttached.mockReturnValue(false)
|
||||
wc.debugger.attach.mockImplementation(() => {
|
||||
throw new Error('Another debugger is already attached to the debug target')
|
||||
})
|
||||
|
||||
await expect(bridge.keypress('a', undefined, 'tab-1')).rejects.toMatchObject({
|
||||
code: 'browser_error'
|
||||
})
|
||||
expect(keyEventCalls(wc)).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,133 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { imeFallbackKeyEvent, parseCdpKeyEvent } from './cdp-keyboard-us-layout'
|
||||
|
||||
describe('parseCdpKeyEvent', () => {
|
||||
it('maps every printable ASCII character to a key event that types that character', () => {
|
||||
const broken: string[] = []
|
||||
for (let charCode = 32; charCode <= 126; charCode++) {
|
||||
const ch = String.fromCharCode(charCode)
|
||||
const parsed = parseCdpKeyEvent(ch)
|
||||
if (!parsed || parsed.text !== ch || parsed.keyCode === 0) {
|
||||
broken.push(ch)
|
||||
}
|
||||
}
|
||||
expect(broken).toEqual([])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['#', 51],
|
||||
['$', 52],
|
||||
['%', 53],
|
||||
['&', 55],
|
||||
["'", 222],
|
||||
['(', 57],
|
||||
['.', 190]
|
||||
])(
|
||||
'gives %s the US-layout key code %i instead of its own char code',
|
||||
(ch: string, keyCode: number) => {
|
||||
// Why: charCodeAt-derived codes put '&' on VK_UP (38) and '.' on VK_DELETE (46),
|
||||
// which Blink executes as caret commands that swallow the character.
|
||||
expect(parseCdpKeyEvent(ch)).toMatchObject({ keyCode, text: ch })
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
['Ctrl+A', { keyCode: 65, key: 'a', modifiers: 2, text: null }],
|
||||
['Control+a', { keyCode: 65, key: 'a', modifiers: 2, text: null }],
|
||||
['Shift+Home', { keyCode: 36, key: 'Home', modifiers: 8, text: null }],
|
||||
['Alt+ArrowDown', { keyCode: 40, key: 'ArrowDown', modifiers: 1, text: null }],
|
||||
['Ctrl+Shift+K', { keyCode: 75, key: 'K', modifiers: 10, text: null }],
|
||||
['Meta+r', { keyCode: 82, key: 'r', modifiers: 4, text: null }],
|
||||
['Control+Shift+r', { keyCode: 82, key: 'R', modifiers: 10, text: null }]
|
||||
])('parses the shortcut %s', (raw: string, expected: object) => {
|
||||
expect(parseCdpKeyEvent(raw)).toMatchObject(expected)
|
||||
})
|
||||
|
||||
it('treats a capital letter in a shortcut as the key name, not a shift request', () => {
|
||||
expect(parseCdpKeyEvent('Ctrl+A')).toMatchObject({ key: 'a', modifiers: 2 })
|
||||
expect(parseCdpKeyEvent('Ctrl+Shift+A')).toMatchObject({ key: 'A', modifiers: 10 })
|
||||
})
|
||||
|
||||
it('shifts a bare capital letter and reports the shifted character as text', () => {
|
||||
expect(parseCdpKeyEvent('R')).toMatchObject({ keyCode: 82, key: 'R', modifiers: 8, text: 'R' })
|
||||
expect(parseCdpKeyEvent('Shift+a')).toMatchObject({ key: 'A', modifiers: 8, text: 'A' })
|
||||
})
|
||||
|
||||
it('maps shifted punctuation onto its base key with shift held', () => {
|
||||
expect(parseCdpKeyEvent('Shift+1')).toMatchObject({ keyCode: 49, key: '!', text: '!' })
|
||||
expect(parseCdpKeyEvent('+')).toMatchObject({ keyCode: 187, modifiers: 8, text: '+' })
|
||||
})
|
||||
|
||||
it.each([
|
||||
['Enter', { keyCode: 13, text: '\r' }],
|
||||
['Space', { keyCode: 32, key: ' ', text: ' ' }],
|
||||
['Esc', { keyCode: 27, key: 'Escape', text: null }],
|
||||
['PgDn', { keyCode: 34, key: 'PageDown', text: null }],
|
||||
['ContextMenu', { keyCode: 93, text: null }],
|
||||
['F5', { keyCode: 116, key: 'F5', code: 'F5', text: null }],
|
||||
['F12', { keyCode: 123, text: null }]
|
||||
])('parses the named key %s', (raw: string, expected: object) => {
|
||||
expect(parseCdpKeyEvent(raw)).toMatchObject(expected)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['Shift', { keyCode: 16, key: 'Shift', code: 'ShiftLeft', modifiers: 8, selfModifier: 8 }],
|
||||
['Ctrl', { keyCode: 17, key: 'Control', code: 'ControlLeft', modifiers: 2, selfModifier: 2 }],
|
||||
['Alt', { keyCode: 18, key: 'Alt', code: 'AltLeft', modifiers: 1, selfModifier: 1 }],
|
||||
['Meta', { keyCode: 91, key: 'Meta', code: 'MetaLeft', modifiers: 4, selfModifier: 4 }]
|
||||
])(
|
||||
'reports the own modifier bit and left-side location for a bare %s press',
|
||||
(raw: string, expected: object) => {
|
||||
expect(parseCdpKeyEvent(raw)).toMatchObject({ ...expected, location: 1, text: null })
|
||||
}
|
||||
)
|
||||
|
||||
it('adds the self bit on top of held modifiers for a modifier-only chord', () => {
|
||||
expect(parseCdpKeyEvent('Ctrl+Shift')).toMatchObject({
|
||||
keyCode: 16,
|
||||
modifiers: 10,
|
||||
selfModifier: 8
|
||||
})
|
||||
})
|
||||
|
||||
it('reports no self bit or location for non-modifier keys', () => {
|
||||
expect(parseCdpKeyEvent('Enter')).toMatchObject({ location: 0, selfModifier: 0 })
|
||||
expect(parseCdpKeyEvent('a')).toMatchObject({ location: 0, selfModifier: 0 })
|
||||
expect(parseCdpKeyEvent('Ctrl+A')).toMatchObject({ location: 0, selfModifier: 0 })
|
||||
})
|
||||
|
||||
it.each([['MediaPlayPause'], ['F25'], [''], ['NoSuchKey']])(
|
||||
'returns null for %s so the caller can fall back',
|
||||
(raw: string) => {
|
||||
expect(parseCdpKeyEvent(raw)).toBeNull()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('imeFallbackKeyEvent', () => {
|
||||
it.each([['é'], ['ß'], ['ñ'], ['ü'], ['漢'], ['한']])(
|
||||
'gives %s the IME key event form with keyCode 229 and its text',
|
||||
(ch: string) => {
|
||||
expect(imeFallbackKeyEvent(ch)).toEqual({
|
||||
keyCode: 229,
|
||||
key: ch,
|
||||
code: '',
|
||||
modifiers: 0,
|
||||
location: 0,
|
||||
selfModifier: 0,
|
||||
text: ch
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
['a table-covered ASCII character', 'a'],
|
||||
['a surrogate-pair emoji', '👍'],
|
||||
['a combining sequence', 'e\u0301'],
|
||||
['a multi-character name', 'MediaPlayPause'],
|
||||
['a chord with a non-US character', 'Ctrl+é'],
|
||||
['an empty string', '']
|
||||
])('returns null for %s so the helper keeps its behavior', (_name: string, raw: string) => {
|
||||
expect(imeFallbackKeyEvent(raw)).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,251 @@
|
||||
// Why: deriving a virtual key code from a character's own char code collides with editing
|
||||
// keys — '&' (38) arrives as VK_UP and '.' (46) as VK_DELETE, so Blink runs the caret
|
||||
// command and silently drops the character. This table maps Orca key names ("a", "&",
|
||||
// "Ctrl+Shift+K", "Alt+ArrowDown", "F5") to the CDP key event a US-layout keyboard
|
||||
// would produce; anything it cannot express returns null so the caller can fall back.
|
||||
|
||||
const CDP_MODIFIER_BITS: Record<string, number> = {
|
||||
alt: 1,
|
||||
option: 1,
|
||||
ctrl: 2,
|
||||
control: 2,
|
||||
cmd: 4,
|
||||
command: 4,
|
||||
meta: 4,
|
||||
super: 4,
|
||||
win: 4,
|
||||
shift: 8
|
||||
}
|
||||
|
||||
// name -> [windowsVirtualKeyCode, key, code, text]
|
||||
const CDP_NAMED_KEYS: Record<string, [number, string, string, string | null]> = {
|
||||
enter: [13, 'Enter', 'Enter', '\r'],
|
||||
return: [13, 'Enter', 'Enter', '\r'],
|
||||
tab: [9, 'Tab', 'Tab', null],
|
||||
backspace: [8, 'Backspace', 'Backspace', null],
|
||||
delete: [46, 'Delete', 'Delete', null],
|
||||
del: [46, 'Delete', 'Delete', null],
|
||||
escape: [27, 'Escape', 'Escape', null],
|
||||
esc: [27, 'Escape', 'Escape', null],
|
||||
space: [32, ' ', 'Space', ' '],
|
||||
spacebar: [32, ' ', 'Space', ' '],
|
||||
arrowup: [38, 'ArrowUp', 'ArrowUp', null],
|
||||
up: [38, 'ArrowUp', 'ArrowUp', null],
|
||||
arrowdown: [40, 'ArrowDown', 'ArrowDown', null],
|
||||
down: [40, 'ArrowDown', 'ArrowDown', null],
|
||||
arrowleft: [37, 'ArrowLeft', 'ArrowLeft', null],
|
||||
left: [37, 'ArrowLeft', 'ArrowLeft', null],
|
||||
arrowright: [39, 'ArrowRight', 'ArrowRight', null],
|
||||
right: [39, 'ArrowRight', 'ArrowRight', null],
|
||||
home: [36, 'Home', 'Home', null],
|
||||
end: [35, 'End', 'End', null],
|
||||
pageup: [33, 'PageUp', 'PageUp', null],
|
||||
pgup: [33, 'PageUp', 'PageUp', null],
|
||||
pagedown: [34, 'PageDown', 'PageDown', null],
|
||||
pgdn: [34, 'PageDown', 'PageDown', null],
|
||||
pgdown: [34, 'PageDown', 'PageDown', null],
|
||||
insert: [45, 'Insert', 'Insert', null],
|
||||
ins: [45, 'Insert', 'Insert', null],
|
||||
contextmenu: [93, 'ContextMenu', 'ContextMenu', null],
|
||||
apps: [93, 'ContextMenu', 'ContextMenu', null],
|
||||
capslock: [20, 'CapsLock', 'CapsLock', null],
|
||||
numlock: [144, 'NumLock', 'NumLock', null],
|
||||
scrolllock: [145, 'ScrollLock', 'ScrollLock', null],
|
||||
pause: [19, 'Pause', 'Pause', null],
|
||||
printscreen: [44, 'PrintScreen', 'PrintScreen', null],
|
||||
shift: [16, 'Shift', 'ShiftLeft', null],
|
||||
control: [17, 'Control', 'ControlLeft', null],
|
||||
ctrl: [17, 'Control', 'ControlLeft', null],
|
||||
alt: [18, 'Alt', 'AltLeft', null],
|
||||
option: [18, 'Alt', 'AltLeft', null],
|
||||
meta: [91, 'Meta', 'MetaLeft', null],
|
||||
cmd: [91, 'Meta', 'MetaLeft', null],
|
||||
command: [91, 'Meta', 'MetaLeft', null]
|
||||
}
|
||||
|
||||
// Characters a US keyboard produces with shift held, and the base key they share.
|
||||
const US_SHIFTED_CHARS: Record<string, string> = {
|
||||
'~': '`',
|
||||
'!': '1',
|
||||
'@': '2',
|
||||
'#': '3',
|
||||
$: '4',
|
||||
'%': '5',
|
||||
'^': '6',
|
||||
'&': '7',
|
||||
'*': '8',
|
||||
'(': '9',
|
||||
')': '0',
|
||||
_: '-',
|
||||
'+': '=',
|
||||
'{': '[',
|
||||
'}': ']',
|
||||
'|': '\\',
|
||||
':': ';',
|
||||
'"': "'",
|
||||
'<': ',',
|
||||
'>': '.',
|
||||
'?': '/'
|
||||
}
|
||||
|
||||
const US_SHIFT_OF: Record<string, string> = {}
|
||||
for (const shifted of Object.keys(US_SHIFTED_CHARS)) {
|
||||
US_SHIFT_OF[US_SHIFTED_CHARS[shifted]] = shifted
|
||||
}
|
||||
|
||||
// char -> [windowsVirtualKeyCode, code], for the keys that are not letters or digits.
|
||||
const US_PUNCTUATION_KEYS: Record<string, [number, string]> = {
|
||||
' ': [32, 'Space'],
|
||||
';': [186, 'Semicolon'],
|
||||
'=': [187, 'Equal'],
|
||||
',': [188, 'Comma'],
|
||||
'-': [189, 'Minus'],
|
||||
'.': [190, 'Period'],
|
||||
'/': [191, 'Slash'],
|
||||
'`': [192, 'Backquote'],
|
||||
'[': [219, 'BracketLeft'],
|
||||
'\\': [220, 'Backslash'],
|
||||
']': [221, 'BracketRight'],
|
||||
"'": [222, 'Quote']
|
||||
}
|
||||
|
||||
type UsKeyboardKey = {
|
||||
keyCode: number
|
||||
code: string
|
||||
shift: boolean
|
||||
}
|
||||
|
||||
function usKeyboardKeyForChar(ch: string): UsKeyboardKey | null {
|
||||
if (ch >= 'a' && ch <= 'z') {
|
||||
return { keyCode: ch.charCodeAt(0) - 32, code: `Key${ch.toUpperCase()}`, shift: false }
|
||||
}
|
||||
if (ch >= 'A' && ch <= 'Z') {
|
||||
return { keyCode: ch.charCodeAt(0), code: `Key${ch}`, shift: true }
|
||||
}
|
||||
if (ch >= '0' && ch <= '9') {
|
||||
return { keyCode: ch.charCodeAt(0), code: `Digit${ch}`, shift: false }
|
||||
}
|
||||
if (Object.hasOwn(US_SHIFTED_CHARS, ch)) {
|
||||
const base = usKeyboardKeyForChar(US_SHIFTED_CHARS[ch])
|
||||
return base === null ? null : { keyCode: base.keyCode, code: base.code, shift: true }
|
||||
}
|
||||
if (Object.hasOwn(US_PUNCTUATION_KEYS, ch)) {
|
||||
return { keyCode: US_PUNCTUATION_KEYS[ch][0], code: US_PUNCTUATION_KEYS[ch][1], shift: false }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export type CdpKeyEvent = {
|
||||
keyCode: number
|
||||
key: string
|
||||
code: string
|
||||
modifiers: number
|
||||
// Why: 1 = left-side key -- the table pins bare modifiers to ShiftLeft/ControlLeft/etc.
|
||||
location: number
|
||||
// Why: a modifier key's own bit is set during its keydown but already cleared on its keyup.
|
||||
selfModifier: number
|
||||
// Why: null means the key produces no character (a rawKeyDown, not a keyDown with text).
|
||||
text: string | null
|
||||
}
|
||||
|
||||
// Why: printable characters outside the table (accented letters, non-latin scripts)
|
||||
// still have an in-process form -- the IME convention, keyCode 229 with the text,
|
||||
// which is how composed input already reaches pages. One BMP code point only:
|
||||
// surrogate pairs and combining sequences keep the helper's behavior.
|
||||
export function imeFallbackKeyEvent(raw: string): CdpKeyEvent | null {
|
||||
if (raw.length !== 1) {
|
||||
return null
|
||||
}
|
||||
const codePoint = raw.charCodeAt(0)
|
||||
if (codePoint < 0xa0 || (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
|
||||
return null
|
||||
}
|
||||
return { keyCode: 229, key: raw, code: '', modifiers: 0, location: 0, selfModifier: 0, text: raw }
|
||||
}
|
||||
|
||||
export function parseCdpKeyEvent(raw: string): CdpKeyEvent | null {
|
||||
if (raw.length === 0) {
|
||||
return null
|
||||
}
|
||||
let rest = raw
|
||||
let modifiers = 0
|
||||
while (rest.length > 1) {
|
||||
const plus = rest.indexOf('+')
|
||||
if (plus <= 0) {
|
||||
break
|
||||
}
|
||||
const name = rest.slice(0, plus).toLowerCase()
|
||||
if (!Object.hasOwn(CDP_MODIFIER_BITS, name)) {
|
||||
break
|
||||
}
|
||||
modifiers |= CDP_MODIFIER_BITS[name]
|
||||
rest = rest.slice(plus + 1)
|
||||
}
|
||||
if (rest.length === 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
let keyCode: number
|
||||
let key: string
|
||||
let code: string
|
||||
let text: string | null
|
||||
let location = 0
|
||||
let selfModifier = 0
|
||||
if (rest.length === 1) {
|
||||
const mapped = usKeyboardKeyForChar(rest)
|
||||
if (mapped === null) {
|
||||
return null
|
||||
}
|
||||
keyCode = mapped.keyCode
|
||||
key = rest
|
||||
code = mapped.code
|
||||
text = rest
|
||||
// Why: a capital letter in a shortcut is how people write the key, not a request for
|
||||
// shift — Ctrl+A means select-all (key 'a'), never Ctrl+Shift+A. Shifted punctuation
|
||||
// is different: on a US keyboard shift is the only way to produce the character.
|
||||
const capitalShortcut = rest >= 'A' && rest <= 'Z' && (modifiers & ~8) !== 0
|
||||
if (capitalShortcut) {
|
||||
key = rest.toLowerCase()
|
||||
text = key
|
||||
} else if (mapped.shift) {
|
||||
modifiers |= 8
|
||||
}
|
||||
} else if (Object.hasOwn(CDP_NAMED_KEYS, rest.toLowerCase())) {
|
||||
const name = rest.toLowerCase()
|
||||
const named = CDP_NAMED_KEYS[name]
|
||||
keyCode = named[0]
|
||||
key = named[1]
|
||||
code = named[2]
|
||||
text = named[3]
|
||||
// Why: Blink reports a modifier's own bit during its keydown (shiftKey is true while
|
||||
// Shift goes down), and the table's modifier entries are the left-side keys.
|
||||
selfModifier = CDP_MODIFIER_BITS[name] ?? 0
|
||||
if (selfModifier !== 0) {
|
||||
modifiers |= selfModifier
|
||||
location = 1
|
||||
}
|
||||
} else {
|
||||
const functionKey = /^f([1-9]|1[0-9]|2[0-4])$/i.exec(rest)
|
||||
if (functionKey === null) {
|
||||
return null
|
||||
}
|
||||
keyCode = 111 + Number(functionKey[1])
|
||||
key = `F${functionKey[1]}`
|
||||
code = key
|
||||
text = null
|
||||
}
|
||||
|
||||
if (text !== null && (modifiers & 8) !== 0) {
|
||||
text = Object.hasOwn(US_SHIFT_OF, text) ? US_SHIFT_OF[text] : text.toUpperCase()
|
||||
// Why: Shift+a is the "A" key as far as the page is concerned.
|
||||
if (rest.length === 1) {
|
||||
key = text
|
||||
}
|
||||
}
|
||||
// Why: with ctrl, alt or meta held the press is a shortcut and produces no character.
|
||||
if ((modifiers & ~8) !== 0) {
|
||||
text = null
|
||||
}
|
||||
|
||||
return { keyCode, key, code, modifiers, location, selfModifier, text }
|
||||
}
|
||||
Reference in New Issue
Block a user