From be3d9b7e863e50cea5711ea9c93dff265ed151fb Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:21:15 +0000 Subject: [PATCH] fix(emulator): type and paste into the Android emulator pane (#37139) The pane's keyboard only sent serve-sim HID frames over the iOS WebSocket. Android sessions have no serve-sim socket, so every key was dropped and every paste failed with "device is not ready". Android panes now send typed characters, paste and editing keys over a new emulator:scrcpyKeyboard IPC beside emulator:scrcpyTouch, keyed by the streamed serial. Main writes them straight to the live scrcpy control socket (text injection, and keycodes with SHIFT_LEFT held for Shift), so keys keep their order with touches. iOS keeps the HID path. --- .../emulator/android/android-input-mapping.ts | 15 +++- .../emulator/android/scrcpy-control-input.ts | 23 +++++- src/main/ipc/emulator-scrcpy-keyboard.test.ts | 73 +++++++++++++++++++ src/main/ipc/emulator-scrcpy-keyboard.ts | 48 ++++++++++++ .../register-core-handlers.test.ts | 4 + .../register-core-handlers.ts | 2 + src/preload/api/emulator-api.ts | 4 + src/preload/api/emulator-bridge.ts | 3 + .../emulator-device-frame.input.test.tsx | 29 ++++++++ .../emulator-pane/emulator-device-frame.tsx | 10 +-- .../emulator-pane/emulator-keyboard-paste.ts | 20 +++++ .../use-emulator-control-stream.ts | 35 ++++++++- .../use-emulator-screen-keyboard.ts | 57 ++++++++++++--- 13 files changed, 299 insertions(+), 24 deletions(-) create mode 100644 src/main/ipc/emulator-scrcpy-keyboard.test.ts create mode 100644 src/main/ipc/emulator-scrcpy-keyboard.ts diff --git a/src/main/emulator/android/android-input-mapping.ts b/src/main/emulator/android/android-input-mapping.ts index f2bc7e9687f..a528fe83171 100644 --- a/src/main/emulator/android/android-input-mapping.ts +++ b/src/main/emulator/android/android-input-mapping.ts @@ -26,8 +26,21 @@ export function normalizedToDevicePixels( } } -// Android KeyEvent KEYCODE_* values, including the common aliases agents use. +// Android KeyEvent KEYCODE_* values, including the common aliases agents use, plus the +// editing keys the emulator pane's keyboard sends. const BUTTON_KEYCODES: Record = { + enter: 66, + tab: 61, + backspace: 67, + forward_delete: 112, + dpad_up: 19, + dpad_down: 20, + dpad_left: 21, + dpad_right: 22, + move_home: 122, + move_end: 123, + page_up: 92, + page_down: 93, home: 3, back: 4, recents: 187, diff --git a/src/main/emulator/android/scrcpy-control-input.ts b/src/main/emulator/android/scrcpy-control-input.ts index 77c9c047e00..884bb6a344d 100644 --- a/src/main/emulator/android/scrcpy-control-input.ts +++ b/src/main/emulator/android/scrcpy-control-input.ts @@ -14,6 +14,11 @@ import type { EmulatorGesturePoint } from '../emulator-gesture-sender' export type ScrcpyTouchPhase = EmulatorGesturePoint['type'] +// KEYCODE_SHIFT_LEFT and AMETA_SHIFT_ON | AMETA_SHIFT_LEFT_ON. Text fields select only while a +// SHIFT key is held, so Shift+key presses SHIFT_LEFT around the key like a hardware keyboard. +const KEYCODE_SHIFT_LEFT = 59 +const META_SHIFT = 0x41 + const TOUCH_ACTION: Record = { begin: 'down', move: 'move', @@ -95,10 +100,22 @@ export function sendScrcpyText(control: ScrcpyControlTarget, text: string): bool return splitScrcpyInjectText(text).every((chunk) => control.sendControl(encodeInjectText(chunk))) } -export function sendScrcpyButton(control: ScrcpyControlTarget, name: string): boolean { +export function sendScrcpyButton( + control: ScrcpyControlTarget, + name: string, + modifiers: { shift?: boolean } = {} +): boolean { const keycode = androidButtonKeycode(name) + const metaState = modifiers.shift ? META_SHIFT : 0 + const press = (action: 'down' | 'up', code: number, meta = metaState) => + control.sendControl(encodeInjectKeycode({ action, keycode: code, metaState: meta })) + if (!modifiers.shift) { + return press('down', keycode) && press('up', keycode) + } return ( - control.sendControl(encodeInjectKeycode({ action: 'down', keycode })) && - control.sendControl(encodeInjectKeycode({ action: 'up', keycode })) + press('down', KEYCODE_SHIFT_LEFT) && + press('down', keycode) && + press('up', keycode) && + press('up', KEYCODE_SHIFT_LEFT, 0) ) } diff --git a/src/main/ipc/emulator-scrcpy-keyboard.test.ts b/src/main/ipc/emulator-scrcpy-keyboard.test.ts new file mode 100644 index 00000000000..b3b18375a05 --- /dev/null +++ b/src/main/ipc/emulator-scrcpy-keyboard.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const ipc = vi.hoisted(() => ({ + handlers: new Map unknown>(), + fromWebContents: vi.fn((): unknown => ({})) +})) + +vi.mock('electron', () => ({ + ipcMain: { + removeHandler: (channel: string) => ipc.handlers.delete(channel), + handle: (channel: string, handler: (event: unknown, payload: unknown) => unknown) => { + ipc.handlers.set(channel, handler) + } + }, + BrowserWindow: { fromWebContents: ipc.fromWebContents } +})) + +import { scrcpyVideoRegistry } from '../emulator/scrcpy-video-registry' +import { registerEmulatorScrcpyKeyboardHandler } from './emulator-scrcpy-keyboard' + +const sendControl = vi.fn((_message: Buffer) => true) + +function sendKeyboard(payload: unknown): unknown { + return ipc.handlers.get('emulator:scrcpyKeyboard')!({ sender: {} }, payload) +} + +beforeEach(() => { + sendControl.mockClear() + ipc.fromWebContents.mockReturnValue({}) + scrcpyVideoRegistry.register('serial-1', () => {}) + scrcpyVideoRegistry.attachControl('serial-1', { + videoSize: () => ({ width: 600, height: 1280 }), + sendControl + }) + registerEmulatorScrcpyKeyboardHandler() +}) + +afterEach(() => { + scrcpyVideoRegistry.stop('serial-1') +}) + +describe('emulator:scrcpyKeyboard', () => { + it('injects text, and keycodes with the Shift meta state, into the live session', () => { + expect(sendKeyboard({ deviceId: 'serial-1', text: 'hi' })).toBe(true) + expect(sendKeyboard({ deviceId: 'serial-1', key: 'dpad_left', shift: true })).toBe(true) + + const [text, ...keys] = sendControl.mock.calls.map(([message]) => message) + expect(text.subarray(5).toString('utf8')).toBe('hi') + // [action, keycode, metaState]: SHIFT_LEFT is held around DPAD_LEFT. + expect(keys.map((key) => [key[1], key.readInt32BE(2), key.readInt32BE(10)])).toEqual([ + [0, 59, 0x41], + [0, 21, 0x41], + [1, 21, 0x41], + [1, 59, 0] + ]) + }) + + it('returns false for malformed input, unknown devices or keys, and non-window senders', () => { + for (const payload of [ + null, + { deviceId: 'serial-1' }, + { deviceId: 7, text: 'hi' }, + { deviceId: 'serial-2', text: 'hi' }, + { deviceId: 'serial-1', key: 'not_a_key', shift: false } + ]) { + expect(sendKeyboard(payload)).toBe(false) + } + ipc.fromWebContents.mockReturnValue(null) + expect(sendKeyboard({ deviceId: 'serial-1', text: 'hi' })).toBe(false) + + expect(sendControl).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/emulator-scrcpy-keyboard.ts b/src/main/ipc/emulator-scrcpy-keyboard.ts new file mode 100644 index 00000000000..384bc3bc23c --- /dev/null +++ b/src/main/ipc/emulator-scrcpy-keyboard.ts @@ -0,0 +1,48 @@ +import { ipcMain, BrowserWindow } from 'electron' +import { scrcpyVideoRegistry } from '../emulator/scrcpy-video-registry' +import { sendScrcpyButton, sendScrcpyText } from '../emulator/android/scrcpy-control-input' + +type ScrcpyKeyboardInput = + | { deviceId: string; text: string } + | { deviceId: string; key: string; shift: boolean } + +function parseScrcpyKeyboardInput(value: unknown): ScrcpyKeyboardInput | null { + if (typeof value !== 'object' || value === null || !('deviceId' in value)) { + return null + } + const { deviceId } = value + if (typeof deviceId !== 'string') { + return null + } + if ('text' in value && typeof value.text === 'string') { + return { deviceId, text: value.text } + } + if ('key' in value && typeof value.key === 'string') { + return { deviceId, key: value.key, shift: 'shift' in value && value.shift === true } + } + return null +} + +// Live Android pane typing and paste go to the same scrcpy control socket as the pane's +// touches (emulator-scrcpy-touch.ts), so keys and taps reach the device in the order sent. +export function registerEmulatorScrcpyKeyboardHandler(): void { + ipcMain.removeHandler('emulator:scrcpyKeyboard') + ipcMain.handle('emulator:scrcpyKeyboard', (event, value: unknown): boolean => { + const input = parseScrcpyKeyboardInput(value) + if (!BrowserWindow.fromWebContents(event.sender) || !input) { + return false + } + const control = scrcpyVideoRegistry.control(input.deviceId) + if (!control) { + return false + } + try { + return 'text' in input + ? sendScrcpyText(control, input.text) + : sendScrcpyButton(control, input.key, { shift: input.shift }) + } catch { + // Unknown key names throw from the keycode table. + return false + } + }) +} diff --git a/src/main/ipc/register-core-handlers/register-core-handlers.test.ts b/src/main/ipc/register-core-handlers/register-core-handlers.test.ts index f500e5a23ec..c48f796b2fa 100644 --- a/src/main/ipc/register-core-handlers/register-core-handlers.test.ts +++ b/src/main/ipc/register-core-handlers/register-core-handlers.test.ts @@ -305,6 +305,10 @@ vi.mock('../emulator-scrcpy-touch', () => ({ registerEmulatorScrcpyTouchHandler: registerEmulatorScrcpyTouchHandlerMock })) +vi.mock('../emulator-scrcpy-keyboard', () => ({ + registerEmulatorScrcpyKeyboardHandler: vi.fn() +})) + vi.mock('../filesystem/filesystem', () => ({ registerFilesystemHandlers: registerFilesystemHandlersMock })) diff --git a/src/main/ipc/register-core-handlers/register-core-handlers.ts b/src/main/ipc/register-core-handlers/register-core-handlers.ts index 5f0e51d593d..ca5f9c16654 100644 --- a/src/main/ipc/register-core-handlers/register-core-handlers.ts +++ b/src/main/ipc/register-core-handlers/register-core-handlers.ts @@ -65,6 +65,7 @@ import { registerUIHandlers, setTrustedUIRendererWebContentsId } from '../ui' import { registerEmulatorFrameStreamHandlers } from '../emulator-frame-stream' import { registerEmulatorVideoStreamHandlers } from '../emulator-video-stream' import { registerEmulatorScrcpyTouchHandler } from '../emulator-scrcpy-touch' +import { registerEmulatorScrcpyKeyboardHandler } from '../emulator-scrcpy-keyboard' import { registerSpeechHandlers } from '../speech' import { registerTerminalRenderDesyncEvidenceHandler } from '../terminal-render-desync-evidence' import { registerOrcaProfileHandlers } from '../orca-profiles' @@ -252,6 +253,7 @@ export function registerCoreHandlers( registerEmulatorFrameStreamHandlers() registerEmulatorVideoStreamHandlers() registerEmulatorScrcpyTouchHandler() + registerEmulatorScrcpyKeyboardHandler() registerWorkspaceSpaceHandlers(store) registerWorkspacePortHandlers(store) registerLinkPreviewHandlers() diff --git a/src/preload/api/emulator-api.ts b/src/preload/api/emulator-api.ts index edf11b90421..1638b4e4420 100644 --- a/src/preload/api/emulator-api.ts +++ b/src/preload/api/emulator-api.ts @@ -31,6 +31,10 @@ export type EmulatorApi = { x: number y: number }) => void + /** Resolves false when the device has no live scrcpy control socket. */ + sendScrcpyKeyboard: ( + input: { deviceId: string; text: string } | { deviceId: string; key: string; shift: boolean } + ) => Promise onVideoStreamFrame: ( callback: (data: { streamId: string diff --git a/src/preload/api/emulator-bridge.ts b/src/preload/api/emulator-bridge.ts index 44bbe2157bd..50fb7c3c166 100644 --- a/src/preload/api/emulator-bridge.ts +++ b/src/preload/api/emulator-bridge.ts @@ -58,6 +58,9 @@ export const emulatorApi = { x: number y: number }): void => ipcRenderer.send('emulator:scrcpyTouch', touch), + sendScrcpyKeyboard: ( + input: { deviceId: string; text: string } | { deviceId: string; key: string; shift: boolean } + ): Promise => ipcRenderer.invoke('emulator:scrcpyKeyboard', input), onVideoStreamFrame: ( callback: (data: { streamId: string diff --git a/src/renderer/src/components/emulator-pane/emulator-device-frame.input.test.tsx b/src/renderer/src/components/emulator-pane/emulator-device-frame.input.test.tsx index 7a14dbf785d..bbd9663685f 100644 --- a/src/renderer/src/components/emulator-pane/emulator-device-frame.input.test.tsx +++ b/src/renderer/src/components/emulator-pane/emulator-device-frame.input.test.tsx @@ -326,6 +326,35 @@ describe('EmulatorDeviceFrame input', () => { ]) }) + it('sends Android keys and paste to the scrcpy control socket in order', async () => { + vi.mocked(toast.error).mockClear() + const sendScrcpyKeyboard = vi.fn( + async () => true + ) + vi.stubGlobal('api', { emulator: { sendScrcpyKeyboard } }) + renderFrame({ previewUrl: 'scrcpy://serial-1', wsUrl: '' }) + const screen = getScreen() + const paste = pasteEvent('hi\r\nyou') + + await act(async () => { + screen.dispatchEvent(keyEvent('keydown', { key: 'Enter' })) + screen.dispatchEvent(keyEvent('keydown', { key: 'a' })) + screen.dispatchEvent(keyEvent('keydown', { key: 'ArrowLeft', shiftKey: true })) + screen.dispatchEvent(paste) + screen.dispatchEvent(keyEvent('keydown', { key: 'Enter' })) + }) + + expect(paste.defaultPrevented).toBe(true) + expect(sendScrcpyKeyboard.mock.calls.map(([input]) => input)).toEqual([ + { deviceId: 'serial-1', text: 'a' }, + { deviceId: 'serial-1', key: 'dpad_left', shift: true }, + { deviceId: 'serial-1', text: 'hi\nyou' }, + { deviceId: 'serial-1', key: 'enter', shift: false } + ]) + expect(toast.error).not.toHaveBeenCalled() + expect(FakeWebSocket.instances).toEqual([]) + }) + it('rejects oversized screen paste without scheduling keyboard frames', async () => { renderFrame() act(() => { diff --git a/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx b/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx index 9fabbe903c1..ec5b5bf3b9b 100644 --- a/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx +++ b/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx @@ -85,8 +85,8 @@ export function EmulatorDeviceFrame({ const [streamSize, setStreamSize] = useState(null) const canInteract = isLive && !loading && !streamError const scrcpyDeviceId = scrcpyDeviceIdFromStreamUrl(previewUrl) - const { cancelKeyboardFrames, reportedOrientation, sendKeyboardFrames, sendTouch } = - useEmulatorControlStream(wsUrl, canInteract, scrcpyDeviceId) + const controlStream = useEmulatorControlStream(wsUrl, canInteract, scrcpyDeviceId) + const { reportedOrientation, sendTouch } = controlStream const visualStreamGeometry = useMemo( () => resolveVisualStreamGeometry(streamSize, visualOrientation, reportedOrientation), [reportedOrientation, streamSize, visualOrientation] @@ -94,11 +94,7 @@ export function EmulatorDeviceFrame({ // serve-sim takes touches in raw panel coordinates, so input undoes the picture's turn. const inputRotation = scrcpyDeviceId ? 0 : visualStreamGeometry.streamRotation const { enableKeyboardCapture, handleBlur, handleKeyDown, handlePaste, keyboardCaptureActive } = - useEmulatorScreenKeyboard({ - cancelKeyboardFrames, - canInteract, - sendKeyboardFrames - }) + useEmulatorScreenKeyboard({ ...controlStream, canInteract }) useEffect(() => { setStreamError(false) diff --git a/src/renderer/src/components/emulator-pane/emulator-keyboard-paste.ts b/src/renderer/src/components/emulator-pane/emulator-keyboard-paste.ts index 061ff3f8f47..7f6fc0f2505 100644 --- a/src/renderer/src/components/emulator-pane/emulator-keyboard-paste.ts +++ b/src/renderer/src/components/emulator-pane/emulator-keyboard-paste.ts @@ -178,6 +178,26 @@ export async function pasteTextIntoEmulatorKeyboard({ return { byteLength: validation.byteLength, chunkCount, status: 'sent' } } +// Android has no HID socket: main injects the whole text over the scrcpy control socket. +export async function pasteTextIntoAndroidEmulator({ + maxBytes, + sendText, + text +}: { + maxBytes?: number + sendText: (text: string) => Promise + text: string +}): Promise { + const validation = validateEmulatorKeyboardPasteText(text, maxBytes) + if (validation.status === 'rejected') { + return validation + } + if (!(await sendText(text.replaceAll('\r', '')))) { + return { byteLength: validation.byteLength, reason: 'target-unavailable', status: 'rejected' } + } + return { byteLength: validation.byteLength, chunkCount: 1, status: 'sent' } +} + function getPositiveIntegerLimit(value: number | undefined, fallback: number): number { return Number.isFinite(value) && (value ?? 0) > 0 ? Math.floor(value ?? fallback) : fallback } diff --git a/src/renderer/src/components/emulator-pane/use-emulator-control-stream.ts b/src/renderer/src/components/emulator-pane/use-emulator-control-stream.ts index acd0aa27303..e2de3401107 100644 --- a/src/renderer/src/components/emulator-pane/use-emulator-control-stream.ts +++ b/src/renderer/src/components/emulator-pane/use-emulator-control-stream.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { encodeServeSimKeyboardFrame, type ServeSimKeyboardFrame @@ -35,9 +35,16 @@ function decodeServeSimOrientation(data: unknown): EmulatorStreamOrientation | u } } +export type ScrcpyKeyboard = { + sendKey: (name: string, shift: boolean) => void + sendText: (text: string) => Promise +} + export type EmulatorControlStream = { cancelKeyboardFrames: () => void connected: boolean + /** Android keys and text for main's scrcpy control socket; null for serve-sim sessions. */ + scrcpyKeyboard: ScrcpyKeyboard | null reportedOrientation: EmulatorStreamOrientation | null sendKeyboardFrames: (frames: ServeSimKeyboardFrame[]) => boolean sendTouch: (touch: ServeSimTouchFrame) => boolean @@ -233,7 +240,31 @@ export function useEmulatorControlStream( [getOpenSocket, sendKeyboardFrameNow] ) + const scrcpyKeyboard = useMemo( + () => + scrcpyDeviceId + ? { + sendKey: (key, shift) => + void window.api.emulator + .sendScrcpyKeyboard({ deviceId: scrcpyDeviceId, key, shift }) + .catch(() => false), + sendText: (text) => + window.api.emulator + .sendScrcpyKeyboard({ deviceId: scrcpyDeviceId, text }) + .catch(() => false) + } + : null, + [scrcpyDeviceId] + ) + const reportedOrientation = reportedConfig && reportedConfig.wsUrl === wsUrl ? reportedConfig.orientation : null - return { cancelKeyboardFrames, connected, reportedOrientation, sendKeyboardFrames, sendTouch } + return { + cancelKeyboardFrames, + connected, + reportedOrientation, + scrcpyKeyboard, + sendKeyboardFrames, + sendTouch + } } diff --git a/src/renderer/src/components/emulator-pane/use-emulator-screen-keyboard.ts b/src/renderer/src/components/emulator-pane/use-emulator-screen-keyboard.ts index 1eac52e1c2a..acba103eca0 100644 --- a/src/renderer/src/components/emulator-pane/use-emulator-screen-keyboard.ts +++ b/src/renderer/src/components/emulator-pane/use-emulator-screen-keyboard.ts @@ -12,20 +12,41 @@ import { buildServeSimKeyboardFramesForKey, type ServeSimKeyboardFrame } from '../../../../shared/emulator-keyboard-frame' +import type { ScrcpyKeyboard } from './use-emulator-control-stream' import { + pasteTextIntoAndroidEmulator, pasteTextIntoEmulatorKeyboard, type EmulatorKeyboardPasteResult } from './emulator-keyboard-paste' +// Android keycode names (see android-input-mapping.ts) for the named keys; others type as text. +const ANDROID_KEY_BUTTONS: Record = { + Enter: 'enter', + Tab: 'tab', + Backspace: 'backspace', + Delete: 'forward_delete', + ArrowUp: 'dpad_up', + ArrowDown: 'dpad_down', + ArrowLeft: 'dpad_left', + ArrowRight: 'dpad_right', + Home: 'move_home', + End: 'move_end', + PageUp: 'page_up', + PageDown: 'page_down' +} + type UseEmulatorScreenKeyboardArgs = { cancelKeyboardFrames: () => void canInteract: boolean + /** Set for Android panes, which have no serve-sim socket for HID frames. */ + scrcpyKeyboard: ScrcpyKeyboard | null sendKeyboardFrames: (frames: ServeSimKeyboardFrame[]) => boolean } export function useEmulatorScreenKeyboard({ cancelKeyboardFrames, canInteract, + scrcpyKeyboard, sendKeyboardFrames }: UseEmulatorScreenKeyboardArgs) { const captureActiveRef = useRef(false) @@ -97,14 +118,25 @@ export function useEmulatorScreenKeyboard({ return } + // The HID table also decides which keys an Android pane accepts. const frames = buildServeSimKeyboardFramesForKey(event.key, { shift: event.shiftKey }) - if (!frames || !sendKeyboardFrames(frames)) { + if (!frames) { + return + } + if (scrcpyKeyboard) { + const button = ANDROID_KEY_BUTTONS[event.key] + if (button) { + scrcpyKeyboard.sendKey(button, event.shiftKey) + } else { + void scrcpyKeyboard.sendText(event.key) + } + } else if (!sendKeyboardFrames(frames)) { return } event.preventDefault() event.stopPropagation() }, - [canInteract, sendKeyboardFrames, setCaptureActive] + [canInteract, scrcpyKeyboard, sendKeyboardFrames, setCaptureActive] ) const handlePaste = useCallback( @@ -121,21 +153,24 @@ export function useEmulatorScreenKeyboard({ cancelActivePaste() const pasteRequestId = pasteRequestIdRef.current - void pasteTextIntoEmulatorKeyboard({ - isCancelled: () => - pasteRequestIdRef.current !== pasteRequestId || - !captureActiveRef.current || - !canInteractRef.current, - sendKeyboardFrames, - text - }).then((result) => { + const paste = scrcpyKeyboard + ? pasteTextIntoAndroidEmulator({ sendText: scrcpyKeyboard.sendText, text }) + : pasteTextIntoEmulatorKeyboard({ + isCancelled: () => + pasteRequestIdRef.current !== pasteRequestId || + !captureActiveRef.current || + !canInteractRef.current, + sendKeyboardFrames, + text + }) + void paste.then((result) => { if (pasteRequestIdRef.current !== pasteRequestId && result.status !== 'cancelled') { return } showEmulatorKeyboardPasteResult(result) }) }, - [canInteract, cancelActivePaste, sendKeyboardFrames] + [canInteract, cancelActivePaste, scrcpyKeyboard, sendKeyboardFrames] ) return {