fix(emulator): type and paste into the Android emulator pane (#37139)

The pane's keyboard only sent serve-sim HID frames over the iOS WebSocket.
Android sessions have no serve-sim socket, so every key was dropped and every
paste failed with "device is not ready".

Android panes now send typed characters, paste and editing keys over a new
emulator:scrcpyKeyboard IPC beside emulator:scrcpyTouch, keyed by the streamed
serial. Main writes them straight to the live scrcpy control socket (text
injection, and keycodes with SHIFT_LEFT held for Shift), so keys keep their
order with touches. iOS keeps the HID path.

src/main/emulator/android/android-input-mapping.ts

@@ -26,8 +26,21 @@ export function normalizedToDevicePixels(
   }
 }
 
-// Android KeyEvent KEYCODE_* values, including the common aliases agents use.
+// Android KeyEvent KEYCODE_* values, including the common aliases agents use, plus the
+// editing keys the emulator pane's keyboard sends.
 const BUTTON_KEYCODES: Record<string, number> = {
+  enter: 66,
+  tab: 61,
+  backspace: 67,
+  forward_delete: 112,
+  dpad_up: 19,
+  dpad_down: 20,
+  dpad_left: 21,
+  dpad_right: 22,
+  move_home: 122,
+  move_end: 123,
+  page_up: 92,
+  page_down: 93,
   home: 3,
   back: 4,
   recents: 187,

src/main/emulator/android/scrcpy-control-input.ts

@@ -14,6 +14,11 @@ import type { EmulatorGesturePoint } from '../emulator-gesture-sender'
 
 export type ScrcpyTouchPhase = EmulatorGesturePoint['type']
 
+// KEYCODE_SHIFT_LEFT and AMETA_SHIFT_ON | AMETA_SHIFT_LEFT_ON. Text fields select only while a
+// SHIFT key is held, so Shift+key presses SHIFT_LEFT around the key like a hardware keyboard.
+const KEYCODE_SHIFT_LEFT = 59
+const META_SHIFT = 0x41
+
 const TOUCH_ACTION: Record<ScrcpyTouchPhase, ScrcpyTouchAction> = {
   begin: 'down',
   move: 'move',
@@ -95,10 +100,22 @@ export function sendScrcpyText(control: ScrcpyControlTarget, text: string): bool
   return splitScrcpyInjectText(text).every((chunk) => control.sendControl(encodeInjectText(chunk)))
 }
 
-export function sendScrcpyButton(control: ScrcpyControlTarget, name: string): boolean {
+export function sendScrcpyButton(
+  control: ScrcpyControlTarget,
+  name: string,
+  modifiers: { shift?: boolean } = {}
+): boolean {
   const keycode = androidButtonKeycode(name)
+  const metaState = modifiers.shift ? META_SHIFT : 0
+  const press = (action: 'down' | 'up', code: number, meta = metaState) =>
+    control.sendControl(encodeInjectKeycode({ action, keycode: code, metaState: meta }))
+  if (!modifiers.shift) {
+    return press('down', keycode) && press('up', keycode)
+  }
   return (
-    control.sendControl(encodeInjectKeycode({ action: 'down', keycode })) &&
-    control.sendControl(encodeInjectKeycode({ action: 'up', keycode }))
+    press('down', KEYCODE_SHIFT_LEFT) &&
+    press('down', keycode) &&
+    press('up', keycode) &&
+    press('up', KEYCODE_SHIFT_LEFT, 0)
   )
 }

src/main/ipc/emulator-scrcpy-keyboard.test.ts

@@ -0,0 +1,73 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const ipc = vi.hoisted(() => ({
+  handlers: new Map<string, (event: unknown, payload: unknown) => unknown>(),
+  fromWebContents: vi.fn((): unknown => ({}))
+}))
+
+vi.mock('electron', () => ({
+  ipcMain: {
+    removeHandler: (channel: string) => ipc.handlers.delete(channel),
+    handle: (channel: string, handler: (event: unknown, payload: unknown) => unknown) => {
+      ipc.handlers.set(channel, handler)
+    }
+  },
+  BrowserWindow: { fromWebContents: ipc.fromWebContents }
+}))
+
+import { scrcpyVideoRegistry } from '../emulator/scrcpy-video-registry'
+import { registerEmulatorScrcpyKeyboardHandler } from './emulator-scrcpy-keyboard'
+
+const sendControl = vi.fn((_message: Buffer) => true)
+
+function sendKeyboard(payload: unknown): unknown {
+  return ipc.handlers.get('emulator:scrcpyKeyboard')!({ sender: {} }, payload)
+}
+
+beforeEach(() => {
+  sendControl.mockClear()
+  ipc.fromWebContents.mockReturnValue({})
+  scrcpyVideoRegistry.register('serial-1', () => {})
+  scrcpyVideoRegistry.attachControl('serial-1', {
+    videoSize: () => ({ width: 600, height: 1280 }),
+    sendControl
+  })
+  registerEmulatorScrcpyKeyboardHandler()
+})
+
+afterEach(() => {
+  scrcpyVideoRegistry.stop('serial-1')
+})
+
+describe('emulator:scrcpyKeyboard', () => {
+  it('injects text, and keycodes with the Shift meta state, into the live session', () => {
+    expect(sendKeyboard({ deviceId: 'serial-1', text: 'hi' })).toBe(true)
+    expect(sendKeyboard({ deviceId: 'serial-1', key: 'dpad_left', shift: true })).toBe(true)
+
+    const [text, ...keys] = sendControl.mock.calls.map(([message]) => message)
+    expect(text.subarray(5).toString('utf8')).toBe('hi')
+    // [action, keycode, metaState]: SHIFT_LEFT is held around DPAD_LEFT.
+    expect(keys.map((key) => [key[1], key.readInt32BE(2), key.readInt32BE(10)])).toEqual([
+      [0, 59, 0x41],
+      [0, 21, 0x41],
+      [1, 21, 0x41],
+      [1, 59, 0]
+    ])
+  })
+
+  it('returns false for malformed input, unknown devices or keys, and non-window senders', () => {
+    for (const payload of [
+      null,
+      { deviceId: 'serial-1' },
+      { deviceId: 7, text: 'hi' },
+      { deviceId: 'serial-2', text: 'hi' },
+      { deviceId: 'serial-1', key: 'not_a_key', shift: false }
+    ]) {
+      expect(sendKeyboard(payload)).toBe(false)
+    }
+    ipc.fromWebContents.mockReturnValue(null)
+    expect(sendKeyboard({ deviceId: 'serial-1', text: 'hi' })).toBe(false)
+
+    expect(sendControl).not.toHaveBeenCalled()
+  })
+})

src/main/ipc/emulator-scrcpy-keyboard.ts

@@ -0,0 +1,48 @@
+import { ipcMain, BrowserWindow } from 'electron'
+import { scrcpyVideoRegistry } from '../emulator/scrcpy-video-registry'
+import { sendScrcpyButton, sendScrcpyText } from '../emulator/android/scrcpy-control-input'
+
+type ScrcpyKeyboardInput =
+  | { deviceId: string; text: string }
+  | { deviceId: string; key: string; shift: boolean }
+
+function parseScrcpyKeyboardInput(value: unknown): ScrcpyKeyboardInput | null {
+  if (typeof value !== 'object' || value === null || !('deviceId' in value)) {
+    return null
+  }
+  const { deviceId } = value
+  if (typeof deviceId !== 'string') {
+    return null
+  }
+  if ('text' in value && typeof value.text === 'string') {
+    return { deviceId, text: value.text }
+  }
+  if ('key' in value && typeof value.key === 'string') {
+    return { deviceId, key: value.key, shift: 'shift' in value && value.shift === true }
+  }
+  return null
+}
+
+// Live Android pane typing and paste go to the same scrcpy control socket as the pane's
+// touches (emulator-scrcpy-touch.ts), so keys and taps reach the device in the order sent.
+export function registerEmulatorScrcpyKeyboardHandler(): void {
+  ipcMain.removeHandler('emulator:scrcpyKeyboard')
+  ipcMain.handle('emulator:scrcpyKeyboard', (event, value: unknown): boolean => {
+    const input = parseScrcpyKeyboardInput(value)
+    if (!BrowserWindow.fromWebContents(event.sender) || !input) {
+      return false
+    }
+    const control = scrcpyVideoRegistry.control(input.deviceId)
+    if (!control) {
+      return false
+    }
+    try {
+      return 'text' in input
+        ? sendScrcpyText(control, input.text)
+        : sendScrcpyButton(control, input.key, { shift: input.shift })
+    } catch {
+      // Unknown key names throw from the keycode table.
+      return false
+    }
+  })
+}

src/main/ipc/register-core-handlers/register-core-handlers.test.ts

@@ -305,6 +305,10 @@ vi.mock('../emulator-scrcpy-touch', () => ({
   registerEmulatorScrcpyTouchHandler: registerEmulatorScrcpyTouchHandlerMock
 }))
 
+vi.mock('../emulator-scrcpy-keyboard', () => ({
+  registerEmulatorScrcpyKeyboardHandler: vi.fn()
+}))
+
 vi.mock('../filesystem/filesystem', () => ({
   registerFilesystemHandlers: registerFilesystemHandlersMock
 }))

src/main/ipc/register-core-handlers/register-core-handlers.ts

@@ -65,6 +65,7 @@ import { registerUIHandlers, setTrustedUIRendererWebContentsId } from '../ui'
 import { registerEmulatorFrameStreamHandlers } from '../emulator-frame-stream'
 import { registerEmulatorVideoStreamHandlers } from '../emulator-video-stream'
 import { registerEmulatorScrcpyTouchHandler } from '../emulator-scrcpy-touch'
+import { registerEmulatorScrcpyKeyboardHandler } from '../emulator-scrcpy-keyboard'
 import { registerSpeechHandlers } from '../speech'
 import { registerTerminalRenderDesyncEvidenceHandler } from '../terminal-render-desync-evidence'
 import { registerOrcaProfileHandlers } from '../orca-profiles'
@@ -252,6 +253,7 @@ export function registerCoreHandlers(
   registerEmulatorFrameStreamHandlers()
   registerEmulatorVideoStreamHandlers()
   registerEmulatorScrcpyTouchHandler()
+  registerEmulatorScrcpyKeyboardHandler()
   registerWorkspaceSpaceHandlers(store)
   registerWorkspacePortHandlers(store)
   registerLinkPreviewHandlers()

src/preload/api/emulator-api.ts

@@ -31,6 +31,10 @@ export type EmulatorApi = {
     x: number
     y: number
   }) => void
+  /** Resolves false when the device has no live scrcpy control socket. */
+  sendScrcpyKeyboard: (
+    input: { deviceId: string; text: string } | { deviceId: string; key: string; shift: boolean }
+  ) => Promise<boolean>
   onVideoStreamFrame: (
     callback: (data: {
       streamId: string

src/preload/api/emulator-bridge.ts

@@ -58,6 +58,9 @@ export const emulatorApi = {
     x: number
     y: number
   }): void => ipcRenderer.send('emulator:scrcpyTouch', touch),
+  sendScrcpyKeyboard: (
+    input: { deviceId: string; text: string } | { deviceId: string; key: string; shift: boolean }
+  ): Promise<boolean> => ipcRenderer.invoke('emulator:scrcpyKeyboard', input),
   onVideoStreamFrame: (
     callback: (data: {
       streamId: string

src/renderer/src/components/emulator-pane/emulator-device-frame.input.test.tsx

@@ -326,6 +326,35 @@ describe('EmulatorDeviceFrame input', () => {
     ])
   })
 
+  it('sends Android keys and paste to the scrcpy control socket in order', async () => {
+    vi.mocked(toast.error).mockClear()
+    const sendScrcpyKeyboard = vi.fn<Window['api']['emulator']['sendScrcpyKeyboard']>(
+      async () => true
+    )
+    vi.stubGlobal('api', { emulator: { sendScrcpyKeyboard } })
+    renderFrame({ previewUrl: 'scrcpy://serial-1', wsUrl: '' })
+    const screen = getScreen()
+    const paste = pasteEvent('hi\r\nyou')
+
+    await act(async () => {
+      screen.dispatchEvent(keyEvent('keydown', { key: 'Enter' }))
+      screen.dispatchEvent(keyEvent('keydown', { key: 'a' }))
+      screen.dispatchEvent(keyEvent('keydown', { key: 'ArrowLeft', shiftKey: true }))
+      screen.dispatchEvent(paste)
+      screen.dispatchEvent(keyEvent('keydown', { key: 'Enter' }))
+    })
+
+    expect(paste.defaultPrevented).toBe(true)
+    expect(sendScrcpyKeyboard.mock.calls.map(([input]) => input)).toEqual([
+      { deviceId: 'serial-1', text: 'a' },
+      { deviceId: 'serial-1', key: 'dpad_left', shift: true },
+      { deviceId: 'serial-1', text: 'hi\nyou' },
+      { deviceId: 'serial-1', key: 'enter', shift: false }
+    ])
+    expect(toast.error).not.toHaveBeenCalled()
+    expect(FakeWebSocket.instances).toEqual([])
+  })
+
   it('rejects oversized screen paste without scheduling keyboard frames', async () => {
     renderFrame()
     act(() => {

src/renderer/src/components/emulator-pane/emulator-device-frame.tsx

@@ -85,8 +85,8 @@ export function EmulatorDeviceFrame({
   const [streamSize, setStreamSize] = useState<StreamSize | null>(null)
   const canInteract = isLive && !loading && !streamError
   const scrcpyDeviceId = scrcpyDeviceIdFromStreamUrl(previewUrl)
-  const { cancelKeyboardFrames, reportedOrientation, sendKeyboardFrames, sendTouch } =
-    useEmulatorControlStream(wsUrl, canInteract, scrcpyDeviceId)
+  const controlStream = useEmulatorControlStream(wsUrl, canInteract, scrcpyDeviceId)
+  const { reportedOrientation, sendTouch } = controlStream
   const visualStreamGeometry = useMemo(
     () => resolveVisualStreamGeometry(streamSize, visualOrientation, reportedOrientation),
     [reportedOrientation, streamSize, visualOrientation]
@@ -94,11 +94,7 @@ export function EmulatorDeviceFrame({
   // serve-sim takes touches in raw panel coordinates, so input undoes the picture's turn.
   const inputRotation = scrcpyDeviceId ? 0 : visualStreamGeometry.streamRotation
   const { enableKeyboardCapture, handleBlur, handleKeyDown, handlePaste, keyboardCaptureActive } =
-    useEmulatorScreenKeyboard({
-      cancelKeyboardFrames,
-      canInteract,
-      sendKeyboardFrames
-    })
+    useEmulatorScreenKeyboard({ ...controlStream, canInteract })
 
   useEffect(() => {
     setStreamError(false)

src/renderer/src/components/emulator-pane/emulator-keyboard-paste.ts

@@ -178,6 +178,26 @@ export async function pasteTextIntoEmulatorKeyboard({
   return { byteLength: validation.byteLength, chunkCount, status: 'sent' }
 }
 
+// Android has no HID socket: main injects the whole text over the scrcpy control socket.
+export async function pasteTextIntoAndroidEmulator({
+  maxBytes,
+  sendText,
+  text
+}: {
+  maxBytes?: number
+  sendText: (text: string) => Promise<boolean>
+  text: string
+}): Promise<EmulatorKeyboardPasteResult> {
+  const validation = validateEmulatorKeyboardPasteText(text, maxBytes)
+  if (validation.status === 'rejected') {
+    return validation
+  }
+  if (!(await sendText(text.replaceAll('\r', '')))) {
+    return { byteLength: validation.byteLength, reason: 'target-unavailable', status: 'rejected' }
+  }
+  return { byteLength: validation.byteLength, chunkCount: 1, status: 'sent' }
+}
+
 function getPositiveIntegerLimit(value: number | undefined, fallback: number): number {
   return Number.isFinite(value) && (value ?? 0) > 0 ? Math.floor(value ?? fallback) : fallback
 }

src/renderer/src/components/emulator-pane/use-emulator-control-stream.ts

@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useRef, useState } from 'react'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import {
   encodeServeSimKeyboardFrame,
   type ServeSimKeyboardFrame
@@ -35,9 +35,16 @@ function decodeServeSimOrientation(data: unknown): EmulatorStreamOrientation | u
   }
 }
 
+export type ScrcpyKeyboard = {
+  sendKey: (name: string, shift: boolean) => void
+  sendText: (text: string) => Promise<boolean>
+}
+
 export type EmulatorControlStream = {
   cancelKeyboardFrames: () => void
   connected: boolean
+  /** Android keys and text for main's scrcpy control socket; null for serve-sim sessions. */
+  scrcpyKeyboard: ScrcpyKeyboard | null
   reportedOrientation: EmulatorStreamOrientation | null
   sendKeyboardFrames: (frames: ServeSimKeyboardFrame[]) => boolean
   sendTouch: (touch: ServeSimTouchFrame) => boolean
@@ -233,7 +240,31 @@ export function useEmulatorControlStream(
     [getOpenSocket, sendKeyboardFrameNow]
   )
 
+  const scrcpyKeyboard = useMemo<ScrcpyKeyboard | null>(
+    () =>
+      scrcpyDeviceId
+        ? {
+            sendKey: (key, shift) =>
+              void window.api.emulator
+                .sendScrcpyKeyboard({ deviceId: scrcpyDeviceId, key, shift })
+                .catch(() => false),
+            sendText: (text) =>
+              window.api.emulator
+                .sendScrcpyKeyboard({ deviceId: scrcpyDeviceId, text })
+                .catch(() => false)
+          }
+        : null,
+    [scrcpyDeviceId]
+  )
+
   const reportedOrientation =
     reportedConfig && reportedConfig.wsUrl === wsUrl ? reportedConfig.orientation : null
-  return { cancelKeyboardFrames, connected, reportedOrientation, sendKeyboardFrames, sendTouch }
+  return {
+    cancelKeyboardFrames,
+    connected,
+    reportedOrientation,
+    scrcpyKeyboard,
+    sendKeyboardFrames,
+    sendTouch
+  }
 }

src/renderer/src/components/emulator-pane/use-emulator-screen-keyboard.ts

@@ -12,20 +12,41 @@ import {
   buildServeSimKeyboardFramesForKey,
   type ServeSimKeyboardFrame
 } from '../../../../shared/emulator-keyboard-frame'
+import type { ScrcpyKeyboard } from './use-emulator-control-stream'
 import {
+  pasteTextIntoAndroidEmulator,
   pasteTextIntoEmulatorKeyboard,
   type EmulatorKeyboardPasteResult
 } from './emulator-keyboard-paste'
 
+// Android keycode names (see android-input-mapping.ts) for the named keys; others type as text.
+const ANDROID_KEY_BUTTONS: Record<string, string> = {
+  Enter: 'enter',
+  Tab: 'tab',
+  Backspace: 'backspace',
+  Delete: 'forward_delete',
+  ArrowUp: 'dpad_up',
+  ArrowDown: 'dpad_down',
+  ArrowLeft: 'dpad_left',
+  ArrowRight: 'dpad_right',
+  Home: 'move_home',
+  End: 'move_end',
+  PageUp: 'page_up',
+  PageDown: 'page_down'
+}
+
 type UseEmulatorScreenKeyboardArgs = {
   cancelKeyboardFrames: () => void
   canInteract: boolean
+  /** Set for Android panes, which have no serve-sim socket for HID frames. */
+  scrcpyKeyboard: ScrcpyKeyboard | null
   sendKeyboardFrames: (frames: ServeSimKeyboardFrame[]) => boolean
 }
 
 export function useEmulatorScreenKeyboard({
   cancelKeyboardFrames,
   canInteract,
+  scrcpyKeyboard,
   sendKeyboardFrames
 }: UseEmulatorScreenKeyboardArgs) {
   const captureActiveRef = useRef(false)
@@ -97,14 +118,25 @@ export function useEmulatorScreenKeyboard({
         return
       }
 
+      // The HID table also decides which keys an Android pane accepts.
       const frames = buildServeSimKeyboardFramesForKey(event.key, { shift: event.shiftKey })
-      if (!frames || !sendKeyboardFrames(frames)) {
+      if (!frames) {
+        return
+      }
+      if (scrcpyKeyboard) {
+        const button = ANDROID_KEY_BUTTONS[event.key]
+        if (button) {
+          scrcpyKeyboard.sendKey(button, event.shiftKey)
+        } else {
+          void scrcpyKeyboard.sendText(event.key)
+        }
+      } else if (!sendKeyboardFrames(frames)) {
         return
       }
       event.preventDefault()
       event.stopPropagation()
     },
-    [canInteract, sendKeyboardFrames, setCaptureActive]
+    [canInteract, scrcpyKeyboard, sendKeyboardFrames, setCaptureActive]
   )
 
   const handlePaste = useCallback(
@@ -121,21 +153,24 @@ export function useEmulatorScreenKeyboard({
 
       cancelActivePaste()
       const pasteRequestId = pasteRequestIdRef.current
-      void pasteTextIntoEmulatorKeyboard({
-        isCancelled: () =>
-          pasteRequestIdRef.current !== pasteRequestId ||
-          !captureActiveRef.current ||
-          !canInteractRef.current,
-        sendKeyboardFrames,
-        text
-      }).then((result) => {
+      const paste = scrcpyKeyboard
+        ? pasteTextIntoAndroidEmulator({ sendText: scrcpyKeyboard.sendText, text })
+        : pasteTextIntoEmulatorKeyboard({
+            isCancelled: () =>
+              pasteRequestIdRef.current !== pasteRequestId ||
+              !captureActiveRef.current ||
+              !canInteractRef.current,
+            sendKeyboardFrames,
+            text
+          })
+      void paste.then((result) => {
         if (pasteRequestIdRef.current !== pasteRequestId && result.status !== 'cancelled') {
           return
         }
         showEmulatorKeyboardPasteResult(result)
       })
     },
-    [canInteract, cancelActivePaste, sendKeyboardFrames]
+    [canInteract, cancelActivePaste, scrcpyKeyboard, sendKeyboardFrames]
   )
 
   return {