From d3c77f3e1f2779adda5d11ded19793da807d1f82 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Thu, 11 Jun 2026 14:57:53 -0700 Subject: [PATCH] Add mobile emulator screenshot export Co-authored-by: Orca --- src/main/ipc/emulator-screenshot.test.ts | 119 ++++++++++++++++++ src/main/ipc/emulator-screenshot.ts | 71 +++++++++++ src/main/ipc/register-core-handlers.test.ts | 12 +- src/main/ipc/register-core-handlers.ts | 2 + src/preload/api-types.ts | 4 + src/preload/index.ts | 5 + .../components/emulator-pane/EmulatorPane.tsx | 57 +++++++++ .../emulator-pane/emulator-device-frame.tsx | 9 +- .../emulator-pane/emulator-pane-toolbar.tsx | 32 ++++- .../emulator-screen-stream-content.test.tsx | 10 +- .../emulator-screen-stream-content.tsx | 6 + .../use-emulator-frame-stream.ts | 13 +- src/renderer/src/i18n/locales/en.json | 9 +- src/renderer/src/i18n/locales/es.json | 9 +- src/renderer/src/i18n/locales/ja.json | 9 +- src/renderer/src/i18n/locales/ko.json | 9 +- src/renderer/src/i18n/locales/zh.json | 9 +- src/renderer/src/web/web-preload-api.ts | 1 + 18 files changed, 359 insertions(+), 27 deletions(-) create mode 100644 src/main/ipc/emulator-screenshot.test.ts create mode 100644 src/main/ipc/emulator-screenshot.ts diff --git a/src/main/ipc/emulator-screenshot.test.ts b/src/main/ipc/emulator-screenshot.test.ts new file mode 100644 index 00000000000..0733048ddfb --- /dev/null +++ b/src/main/ipc/emulator-screenshot.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + createFromBufferMock, + fromWebContentsMock, + handleMock, + showSaveDialogMock, + toPngMock, + writeFileMock +} = vi.hoisted(() => ({ + createFromBufferMock: vi.fn(), + fromWebContentsMock: vi.fn(), + handleMock: vi.fn(), + showSaveDialogMock: vi.fn(), + toPngMock: vi.fn(), + writeFileMock: vi.fn() +})) + +vi.mock('electron', () => ({ + BrowserWindow: { + fromWebContents: fromWebContentsMock + }, + dialog: { + showSaveDialog: showSaveDialogMock + }, + ipcMain: { + handle: handleMock + }, + nativeImage: { + createFromBuffer: createFromBufferMock + } +})) + +vi.mock('node:fs/promises', () => ({ + writeFile: writeFileMock +})) + +import { + buildEmulatorScreenshotDefaultPath, + encodeEmulatorScreenshotPng, + registerEmulatorScreenshotHandlers +} from './emulator-screenshot' + +describe('emulator screenshot IPC', () => { + beforeEach(() => { + createFromBufferMock.mockReset() + fromWebContentsMock.mockReset() + handleMock.mockReset() + showSaveDialogMock.mockReset() + toPngMock.mockReset() + writeFileMock.mockReset() + createFromBufferMock.mockReturnValue({ + isEmpty: () => false, + toPNG: toPngMock + }) + toPngMock.mockReturnValue(Buffer.from('png-bytes')) + }) + + it('builds a sanitized PNG default filename', () => { + expect( + buildEmulatorScreenshotDefaultPath( + 'iPhone 16 Pro / Debug', + new Date('2026-06-11T20:31:04.123Z') + ) + ).toBe('orca-iPhone-16-Pro-Debug-2026-06-11-20-31-04.png') + }) + + it('encodes the frame as PNG', () => { + const input = new Uint8Array([1, 2, 3]).buffer + + expect(encodeEmulatorScreenshotPng(input)).toEqual(Buffer.from('png-bytes')) + expect(createFromBufferMock).toHaveBeenCalledWith(Buffer.from([1, 2, 3])) + }) + + it('throws when the frame is not a valid image', () => { + createFromBufferMock.mockReturnValue({ + isEmpty: () => true, + toPNG: toPngMock + }) + + expect(() => encodeEmulatorScreenshotPng(new Uint8Array([1]).buffer)).toThrow( + 'Emulator screenshot frame is not a valid image.' + ) + }) + + it('writes the selected screenshot file', async () => { + const parentWindow = { id: 1 } + fromWebContentsMock.mockReturnValue(parentWindow) + showSaveDialogMock.mockResolvedValue({ canceled: false, filePath: '/tmp/sim.png' }) + registerEmulatorScreenshotHandlers() + const handler = handleMock.mock.calls.find( + ([channel]) => channel === 'emulator:saveScreenshot' + )?.[1] + + await expect( + handler({ sender: { id: 7 } }, { bytes: new Uint8Array([4, 5]).buffer, deviceName: 'Phone' }) + ).resolves.toEqual({ canceled: false, destinationPath: '/tmp/sim.png' }) + + expect(showSaveDialogMock).toHaveBeenCalledWith(parentWindow, { + defaultPath: expect.stringMatching(/^orca-Phone-/), + filters: [{ name: 'PNG Image', extensions: ['png'] }] + }) + expect(writeFileMock).toHaveBeenCalledWith('/tmp/sim.png', Buffer.from('png-bytes')) + }) + + it('does not write a file when the save dialog is canceled', async () => { + showSaveDialogMock.mockResolvedValue({ canceled: true }) + registerEmulatorScreenshotHandlers() + const handler = handleMock.mock.calls.find( + ([channel]) => channel === 'emulator:saveScreenshot' + )?.[1] + + await expect( + handler({ sender: {} }, { bytes: new Uint8Array([4, 5]).buffer, deviceName: 'Phone' }) + ).resolves.toEqual({ canceled: true }) + + expect(writeFileMock).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/emulator-screenshot.ts b/src/main/ipc/emulator-screenshot.ts new file mode 100644 index 00000000000..94861d40429 --- /dev/null +++ b/src/main/ipc/emulator-screenshot.ts @@ -0,0 +1,71 @@ +import { BrowserWindow, dialog, ipcMain, nativeImage } from 'electron' +import { writeFile } from 'node:fs/promises' + +export type SaveEmulatorScreenshotResult = + | { canceled: true } + | { canceled: false; destinationPath: string } + +type SaveEmulatorScreenshotArgs = { + bytes: ArrayBuffer + deviceName?: string +} + +function sanitizeFilenameSegment(value: string): string { + const sanitized = value + .trim() + .replace(/[^a-z0-9._-]+/gi, '-') + .replace(/^-+|-+$/g, '') + return sanitized || 'mobile-emulator' +} + +function timestampForFilename(date = new Date()): string { + return date + .toISOString() + .replace(/\.\d{3}Z$/, '') + .replace(/[:T]/g, '-') +} + +export function buildEmulatorScreenshotDefaultPath(deviceName?: string, date = new Date()): string { + const device = sanitizeFilenameSegment(deviceName || 'mobile-emulator') + return `orca-${device}-${timestampForFilename(date)}.png` +} + +function bufferFromArrayBuffer(bytes: ArrayBuffer): Buffer { + return Buffer.from(new Uint8Array(bytes)) +} + +export function encodeEmulatorScreenshotPng(bytes: ArrayBuffer): Buffer { + const image = nativeImage.createFromBuffer(bufferFromArrayBuffer(bytes)) + if (image.isEmpty()) { + throw new Error('Emulator screenshot frame is not a valid image.') + } + return image.toPNG() +} + +export function registerEmulatorScreenshotHandlers(): void { + ipcMain.handle( + 'emulator:saveScreenshot', + async (event, args: SaveEmulatorScreenshotArgs): Promise => { + if (!(args?.bytes instanceof ArrayBuffer) || args.bytes.byteLength === 0) { + throw new Error('Emulator screenshot frame is empty.') + } + + const pngBuffer = encodeEmulatorScreenshotPng(args.bytes) + const defaultPath = buildEmulatorScreenshotDefaultPath(args.deviceName) + const parentWindow = BrowserWindow.fromWebContents(event.sender) ?? undefined + const dialogOptions = { + defaultPath, + filters: [{ name: 'PNG Image', extensions: ['png'] }] + } + const result = parentWindow + ? await dialog.showSaveDialog(parentWindow, dialogOptions) + : await dialog.showSaveDialog(dialogOptions) + if (result.canceled || !result.filePath) { + return { canceled: true } + } + + await writeFile(result.filePath, pngBuffer) + return { canceled: false, destinationPath: result.filePath } + } + ) +} diff --git a/src/main/ipc/register-core-handlers.test.ts b/src/main/ipc/register-core-handlers.test.ts index 7c55815643f..28de918d8f4 100644 --- a/src/main/ipc/register-core-handlers.test.ts +++ b/src/main/ipc/register-core-handlers.test.ts @@ -50,7 +50,8 @@ const { registerSkillsHandlersMock, registerWorkspaceSpaceHandlersMock, registerWorkspacePortHandlersMock, - registerEmulatorFrameStreamHandlersMock + registerEmulatorFrameStreamHandlersMock, + registerEmulatorScreenshotHandlersMock } = vi.hoisted(() => ({ registerCliHandlersMock: vi.fn(), registerPreflightHandlersMock: vi.fn(), @@ -99,7 +100,8 @@ const { registerSkillsHandlersMock: vi.fn(), registerWorkspaceSpaceHandlersMock: vi.fn(), registerWorkspacePortHandlersMock: vi.fn(), - registerEmulatorFrameStreamHandlersMock: vi.fn() + registerEmulatorFrameStreamHandlersMock: vi.fn(), + registerEmulatorScreenshotHandlersMock: vi.fn() })) vi.mock('./onboarding', () => ({ @@ -214,6 +216,10 @@ vi.mock('./emulator-frame-stream', () => ({ registerEmulatorFrameStreamHandlers: registerEmulatorFrameStreamHandlersMock })) +vi.mock('./emulator-screenshot', () => ({ + registerEmulatorScreenshotHandlers: registerEmulatorScreenshotHandlersMock +})) + vi.mock('./filesystem', () => ({ registerFilesystemHandlers: registerFilesystemHandlersMock })) @@ -339,6 +345,7 @@ describe('registerCoreHandlers', () => { registerWorkspaceSpaceHandlersMock.mockReset() registerWorkspacePortHandlersMock.mockReset() registerEmulatorFrameStreamHandlersMock.mockReset() + registerEmulatorScreenshotHandlersMock.mockReset() }) it('passes the store through to handler registrars that need it', () => { @@ -403,6 +410,7 @@ describe('registerCoreHandlers', () => { expect(registerSessionHandlersMock).toHaveBeenCalledWith(store) expect(registerUIHandlersMock).toHaveBeenCalledWith(store) expect(registerEmulatorFrameStreamHandlersMock).toHaveBeenCalled() + expect(registerEmulatorScreenshotHandlersMock).toHaveBeenCalled() expect(registerFilesystemHandlersMock).toHaveBeenCalledWith(store) expect(registerRuntimeHandlersMock).toHaveBeenCalledWith(runtime) expect(registerRuntimeEnvironmentHandlersMock).toHaveBeenCalled() diff --git a/src/main/ipc/register-core-handlers.ts b/src/main/ipc/register-core-handlers.ts index 9b607c40c31..96ed23d6cac 100644 --- a/src/main/ipc/register-core-handlers.ts +++ b/src/main/ipc/register-core-handlers.ts @@ -44,6 +44,7 @@ import { registerShellHandlers } from './shell' import { registerPetHandlers } from './pet' import { registerUIHandlers } from './ui' import { registerEmulatorFrameStreamHandlers } from './emulator-frame-stream' +import { registerEmulatorScreenshotHandlers } from './emulator-screenshot' import { registerSpeechHandlers } from './speech' import { registerCodexAccountHandlers } from './codex-accounts' import { registerAgentHookHandlers } from './agent-hooks' @@ -147,6 +148,7 @@ export function registerCoreHandlers( registerSessionHandlers(store) registerUIHandlers(store) registerEmulatorFrameStreamHandlers() + registerEmulatorScreenshotHandlers() registerWorkspaceSpaceHandlers(store) registerWorkspacePortHandlers(store) if (commitMessageAgentEnv) { diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 549a266b227..aaafd1e25a2 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -443,6 +443,10 @@ export type EmulatorApi = { streamId: string }> stopFrameStream: (args: { streamId: string }) => Promise + saveScreenshot: (args: { + bytes: ArrayBuffer + deviceName?: string + }) => Promise<{ canceled: true } | { canceled: false; destinationPath: string }> onFrameStreamFrame: ( callback: (data: { streamId: string; bytes: ArrayBuffer }) => void ) => () => void diff --git a/src/preload/index.ts b/src/preload/index.ts index 48a05a4bd92..220b192cce1 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -2133,6 +2133,11 @@ const api = { }> => ipcRenderer.invoke('emulator:frameStreamStart', args), stopFrameStream: (args: { streamId: string }): Promise => ipcRenderer.invoke('emulator:frameStreamStop', args), + saveScreenshot: (args: { + bytes: ArrayBuffer + deviceName?: string + }): Promise<{ canceled: true } | { canceled: false; destinationPath: string }> => + ipcRenderer.invoke('emulator:saveScreenshot', args), onFrameStreamFrame: ( callback: (data: { streamId: string; bytes: ArrayBuffer }) => void ): (() => void) => { diff --git a/src/renderer/src/components/emulator-pane/EmulatorPane.tsx b/src/renderer/src/components/emulator-pane/EmulatorPane.tsx index 1b3499e827c..35ac631ef10 100644 --- a/src/renderer/src/components/emulator-pane/EmulatorPane.tsx +++ b/src/renderer/src/components/emulator-pane/EmulatorPane.tsx @@ -1,10 +1,13 @@ import type { Tab } from '../../../../shared/types' +import { useCallback, useState } from 'react' +import { toast } from 'sonner' import { isMacOs } from './emulator-pane-types' import { EmulatorUnavailablePane } from './emulator-unavailable-pane' import { EmulatorPaneToolbar } from './emulator-pane-toolbar' import { EmulatorDeviceFrame } from './emulator-device-frame' import { useEmulatorPaneSession } from './use-emulator-pane-session' import { translate } from '@/i18n/i18n' +import { extractIpcErrorMessage } from '@/lib/ipc-error' type EmulatorPaneProps = { tab?: Tab @@ -22,6 +25,7 @@ export default function EmulatorPane({ tab, worktreeId, isActive = true }: Emula } function EmulatorPaneContent({ tab, worktreeId, isActive = true }: EmulatorPaneProps) { + const [latestFrameBytes, setLatestFrameBytes] = useState(null) const { devices, selectedUdid, @@ -45,6 +49,56 @@ function EmulatorPaneContent({ tab, worktreeId, isActive = true }: EmulatorPaneP autoAttachOnMount: isActive }) + const handleFrameBytes = useCallback((bytes: ArrayBuffer | null) => { + setLatestFrameBytes(bytes) + }, []) + + const handleSaveScreenshot = useCallback(async () => { + if (!latestFrameBytes) { + toast.error( + translate( + 'auto.components.emulator.pane.EmulatorPane.5d5fd13391', + 'No emulator frame is available yet.' + ) + ) + return + } + try { + const result = await window.api.emulator.saveScreenshot({ + bytes: latestFrameBytes, + deviceName: displayName + }) + // Why: native save-dialog cancellation is an intentional user choice, + // not a failed screenshot action. + if (result.canceled) { + return + } + toast.success( + translate('auto.components.emulator.pane.EmulatorPane.360a8a4e68', 'Screenshot saved'), + { + action: result.destinationPath + ? { + label: translate('auto.components.emulator.pane.EmulatorPane.1a3df04ae1', 'Open'), + onClick: () => { + void window.api.shell.openPath(result.destinationPath as string) + } + } + : undefined + } + ) + } catch (error) { + toast.error( + extractIpcErrorMessage( + error, + translate( + 'auto.components.emulator.pane.EmulatorPane.bef0bb890f', + 'Failed to save screenshot.' + ) + ) + ) + } + }, [displayName, latestFrameBytes]) + return (
void attach(selectedUdid ?? undefined)} + onSaveScreenshot={() => void handleSaveScreenshot()} onShutdown={() => void shutdown(selectedUdid ?? undefined)} onHome={() => void sendButton('home')} onRotate={() => void sendRotate()} + canSaveScreenshot={isLive && Boolean(latestFrameBytes)} /> {error ? ( @@ -85,6 +141,7 @@ function EmulatorPaneContent({ tab, worktreeId, isActive = true }: EmulatorPaneP deviceName={displayName} loading={loading} isLive={isLive} + onFrameBytes={handleFrameBytes} onTap={(x, y) => void sendTap(x, y)} onGesture={(points) => void sendGesture(points)} /> diff --git a/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx b/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx index fcf50fca621..bfd66634aeb 100644 --- a/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx +++ b/src/renderer/src/components/emulator-pane/emulator-device-frame.tsx @@ -39,6 +39,7 @@ type EmulatorDeviceFrameProps = { deviceName?: string loading: boolean isLive: boolean + onFrameBytes?: (bytes: ArrayBuffer | null) => void onTap: (x: number, y: number) => void onGesture: (points: EmulatorGesturePoint[]) => void } @@ -53,11 +54,7 @@ type PendingWheelGesture = { timerId: number | null } -type ScreenCoordinateEvent = { - clientX: number - clientY: number - currentTarget: HTMLDivElement -} +type ScreenCoordinateEvent = PointerEvent export function EmulatorDeviceFrame({ previewUrl, @@ -66,6 +63,7 @@ export function EmulatorDeviceFrame({ deviceName, loading, isLive, + onFrameBytes, onTap, onGesture }: EmulatorDeviceFrameProps) { @@ -404,6 +402,7 @@ export function EmulatorDeviceFrame({ chrome doubles up with iOS's real status bar and makes bezels lie. */} void onAttach: () => void + onSaveScreenshot: () => void onShutdown: () => void onHome: () => void onRotate: () => void + canSaveScreenshot: boolean } export function EmulatorPaneToolbar({ @@ -33,9 +35,11 @@ export function EmulatorPaneToolbar({ selectedUdid, onSelectDevice, onAttach, + onSaveScreenshot, onShutdown, onHome, - onRotate + onRotate, + canSaveScreenshot }: EmulatorPaneToolbarProps) { // Why: the toolbar chip describes Orca's preview/control stream, not the // lower-level CoreSimulator boot state. @@ -80,6 +84,30 @@ export function EmulatorPaneToolbar({ ))} + + + + + + {translate( + 'auto.components.emulator.pane.emulator.pane.toolbar.1e5dfdd399', + 'Save screenshot' + )} + +