From 92f943141697202853bb9aafaad4d130e5ae4eff Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:34:03 +0000 Subject: [PATCH] fix(browser): give agent set device and viewport --mobile touch input and pointer: coarse (#35982) * fix(browser): turn on touch when an agent sets a mobile device or viewport * docs(orca-cli): say every page load resets agent device emulation --- skill-guides/orca-cli/references/browser.md | 3 ++ src/cli/bundled-skill-guides.ts | 3 ++ .../agent-browser-bridge-mouse-commands.ts | 18 +++++++++-- .../agent-browser-bridge-mouse-input.test.ts | 30 +++++++++++++++++++ .../agent-browser-bridge-state-commands.ts | 6 ++++ src/main/browser/browser-manager-viewport.ts | 6 ++-- src/main/browser/guest-touch-emulation.ts | 7 +++++ 7 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 src/main/browser/guest-touch-emulation.ts diff --git a/skill-guides/orca-cli/references/browser.md b/skill-guides/orca-cli/references/browser.md index ea5db962ed6..b07f91ea782 100644 --- a/skill-guides/orca-cli/references/browser.md +++ b/skill-guides/orca-cli/references/browser.md @@ -34,6 +34,8 @@ ORCA wait --url --json ORCA wait --selector --json ORCA wait --load networkidle --json ORCA eval --expression --json +ORCA set device --name "iPhone 15" --json +ORCA viewport --width 390 --height 844 --scale 3 --mobile --json ORCA tab list --json ORCA tab create --url --json ORCA tab switch --index --json @@ -52,6 +54,7 @@ Browser rules: - Browser commands default to the current worktree and its active tab. Use `--worktree all` only intentionally. - For concurrent browser work, run `ORCA tab list --json`, read `tabs[].browserPageId`, and pass `--page ` on later commands. - Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command "tab ..."`, so Orca keeps UI state synchronized. +- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command "set device ..."`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating. - Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts. - Anything not listed above goes through `ORCA exec --command ""`. - If `fill` or `type` fails on a custom input, try `ORCA focus --element @e1 --json` then `ORCA inserttext --text "text" --json`. diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index cb0e9344181..19b6372201f 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -816,6 +816,8 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [ "ORCA wait --selector --json", "ORCA wait --load networkidle --json", "ORCA eval --expression --json", + "ORCA set device --name \"iPhone 15\" --json", + "ORCA viewport --width 390 --height 844 --scale 3 --mobile --json", "ORCA tab list --json", "ORCA tab create --url --json", "ORCA tab switch --index --json", @@ -834,6 +836,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [ "- Browser commands default to the current worktree and its active tab. Use `--worktree all` only intentionally.", "- For concurrent browser work, run `ORCA tab list --json`, read `tabs[].browserPageId`, and pass `--page ` on later commands.", "- Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command \"tab ...\"`, so Orca keeps UI state synchronized.", + "- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command \"set device ...\"`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.", "- Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts.", "- Anything not listed above goes through `ORCA exec --command \"\"`.", "- If `fill` or `type` fails on a custom input, try `ORCA focus --element @e1 --json` then `ORCA inserttext --text \"text\" --json`.", diff --git a/src/main/browser/agent-browser-bridge-mouse-commands.ts b/src/main/browser/agent-browser-bridge-mouse-commands.ts index 772fbf95d24..c631eb18c21 100644 --- a/src/main/browser/agent-browser-bridge-mouse-commands.ts +++ b/src/main/browser/agent-browser-bridge-mouse-commands.ts @@ -8,6 +8,7 @@ import { } from './agent-browser-bridge-mouse' import { acquireElectronDebugger } from './electron-debugger-lease' import { sendGuestCdpCommand } from './guest-cdp-command' +import { touchEmulationParams } from './guest-touch-emulation' import { assertRemoteCliBrowserAccess } from '../runtime/desktop-browser-bridge/remote-cli-browser-scope' import { AgentBrowserBridgePointerCommands } from './agent-browser-bridge-pointer-commands' import { agentBrowserSetMediaArguments } from './agent-browser-set-media-arguments' @@ -111,8 +112,21 @@ export abstract class AgentBrowserBridgeMouseCommands extends AgentBrowserBridge } async setDevice(name: string, worktreeId?: string, browserPageId?: string): Promise { - return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => { - return await this.execAgentBrowser(sessionName, ['set', 'device', name]) + return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName, target) => { + const result = await this.execAgentBrowser(sessionName, ['set', 'device', name]) + const wc = this.requireTargetWebContents(target) + // agent-browser sets only metrics and user agent; without touch the page still reports pointer: fine. + const mobile = + typeof result === 'object' && + result !== null && + 'mobile' in result && + result.mobile === true + await sendGuestCdpCommand( + wc, + 'Emulation.setTouchEmulationEnabled', + touchEmulationParams(mobile) + ) + return result }) } diff --git a/src/main/browser/agent-browser-bridge-mouse-input.test.ts b/src/main/browser/agent-browser-bridge-mouse-input.test.ts index 9b3ebbc735b..cc2295b0b70 100644 --- a/src/main/browser/agent-browser-bridge-mouse-input.test.ts +++ b/src/main/browser/agent-browser-bridge-mouse-input.test.ts @@ -54,6 +54,7 @@ vi.mock('./cdp-bridge', () => ({ import { AgentBrowserBridge } from './agent-browser-bridge' import { + createSucceedWith, mockBrowserManager, mockWebContents, overrideBridgeWebContentsLookup, @@ -198,4 +199,33 @@ describe('AgentBrowserBridge', () => { expect(mouseCalls[0]?.[1]).toMatchObject({ type: 'mousePressed', x: 12, y: 34, modifiers: 4 }) expect(mouseCalls[1]?.[1]).toMatchObject({ type: 'mouseReleased', x: 12, y: 34, modifiers: 4 }) }) + + function touchEmulationCalls(wc: ReturnType): unknown[] { + return wc.debugger.sendCommand.mock.calls + .filter((call) => call[0] === 'Emulation.setTouchEmulationEnabled') + .map((call) => call[1]) + } + + it('turns on touch for a mobile device, which agent-browser leaves off', async () => { + const wc = mockWebContents(100) + webContentsFromIdMock.mockReturnValue(wc) + createSucceedWith(execFileMock, stdinWrites)({ device: 'iPhone 15', width: 393, mobile: true }) + + await bridge.setDevice('iPhone 15', undefined, 'tab-1') + + expect(touchEmulationCalls(wc)).toEqual([{ enabled: true, maxTouchPoints: 5 }]) + }) + + it('matches touch to the mobile flag of a viewport', async () => { + const wc = mockWebContents(100) + webContentsFromIdMock.mockReturnValue(wc) + + await bridge.setViewport(390, 844, 3, true, undefined, 'tab-1') + await bridge.setViewport(1280, 800, 1, false, undefined, 'tab-1') + + expect(touchEmulationCalls(wc)).toEqual([ + { enabled: true, maxTouchPoints: 5 }, + { enabled: false } + ]) + }) }) diff --git a/src/main/browser/agent-browser-bridge-state-commands.ts b/src/main/browser/agent-browser-bridge-state-commands.ts index 0469303b3bc..01e12337f17 100644 --- a/src/main/browser/agent-browser-bridge-state-commands.ts +++ b/src/main/browser/agent-browser-bridge-state-commands.ts @@ -22,6 +22,7 @@ import { type AgentBrowserConsoleOutput, type AgentBrowserNetworkOutput } from './agent-browser-log-entries' +import { touchEmulationParams } from './guest-touch-emulation' export abstract class AgentBrowserBridgeStateCommands extends AgentBrowserBridgeInteractionCommands { async cookieGet( @@ -110,6 +111,11 @@ export abstract class AgentBrowserBridgeStateCommands extends AgentBrowserBridge deviceScaleFactor: scale, mobile }) + await sendGuestCdpCommand( + wc, + 'Emulation.setTouchEmulationEnabled', + touchEmulationParams(mobile) + ) // Metrics-only resize can leave the compositor at its old size and crop remote screencasts. await sendGuestCdpCommand(wc, 'Emulation.setVisibleSize', { width, height }).catch(() => {}) diff --git a/src/main/browser/browser-manager-viewport.ts b/src/main/browser/browser-manager-viewport.ts index b349963e2a7..3b1a3e8f534 100644 --- a/src/main/browser/browser-manager-viewport.ts +++ b/src/main/browser/browser-manager-viewport.ts @@ -7,11 +7,9 @@ import { import type { BrowserViewportOverride } from '../../shared/browser-workspace-types' import { BrowserManagerDownloadLifecycle } from './browser-manager-download-lifecycle' import { sendGuestCdpCommand } from './guest-cdp-command' +import { touchEmulationParams } from './guest-touch-emulation' import type { BrowserDevToolsDockBounds } from '../../shared/browser-devtools-dock' -// Omit maxTouchPoints when disabling: Chromium rejects 0 even then (#22749). -const TOUCH_EMULATION_DISABLED = { enabled: false } as const - export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifecycle { async openDevTools( browserTabId: string, @@ -259,7 +257,7 @@ export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifec const touchApplied = await this.runViewportEmulationStep(browserTabId, 'touch emulation', () => dbg.sendCommand( 'Emulation.setTouchEmulationEnabled', - override?.mobile ? { enabled: true, maxTouchPoints: 5 } : TOUCH_EMULATION_DISABLED + touchEmulationParams(override?.mobile === true) ) ) if (this.webContentsIdByTabId.get(browserTabId) !== webContentsId) { diff --git a/src/main/browser/guest-touch-emulation.ts b/src/main/browser/guest-touch-emulation.ts new file mode 100644 index 00000000000..ee89b6c2543 --- /dev/null +++ b/src/main/browser/guest-touch-emulation.ts @@ -0,0 +1,7 @@ +// Omit maxTouchPoints when disabling: Chromium rejects 0 even then (#22749). +const TOUCH_EMULATION_DISABLED = { enabled: false } as const +const TOUCH_EMULATION_MOBILE = { enabled: true, maxTouchPoints: 5 } as const + +export function touchEmulationParams(mobile: boolean) { + return mobile ? TOUCH_EMULATION_MOBILE : TOUCH_EMULATION_DISABLED +}