fix(browser): give agent set device and viewport --mobile touch input and pointer: coarse (#35982)

* fix(browser): turn on touch when an agent sets a mobile device or viewport

* docs(orca-cli): say every page load resets agent device emulation

skill-guides/orca-cli/references/browser.md

@@ -34,6 +34,8 @@ ORCA wait --url <substring> --json
 ORCA wait --selector <css> --json
 ORCA wait --load networkidle --json
 ORCA eval --expression <js> --json
+ORCA set device --name "iPhone 15" --json
+ORCA viewport --width 390 --height 844 --scale 3 --mobile --json
 ORCA tab list --json
 ORCA tab create --url <url> --json
 ORCA tab switch --index <n> --json
@@ -52,6 +54,7 @@ Browser rules:
 - Browser commands default to the current worktree and its active tab. Use `--worktree all` only intentionally.
 - For concurrent browser work, run `ORCA tab list --json`, read `tabs[].browserPageId`, and pass `--page <browserPageId>` on later commands.
 - Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command "tab ..."`, so Orca keeps UI state synchronized.
+- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command "set device ..."`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.
 - Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts.
 - Anything not listed above goes through `ORCA exec --command "<agent-browser command>"`.
 - If `fill` or `type` fails on a custom input, try `ORCA focus --element @e1 --json` then `ORCA inserttext --text "text" --json`.

src/cli/bundled-skill-guides.ts

@@ -816,6 +816,8 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [
   "ORCA wait --selector <css> --json",
   "ORCA wait --load networkidle --json",
   "ORCA eval --expression <js> --json",
+  "ORCA set device --name \"iPhone 15\" --json",
+  "ORCA viewport --width 390 --height 844 --scale 3 --mobile --json",
   "ORCA tab list --json",
   "ORCA tab create --url <url> --json",
   "ORCA tab switch --index <n> --json",
@@ -834,6 +836,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [
   "- Browser commands default to the current worktree and its active tab. Use `--worktree all` only intentionally.",
   "- For concurrent browser work, run `ORCA tab list --json`, read `tabs[].browserPageId`, and pass `--page <browserPageId>` on later commands.",
   "- Use typed tab commands (`ORCA tab list/create/close/switch`), not `ORCA exec --command \"tab ...\"`, so Orca keeps UI state synchronized.",
+  "- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command \"set device ...\"`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.",
   "- Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts.",
   "- Anything not listed above goes through `ORCA exec --command \"<agent-browser command>\"`.",
   "- If `fill` or `type` fails on a custom input, try `ORCA focus --element @e1 --json` then `ORCA inserttext --text \"text\" --json`.",

src/main/browser/agent-browser-bridge-mouse-commands.ts

@@ -8,6 +8,7 @@ import {
 } from './agent-browser-bridge-mouse'
 import { acquireElectronDebugger } from './electron-debugger-lease'
 import { sendGuestCdpCommand } from './guest-cdp-command'
+import { touchEmulationParams } from './guest-touch-emulation'
 import { assertRemoteCliBrowserAccess } from '../runtime/desktop-browser-bridge/remote-cli-browser-scope'
 import { AgentBrowserBridgePointerCommands } from './agent-browser-bridge-pointer-commands'
 import { agentBrowserSetMediaArguments } from './agent-browser-set-media-arguments'
@@ -111,8 +112,21 @@ export abstract class AgentBrowserBridgeMouseCommands extends AgentBrowserBridge
   }
 
   async setDevice(name: string, worktreeId?: string, browserPageId?: string): Promise<unknown> {
-    return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
-      return await this.execAgentBrowser(sessionName, ['set', 'device', name])
+    return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName, target) => {
+      const result = await this.execAgentBrowser(sessionName, ['set', 'device', name])
+      const wc = this.requireTargetWebContents(target)
+      // agent-browser sets only metrics and user agent; without touch the page still reports pointer: fine.
+      const mobile =
+        typeof result === 'object' &&
+        result !== null &&
+        'mobile' in result &&
+        result.mobile === true
+      await sendGuestCdpCommand(
+        wc,
+        'Emulation.setTouchEmulationEnabled',
+        touchEmulationParams(mobile)
+      )
+      return result
     })
   }
 

src/main/browser/agent-browser-bridge-mouse-input.test.ts

@@ -54,6 +54,7 @@ vi.mock('./cdp-bridge', () => ({
 
 import { AgentBrowserBridge } from './agent-browser-bridge'
 import {
+  createSucceedWith,
   mockBrowserManager,
   mockWebContents,
   overrideBridgeWebContentsLookup,
@@ -198,4 +199,33 @@ describe('AgentBrowserBridge', () => {
     expect(mouseCalls[0]?.[1]).toMatchObject({ type: 'mousePressed', x: 12, y: 34, modifiers: 4 })
     expect(mouseCalls[1]?.[1]).toMatchObject({ type: 'mouseReleased', x: 12, y: 34, modifiers: 4 })
   })
+
+  function touchEmulationCalls(wc: ReturnType<typeof mockWebContents>): unknown[] {
+    return wc.debugger.sendCommand.mock.calls
+      .filter((call) => call[0] === 'Emulation.setTouchEmulationEnabled')
+      .map((call) => call[1])
+  }
+
+  it('turns on touch for a mobile device, which agent-browser leaves off', async () => {
+    const wc = mockWebContents(100)
+    webContentsFromIdMock.mockReturnValue(wc)
+    createSucceedWith(execFileMock, stdinWrites)({ device: 'iPhone 15', width: 393, mobile: true })
+
+    await bridge.setDevice('iPhone 15', undefined, 'tab-1')
+
+    expect(touchEmulationCalls(wc)).toEqual([{ enabled: true, maxTouchPoints: 5 }])
+  })
+
+  it('matches touch to the mobile flag of a viewport', async () => {
+    const wc = mockWebContents(100)
+    webContentsFromIdMock.mockReturnValue(wc)
+
+    await bridge.setViewport(390, 844, 3, true, undefined, 'tab-1')
+    await bridge.setViewport(1280, 800, 1, false, undefined, 'tab-1')
+
+    expect(touchEmulationCalls(wc)).toEqual([
+      { enabled: true, maxTouchPoints: 5 },
+      { enabled: false }
+    ])
+  })
 })

src/main/browser/agent-browser-bridge-state-commands.ts

@@ -22,6 +22,7 @@ import {
   type AgentBrowserConsoleOutput,
   type AgentBrowserNetworkOutput
 } from './agent-browser-log-entries'
+import { touchEmulationParams } from './guest-touch-emulation'
 
 export abstract class AgentBrowserBridgeStateCommands extends AgentBrowserBridgeInteractionCommands {
   async cookieGet(
@@ -110,6 +111,11 @@ export abstract class AgentBrowserBridgeStateCommands extends AgentBrowserBridge
         deviceScaleFactor: scale,
         mobile
       })
+      await sendGuestCdpCommand(
+        wc,
+        'Emulation.setTouchEmulationEnabled',
+        touchEmulationParams(mobile)
+      )
       // Metrics-only resize can leave the compositor at its old size and crop remote screencasts.
       await sendGuestCdpCommand(wc, 'Emulation.setVisibleSize', { width, height }).catch(() => {})
 

src/main/browser/browser-manager-viewport.ts

@@ -7,11 +7,9 @@ import {
 import type { BrowserViewportOverride } from '../../shared/browser-workspace-types'
 import { BrowserManagerDownloadLifecycle } from './browser-manager-download-lifecycle'
 import { sendGuestCdpCommand } from './guest-cdp-command'
+import { touchEmulationParams } from './guest-touch-emulation'
 import type { BrowserDevToolsDockBounds } from '../../shared/browser-devtools-dock'
 
-// Omit maxTouchPoints when disabling: Chromium rejects 0 even then (#22749).
-const TOUCH_EMULATION_DISABLED = { enabled: false } as const
-
 export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifecycle {
   async openDevTools(
     browserTabId: string,
@@ -259,7 +257,7 @@ export abstract class BrowserManagerViewport extends BrowserManagerDownloadLifec
     const touchApplied = await this.runViewportEmulationStep(browserTabId, 'touch emulation', () =>
       dbg.sendCommand(
         'Emulation.setTouchEmulationEnabled',
-        override?.mobile ? { enabled: true, maxTouchPoints: 5 } : TOUCH_EMULATION_DISABLED
+        touchEmulationParams(override?.mobile === true)
       )
     )
     if (this.webContentsIdByTabId.get(browserTabId) !== webContentsId) {

src/main/browser/guest-touch-emulation.ts

@@ -0,0 +1,7 @@
+// Omit maxTouchPoints when disabling: Chromium rejects 0 even then (#22749).
+const TOUCH_EMULATION_DISABLED = { enabled: false } as const
+const TOUCH_EMULATION_MOBILE = { enabled: true, maxTouchPoints: 5 } as const
+
+export function touchEmulationParams(mobile: boolean) {
+  return mobile ? TOUCH_EMULATION_MOBILE : TOUCH_EMULATION_DISABLED
+}