From f9b5275683520251858096bc705e2ef598446dd0 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 7 Sep 2026 14:55:33 -0400 Subject: [PATCH 1/4] feat(desktop): own browser input, navigation and screencast for the page The page reaches these through the generic host lane, so the desktop is now the only place that scopes the worktree, strips tab URLs, bounds titles and rate limits pointer/keyboard traffic. Binary screencast frames cross the JSON lane as base64 chunks. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- src/main/runtime/rpc/methods/index.ts | 6 + .../mobile-web-browser-command-dispatch.ts | 41 ++++ .../mobile-web-browser-frame-chunks.ts | 35 ++++ .../mobile-web-browser-input-rate-limit.ts | 41 ++++ .../methods/mobile-web-browser-input.test.ts | 155 +++++++++++++++ .../rpc/methods/mobile-web-browser-input.ts | 139 ++++++++++++++ .../mobile-web-browser-navigation.test.ts | 66 +++++++ .../methods/mobile-web-browser-navigation.ts | 58 ++++++ .../methods/mobile-web-browser-page-event.ts | 51 +++++ .../methods/mobile-web-browser-stream.test.ts | 178 ++++++++++++++++++ .../rpc/methods/mobile-web-browser-stream.ts | 113 +++++++++++ .../methods/mobile-web-host-rpc-allowlist.ts | 6 + ...time-rpc-mobile-web-host-allowlist.test.ts | 1 + 13 files changed, 890 insertions(+) create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-command-dispatch.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-frame-chunks.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-input-rate-limit.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-input.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-navigation.test.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-navigation.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-page-event.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts create mode 100644 src/main/runtime/rpc/methods/mobile-web-browser-stream.ts diff --git a/src/main/runtime/rpc/methods/index.ts b/src/main/runtime/rpc/methods/index.ts index 0167692c868..2a39f3df871 100644 --- a/src/main/runtime/rpc/methods/index.ts +++ b/src/main/runtime/rpc/methods/index.ts @@ -8,6 +8,9 @@ import { MOBILE_WEB_SESSION_TERMINAL_CREATION_METHODS } from './mobile-web-sessi import { MOBILE_WEB_NATIVE_CHAT_FILE_METHODS } from './mobile-web-native-chat-files' import { MOBILE_WEB_TERMINAL_ACTION_METHODS } from './mobile-web-terminal-actions' import { MOBILE_WEB_NATIVE_CHAT_STREAM_METHOD } from './mobile-web-native-chat-stream' +import { MOBILE_WEB_BROWSER_INPUT_METHODS } from './mobile-web-browser-input' +import { MOBILE_WEB_BROWSER_NAVIGATION_METHODS } from './mobile-web-browser-navigation' +import { MOBILE_WEB_BROWSER_STREAM_METHODS } from './mobile-web-browser-stream' import { MOBILE_WEB_NATIVE_CHAT_METHODS } from './mobile-web-native-chat' import type { RpcAnyMethod } from '../core' import { STATUS_METHODS } from './status' @@ -126,6 +129,9 @@ export const ALL_RPC_METHODS: readonly RpcAnyMethod[] = [ MOBILE_WEB_SESSION_CAPABILITIES_METHOD, ...MOBILE_WEB_SESSION_QUICK_COMMAND_METHODS, MOBILE_WEB_SESSION_BROWSER_CREATE_METHOD, + ...MOBILE_WEB_BROWSER_INPUT_METHODS, + ...MOBILE_WEB_BROWSER_NAVIGATION_METHODS, + ...MOBILE_WEB_BROWSER_STREAM_METHODS, MOBILE_WEB_NATIVE_CHAT_STREAM_METHOD, ...MOBILE_WEB_PACKAGE_METHODS ] diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-command-dispatch.ts b/src/main/runtime/rpc/methods/mobile-web-browser-command-dispatch.ts new file mode 100644 index 00000000000..12157ac2aa1 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-command-dispatch.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import type { RpcContext } from '../core' +import { BROWSER_CORE_METHODS } from './browser-core' +import { BROWSER_EXTRA_METHODS } from './browser-extras' + +/** The shell rewrites the page's workspace handle into `worktree`; `page` is the host browser page + * id the page already holds from `mobileWeb.session.createBrowser`. */ +export const MobileWebBrowserTarget = z.object({ + worktree: z.string().min(1).max(4096), + page: z.string().min(1).max(512) +}) + +export const MOBILE_WEB_BROWSER_APPLIED = { applied: true } as const + +export const MobileWebBrowserCoordinate = z.number().finite().min(-100_000).max(100_000) + +const commands = new Map( + [...BROWSER_CORE_METHODS, ...BROWSER_EXTRA_METHODS].map((method) => [method.name, method]) +) + +export async function dispatchMobileWebBrowserCommand( + name: string, + fields: Record, + context: RpcContext +): Promise { + const command = commands.get(name) + if (!command) { + throw new Error('method_not_found') + } + if (context.signal?.aborted) { + throw new Error('runtime_unavailable') + } + return command.handler(command.params!.parse(fields), context) +} + +export function mobileWebBrowserTargetFields(params: z.infer): { + worktree: string + page: string +} { + return { worktree: params.worktree, page: params.page } +} diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-frame-chunks.ts b/src/main/runtime/rpc/methods/mobile-web-browser-frame-chunks.ts new file mode 100644 index 00000000000..ff1672292b9 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-frame-chunks.ts @@ -0,0 +1,35 @@ +import type { BrowserScreencastFrame } from '../../../../shared/browser-screencast-protocol' +import { + MobileWebBrowserEventSchema, + MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES, + type MobileWebBrowserFrameChunk +} from '../../../../shared/mobile-web/browser-operation-contract' + +/** The generic host lane carries JSON only, so a binary screencast frame crosses it as a run of + * base64 chunks the page reassembles. Null when the frame cannot be described within the page + * contract's bounds. */ +export function mobileWebBrowserFrameChunks( + frame: BrowserScreencastFrame +): MobileWebBrowserFrameChunk[] | null { + const chunkCount = Math.ceil(frame.image.byteLength / MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES) + const chunks: MobileWebBrowserFrameChunk[] = [] + for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex += 1) { + const start = chunkIndex * MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES + const end = Math.min(frame.image.byteLength, start + MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES) + const parsed = MobileWebBrowserEventSchema.safeParse({ + type: 'frameChunk', + frameSequence: frame.seq, + format: frame.format, + metadata: frame.metadata, + imageBytes: frame.image.byteLength, + chunkIndex, + chunkCount, + data: Buffer.from(frame.image.subarray(start, end)).toString('base64') + }) + if (!parsed.success || parsed.data.type !== 'frameChunk') { + return null + } + chunks.push(parsed.data) + } + return chunks.length > 0 ? chunks : null +} diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-input-rate-limit.ts b/src/main/runtime/rpc/methods/mobile-web-browser-input-rate-limit.ts new file mode 100644 index 00000000000..a9ab26139b8 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-input-rate-limit.ts @@ -0,0 +1,41 @@ +const CAPACITY = 40 +const REFILL_PER_SECOND = 20 +const IDLE_EVICTION_MS = 60_000 + +type Bucket = { tokens: number; updatedAt: number } + +const buckets = new Map() + +/** Pointer and keyboard traffic is per-gesture, and the shell forwards it blind, so the desktop is + * the only place the rate is bounded. One bucket per connection, shared by both. */ +export function takeMobileWebBrowserInputToken( + connectionId: string | undefined, + now = Date.now() +): boolean { + const key = connectionId ?? 'local' + const bucket = buckets.get(key) ?? { tokens: CAPACITY, updatedAt: now } + if (!buckets.has(key)) { + evictIdleBuckets(now) + } + const refill = ((now - bucket.updatedAt) / 1000) * REFILL_PER_SECOND + bucket.tokens = Math.min(CAPACITY, bucket.tokens + Math.max(0, refill)) + bucket.updatedAt = now + buckets.set(key, bucket) + if (bucket.tokens < 1) { + return false + } + bucket.tokens -= 1 + return true +} + +export function resetMobileWebBrowserInputRateLimit(): void { + buckets.clear() +} + +function evictIdleBuckets(now: number): void { + for (const [key, bucket] of buckets) { + if (now - bucket.updatedAt >= IDLE_EVICTION_MS) { + buckets.delete(key) + } + } +} diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts b/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts new file mode 100644 index 00000000000..576568f5285 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts @@ -0,0 +1,155 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { RpcContext } from '../core' +import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist' +import { MOBILE_WEB_BROWSER_INPUT_METHODS } from './mobile-web-browser-input' +import { resetMobileWebBrowserInputRateLimit } from './mobile-web-browser-input-rate-limit' + +const methods = new Map(MOBILE_WEB_BROWSER_INPUT_METHODS.map((method) => [method.name, method])) +const pointer = methods.get('mobileWeb.browser.pointer')! +const keyboard = methods.get('mobileWeb.browser.keyboard')! +const dialog = methods.get('mobileWeb.browser.dialog')! + +const TARGET = { worktree: 'id:workspace', page: 'page-1' } + +function fixture() { + const runtime = { + browserMouseMove: vi.fn().mockResolvedValue({}), + browserMouseWheel: vi.fn().mockResolvedValue({}), + browserMouseClick: vi.fn().mockResolvedValue({}), + browserMouseDown: vi.fn().mockResolvedValue({}), + browserMouseUp: vi.fn().mockResolvedValue({}), + browserKeypress: vi.fn().mockResolvedValue({}), + browserKeyboardInsertText: vi.fn().mockResolvedValue({}), + browserDialogAccept: vi.fn().mockResolvedValue({}), + browserDialogDismiss: vi.fn().mockResolvedValue({}) + } + const context = { runtime, connectionId: 'connection' } as unknown as RpcContext + return { context, runtime } +} + +beforeEach(() => resetMobileWebBrowserInputRateLimit()) + +describe('host-owned browser input', () => { + it('turns one scroll into the host move and wheel pair', async () => { + const f = fixture() + + expect( + await pointer.handler( + { ...TARGET, action: 'scroll', x: 10, y: 20, dx: 0, dy: -40 }, + f.context + ) + ).toEqual({ applied: true }) + + expect(f.runtime.browserMouseMove).toHaveBeenCalledWith({ ...TARGET, x: 10, y: 20 }) + expect(f.runtime.browserMouseWheel).toHaveBeenCalledWith({ ...TARGET, dx: 0, dy: -40 }) + }) + + it('clicks with the requested button, modifiers and radius', async () => { + const f = fixture() + + await pointer.handler( + { ...TARGET, action: 'click', x: 5, y: 6, button: 'right', modifiers: ['shift'], radius: 12 }, + f.context + ) + + expect(f.runtime.browserMouseClick).toHaveBeenCalledWith({ + ...TARGET, + x: 5, + y: 6, + button: 'right', + modifiers: ['shift'], + radius: 12 + }) + expect(f.runtime.browserMouseDown).not.toHaveBeenCalled() + }) + + it('falls back to press and release when a plain click fails', async () => { + const f = fixture() + f.runtime.browserMouseClick.mockRejectedValueOnce(new Error('browser_error')) + + await pointer.handler( + { ...TARGET, action: 'click', x: 5, y: 6, button: 'left', modifiers: [] }, + f.context + ) + + expect(f.runtime.browserMouseMove).toHaveBeenCalledWith({ ...TARGET, x: 5, y: 6 }) + expect(f.runtime.browserMouseDown).toHaveBeenCalledWith({ ...TARGET, button: 'left' }) + expect(f.runtime.browserMouseUp).toHaveBeenCalledWith({ ...TARGET, button: 'left' }) + }) + + it('never synthesizes a press for a modified click that failed', async () => { + const f = fixture() + f.runtime.browserMouseClick.mockRejectedValueOnce(new Error('browser_error')) + + await expect( + pointer.handler( + { ...TARGET, action: 'click', x: 5, y: 6, button: 'left', modifiers: ['cmd'] }, + f.context + ) + ).rejects.toThrow('browser_error') + expect(f.runtime.browserMouseDown).not.toHaveBeenCalled() + }) + + it('routes keyboard actions to the matching host command', async () => { + const f = fixture() + + expect( + await keyboard.handler({ ...TARGET, action: 'insertText', text: 'hello' }, f.context) + ).toEqual({ applied: true }) + await keyboard.handler({ ...TARGET, action: 'keypress', key: 'Enter' }, f.context) + + expect(f.runtime.browserKeyboardInsertText).toHaveBeenCalledWith({ ...TARGET, text: 'hello' }) + expect(f.runtime.browserKeypress).toHaveBeenCalledWith({ ...TARGET, key: 'Enter' }) + }) + + it('routes dialog actions to accept and dismiss', async () => { + const f = fixture() + + await dialog.handler({ ...TARGET, action: 'accept' }, f.context) + await dialog.handler({ ...TARGET, action: 'dismiss' }, f.context) + + expect(f.runtime.browserDialogAccept).toHaveBeenCalledWith(TARGET) + expect(f.runtime.browserDialogDismiss).toHaveBeenCalledWith(TARGET) + }) + + it('accepts only the four keys and two buttons the page may send', () => { + expect(keyboard.params!.safeParse({ ...TARGET, action: 'keypress', key: 'F12' }).success).toBe( + false + ) + expect(keyboard.params!.safeParse({ ...TARGET, action: 'keypress', key: 'Tab' }).success).toBe( + true + ) + expect( + pointer.params!.safeParse({ + ...TARGET, + action: 'click', + x: 1, + y: 1, + button: 'middle', + modifiers: [] + }).success + ).toBe(false) + }) + + it('rate-limits input per connection and lets a second connection through', async () => { + const f = fixture() + const other = { ...f.context, connectionId: 'other' } as RpcContext + const move = { ...TARGET, action: 'scroll', x: 1, y: 1, dx: 0, dy: 1 } as const + + let rejected = 0 + for (let attempt = 0; attempt < 60; attempt += 1) { + await pointer.handler(move, f.context).catch(() => { + rejected += 1 + }) + } + + expect(rejected).toBeGreaterThan(0) + await expect(pointer.handler(move, other)).resolves.toEqual({ applied: true }) + }) + + it('exposes every input method to the page lane', () => { + for (const name of methods.keys()) { + expect(isMobileWebHostRpcMethod(name), name).toBe(true) + } + }) +}) diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-input.ts b/src/main/runtime/rpc/methods/mobile-web-browser-input.ts new file mode 100644 index 00000000000..f91b6dce9fc --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-input.ts @@ -0,0 +1,139 @@ +import { z } from 'zod' +import { defineMethod, type RpcContext } from '../core' +import { + dispatchMobileWebBrowserCommand, + mobileWebBrowserTargetFields, + MOBILE_WEB_BROWSER_APPLIED, + MobileWebBrowserCoordinate, + MobileWebBrowserTarget +} from './mobile-web-browser-command-dispatch' +import { takeMobileWebBrowserInputToken } from './mobile-web-browser-input-rate-limit' + +const PointerParams = z.discriminatedUnion('action', [ + MobileWebBrowserTarget.extend({ + action: z.literal('scroll'), + x: MobileWebBrowserCoordinate, + y: MobileWebBrowserCoordinate, + dx: MobileWebBrowserCoordinate, + dy: MobileWebBrowserCoordinate + }), + MobileWebBrowserTarget.extend({ + action: z.literal('click'), + x: MobileWebBrowserCoordinate, + y: MobileWebBrowserCoordinate, + button: z.enum(['left', 'right']), + modifiers: z.array(z.enum(['cmd', 'ctrl', 'alt', 'shift'])).max(4), + radius: z.number().finite().min(0).max(1000).optional() + }) +]) + +const KeyboardParams = z.discriminatedUnion('action', [ + MobileWebBrowserTarget.extend({ + action: z.literal('insertText'), + text: z + .string() + .min(1) + .max(32 * 1024) + }), + MobileWebBrowserTarget.extend({ + action: z.literal('keypress'), + key: z.enum(['Enter', 'Backspace', 'Tab', 'Escape']) + }) +]) + +function requireInputToken(context: RpcContext): void { + if (!takeMobileWebBrowserInputToken(context.connectionId)) { + throw new Error('rate_limited') + } +} + +export const MOBILE_WEB_BROWSER_INPUT_METHODS = [ + defineMethod({ + name: 'mobileWeb.browser.pointer', + params: PointerParams, + handler: async (params, context) => { + requireInputToken(context) + const target = mobileWebBrowserTargetFields(params) + if (params.action === 'scroll') { + await dispatchMobileWebBrowserCommand( + 'browser.mouseMove', + { ...target, x: params.x, y: params.y }, + context + ) + await dispatchMobileWebBrowserCommand( + 'browser.mouseWheel', + { ...target, dx: params.dx, dy: params.dy }, + context + ) + return MOBILE_WEB_BROWSER_APPLIED + } + try { + await dispatchMobileWebBrowserCommand( + 'browser.mouseClick', + { + ...target, + x: params.x, + y: params.y, + button: params.button, + modifiers: params.modifiers, + ...(params.radius === undefined ? {} : { radius: params.radius }) + }, + context + ) + } catch (error) { + // A modified click has no press/release equivalent, so only a plain click falls back. + if (params.modifiers.length > 0) { + throw error + } + await dispatchMobileWebBrowserCommand( + 'browser.mouseMove', + { ...target, x: params.x, y: params.y }, + context + ) + await dispatchMobileWebBrowserCommand( + 'browser.mouseDown', + { ...target, button: params.button }, + context + ) + await dispatchMobileWebBrowserCommand( + 'browser.mouseUp', + { ...target, button: params.button }, + context + ) + } + return MOBILE_WEB_BROWSER_APPLIED + } + }), + defineMethod({ + name: 'mobileWeb.browser.keyboard', + params: KeyboardParams, + handler: async (params, context) => { + requireInputToken(context) + const target = mobileWebBrowserTargetFields(params) + await (params.action === 'insertText' + ? dispatchMobileWebBrowserCommand( + 'browser.keyboardInsertText', + { ...target, text: params.text }, + context + ) + : dispatchMobileWebBrowserCommand( + 'browser.keypress', + { ...target, key: params.key }, + context + )) + return MOBILE_WEB_BROWSER_APPLIED + } + }), + defineMethod({ + name: 'mobileWeb.browser.dialog', + params: MobileWebBrowserTarget.extend({ action: z.enum(['accept', 'dismiss']) }), + handler: async (params, context) => { + await dispatchMobileWebBrowserCommand( + params.action === 'accept' ? 'browser.dialogAccept' : 'browser.dialogDismiss', + mobileWebBrowserTargetFields(params), + context + ) + return MOBILE_WEB_BROWSER_APPLIED + } + }) +] diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-navigation.test.ts b/src/main/runtime/rpc/methods/mobile-web-browser-navigation.test.ts new file mode 100644 index 00000000000..039c255b0de --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-navigation.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RpcContext } from '../core' +import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist' +import { MOBILE_WEB_BROWSER_NAVIGATION_METHODS } from './mobile-web-browser-navigation' + +const methods = new Map( + MOBILE_WEB_BROWSER_NAVIGATION_METHODS.map((method) => [method.name, method]) +) +const navigate = methods.get('mobileWeb.browser.navigate')! +const history = methods.get('mobileWeb.browser.history')! + +const TARGET = { worktree: 'id:workspace', page: 'page-1' } + +function fixture(landing: unknown) { + const runtime = { + browserGoto: vi.fn().mockResolvedValue(landing), + browserBack: vi.fn().mockResolvedValue({ url: 'https://secret.example/?token=abc' }), + browserForward: vi.fn().mockResolvedValue({ url: 'https://secret.example/' }), + browserReload: vi.fn().mockResolvedValue({ url: 'https://secret.example/' }) + } + return { runtime, context: { runtime } as unknown as RpcContext } +} + +describe('host-owned browser navigation', () => { + it('strips credentials from the URL the page is told it landed on', async () => { + const f = fixture({ url: 'https://app.example/callback?code=secret&view=1', title: 'App' }) + + expect(await navigate.handler({ ...TARGET, url: 'https://app.example/' }, f.context)).toEqual({ + url: 'https://app.example/callback?view=1' + }) + expect(f.runtime.browserGoto).toHaveBeenCalledWith({ ...TARGET, url: 'https://app.example/' }) + }) + + it('answers about:blank when the host reports no usable URL', async () => { + const f = fixture({ title: 'App' }) + + expect(await navigate.handler({ ...TARGET, url: 'https://app.example/' }, f.context)).toEqual({ + url: 'about:blank' + }) + }) + + it('refuses a navigation URL that is not a credential-free http target', () => { + for (const url of ['file:///etc/passwd', 'javascript:alert(1)', 'https://a.example/?token=t']) { + expect(navigate.params!.safeParse({ ...TARGET, url }).success, url).toBe(false) + } + expect(navigate.params!.safeParse({ ...TARGET, url: 'https://a.example/x' }).success).toBe(true) + }) + + it('acknowledges history moves without echoing the host tab URL', async () => { + const f = fixture({ url: 'https://app.example/' }) + + for (const action of ['back', 'forward', 'reload'] as const) { + expect(await history.handler({ ...TARGET, action }, f.context)).toEqual({ applied: true }) + } + + expect(f.runtime.browserBack).toHaveBeenCalledWith(TARGET) + expect(f.runtime.browserForward).toHaveBeenCalledWith(TARGET) + expect(f.runtime.browserReload).toHaveBeenCalledWith(TARGET) + }) + + it('exposes every navigation method to the page lane', () => { + for (const name of methods.keys()) { + expect(isMobileWebHostRpcMethod(name), name).toBe(true) + } + }) +}) diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-navigation.ts b/src/main/runtime/rpc/methods/mobile-web-browser-navigation.ts new file mode 100644 index 00000000000..33b5723ac43 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-navigation.ts @@ -0,0 +1,58 @@ +import { z } from 'zod' +import { defineMethod } from '../core' +import { + isMobileWebPageBrowserNavigationUrl, + mobileWebPageBrowserUrl, + MOBILE_WEB_PAGE_BROWSER_URL_MAX_LENGTH +} from '../../../../shared/mobile-web/browser-url-privacy' +import { + dispatchMobileWebBrowserCommand, + mobileWebBrowserTargetFields, + MOBILE_WEB_BROWSER_APPLIED, + MobileWebBrowserTarget +} from './mobile-web-browser-command-dispatch' + +const HISTORY_COMMANDS = { + back: 'browser.back', + forward: 'browser.forward', + reload: 'browser.reload' +} as const + +export const MOBILE_WEB_BROWSER_NAVIGATION_METHODS = [ + defineMethod({ + name: 'mobileWeb.browser.navigate', + params: MobileWebBrowserTarget.extend({ + url: z + .string() + .min(1) + .max(MOBILE_WEB_PAGE_BROWSER_URL_MAX_LENGTH) + .refine(isMobileWebPageBrowserNavigationUrl, 'Unsupported browser URL') + }), + handler: async (params, context) => { + const result = await dispatchMobileWebBrowserCommand( + 'browser.goto', + { ...mobileWebBrowserTargetFields(params), url: params.url }, + context + ) + // The landing URL can carry credentials the page must never see, so it is stripped here. + return { + url: mobileWebPageBrowserUrl( + typeof result === 'object' && result !== null && 'url' in result ? result.url : undefined + ) + } + } + }), + defineMethod({ + name: 'mobileWeb.browser.history', + params: MobileWebBrowserTarget.extend({ action: z.enum(['back', 'forward', 'reload']) }), + handler: async (params, context) => { + // The host result carries the raw tab URL; the page learns the new location from the stream. + await dispatchMobileWebBrowserCommand( + HISTORY_COMMANDS[params.action], + mobileWebBrowserTargetFields(params), + context + ) + return MOBILE_WEB_BROWSER_APPLIED + } + }) +] diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-page-event.ts b/src/main/runtime/rpc/methods/mobile-web-browser-page-event.ts new file mode 100644 index 00000000000..68aa22073e4 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-page-event.ts @@ -0,0 +1,51 @@ +import { + MobileWebBrowserEventSchema, + type MobileWebBrowserEvent +} from '../../../../shared/mobile-web/browser-operation-contract' +import { mobileWebPageBrowserUrl } from '../../../../shared/mobile-web/browser-url-privacy' + +/** Screencast events carry raw tab URLs, host page ids and driver-supplied text. This is the only + * projection the page sees, so anything not named here never reaches it. */ +export function mobileWebBrowserPageEvent(value: unknown): MobileWebBrowserEvent | null { + if (!isRecord(value)) { + return null + } + if (value.type === 'ready' || value.type === 'navigation') { + const tab = isRecord(value.tab) ? value.tab : {} + return MobileWebBrowserEventSchema.parse({ + type: value.type, + tab: { + url: mobileWebPageBrowserUrl(tab.url), + title: boundedText(tab.title, 240, ''), + canGoBack: tab.canGoBack === true, + canGoForward: tab.canGoForward === true + } + }) + } + if (value.type === 'end' || value.type === 'dialogClosed') { + return { type: value.type } + } + if (value.type === 'dialog') { + return MobileWebBrowserEventSchema.parse({ + type: 'dialog', + dialogType: isDialogType(value.dialogType) ? value.dialogType : 'alert', + message: boundedText(value.message, 8192, 'Browser dialog') + }) + } + if (value.type === 'error') { + return { type: 'error', message: 'Browser stream failed.' } + } + return null +} + +function boundedText(value: unknown, maximum: number, fallback: string): string { + return typeof value === 'string' ? value.slice(0, maximum) : fallback +} + +function isDialogType(value: unknown): value is 'alert' | 'confirm' | 'prompt' | 'beforeunload' { + return value === 'alert' || value === 'confirm' || value === 'prompt' || value === 'beforeunload' +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts b/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts new file mode 100644 index 00000000000..a075693e234 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it, vi } from 'vitest' +import { + BrowserScreencastOpcode, + encodeBrowserScreencastFrame +} from '../../../../shared/browser-screencast-protocol' +import type { RpcContext } from '../core' +import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist' +import { MOBILE_WEB_BROWSER_STREAM_METHODS } from './mobile-web-browser-stream' + +const methods = new Map(MOBILE_WEB_BROWSER_STREAM_METHODS.map((method) => [method.name, method])) +const subscribe = methods.get('mobileWeb.browser.subscribe')! +const unsubscribe = methods.get('mobileWeb.browser.unsubscribe')! + +const REQUEST = { + worktree: 'id:workspace', + page: 'page-1', + format: 'jpeg' as const, + quality: 72, + maxWidth: 800, + maxHeight: 600, + everyNthFrame: 1, + minFrameIntervalMs: 100 +} + +type ScreencastOptions = { + emit: (event: unknown) => void + sendBinary: (bytes: Uint8Array) => boolean | void + signal?: AbortSignal +} + +function fixture() { + const cleanups = new Map void>() + let host!: ScreencastOptions + let release!: () => void + const started = new Promise((resolve) => { + release = resolve + }) + const runtime = { + browserScreencast: vi.fn(async (_params: unknown, options: ScreencastOptions) => { + host = options + release() + await new Promise((resolve) => + options.signal?.addEventListener('abort', () => resolve()) + ) + }), + registerSubscriptionCleanup: vi.fn((key: string, cleanup: () => void) => + cleanups.set(key, cleanup) + ), + cleanupSubscription: vi.fn((key: string) => cleanups.get(key)?.()) + } + const events: unknown[] = [] + const context = { runtime, connectionId: 'connection' } as unknown as RpcContext + const done = subscribe.handler(REQUEST, context, (event) => events.push(event)) + return { + context, + done, + events, + runtime, + started, + getHost: () => host + } +} + +function frameChunkOf(events: unknown[]) { + return events.filter( + (event): event is { type: string; data: string; chunkCount: number; frameSequence: number } => + typeof event === 'object' && + event !== null && + (event as { type?: string }).type === 'frameChunk' + ) +} + +describe('host-owned browser screencast', () => { + it('announces the cancel id before any stream event', async () => { + const f = fixture() + await f.started + + expect(f.events[0]).toMatchObject({ type: 'ready' }) + expect(typeof (f.events[0] as { subscriptionId: string }).subscriptionId).toBe('string') + expect(f.events[0]).not.toHaveProperty('tab') + + f.getHost().signal?.dispatchEvent(new Event('abort')) + await f.done + }) + + it('strips the tab URL and bounds the title before the page sees it', async () => { + const f = fixture() + await f.started + + f.getHost().emit({ + type: 'ready', + subscriptionId: 'browser-screencast:host', + browserPageId: 'private-page', + tab: { + url: 'https://app.example/?session_token=secret&keep=1', + title: 'x'.repeat(400), + canGoBack: true, + canGoForward: false + } + }) + + expect(f.events[1]).toEqual({ + type: 'ready', + tab: { + url: 'https://app.example/?keep=1', + title: 'x'.repeat(240), + canGoBack: true, + canGoForward: false + } + }) + expect(JSON.stringify(f.events)).not.toContain('private-page') + f.getHost().signal?.dispatchEvent(new Event('abort')) + await f.done + }) + + it('carries a binary frame across the JSON lane as base64 chunks', async () => { + const f = fixture() + await f.started + + f.getHost().sendBinary( + encodeBrowserScreencastFrame({ + opcode: BrowserScreencastOpcode.Frame, + seq: 7, + format: 'jpeg', + metadata: { imageWidth: 400, imageHeight: 300 }, + image: new Uint8Array([1, 2, 3, 4]) + }) + ) + + const chunks = frameChunkOf(f.events) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toMatchObject({ frameSequence: 7, chunkCount: 1, chunkIndex: 0 }) + expect(Buffer.from(chunks[0]!.data, 'base64')).toEqual(Buffer.from([1, 2, 3, 4])) + f.getHost().signal?.dispatchEvent(new Event('abort')) + await f.done + }) + + it('reports an undisplayable frame instead of dropping the stream', async () => { + const f = fixture() + await f.started + + f.getHost().sendBinary( + encodeBrowserScreencastFrame({ + opcode: BrowserScreencastOpcode.Frame, + seq: 1, + format: 'jpeg', + // Beyond the page contract's dimension bound. + metadata: { imageWidth: 99_999 }, + image: new Uint8Array([9]) + }) + ) + + expect(f.events.at(-1)).toEqual({ + type: 'error', + message: 'Browser frame is too large to display safely.' + }) + f.getHost().signal?.dispatchEvent(new Event('abort')) + await f.done + }) + + it('ends the page stream when the host cleanup for this connection runs', async () => { + const f = fixture() + await f.started + const key = f.runtime.registerSubscriptionCleanup.mock.calls[0]![0] as string + const subscriptionId = (f.events[0] as { subscriptionId: string }).subscriptionId + + expect(key).toBe(`mobileWeb.browser:connection:${subscriptionId}`) + await unsubscribe.handler({ subscriptionId }, f.context) + await f.done + + expect(f.events.at(-1)).toEqual({ type: 'end' }) + }) + + it('exposes the stream and its cancel to the page lane', () => { + expect(isMobileWebHostRpcMethod('mobileWeb.browser.subscribe')).toBe(true) + expect(isMobileWebHostRpcMethod('mobileWeb.browser.unsubscribe')).toBe(true) + }) +}) diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts b/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts new file mode 100644 index 00000000000..194ded6bc12 --- /dev/null +++ b/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts @@ -0,0 +1,113 @@ +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { decodeBrowserScreencastFrame } from '../../../../shared/browser-screencast-protocol' +import type { MobileWebBrowserEvent } from '../../../../shared/mobile-web/browser-operation-contract' +import { defineMethod, defineStreamingMethod, isStreamingMethod } from '../core' +import { BROWSER_SCREENCAST_METHODS } from './browser-screencast' +import { MobileWebBrowserTarget } from './mobile-web-browser-command-dispatch' +import { mobileWebBrowserFrameChunks } from './mobile-web-browser-frame-chunks' +import { mobileWebBrowserPageEvent } from './mobile-web-browser-page-event' + +const source = BROWSER_SCREENCAST_METHODS.find((method) => method.name === 'browser.screencast') +if (!source || !isStreamingMethod(source)) { + throw new Error('Missing browser screencast stream') +} +const stream = source + +function subscriptionKey(connectionId: string | undefined, subscriptionId: string): string { + return `mobileWeb.browser:${connectionId ?? 'local'}:${subscriptionId}` +} + +export const MOBILE_WEB_BROWSER_STREAM_METHODS = [ + defineStreamingMethod({ + name: 'mobileWeb.browser.subscribe', + params: MobileWebBrowserTarget.extend({ + format: z.enum(['jpeg', 'png']), + quality: z.number().int().min(1).max(100), + maxWidth: z.number().int().min(1).max(2400), + maxHeight: z.number().int().min(1).max(2160), + viewportWidth: z.number().int().min(1).max(10_000).optional(), + viewportHeight: z.number().int().min(1).max(10_000).optional(), + deviceScaleFactor: z.number().finite().min(0.1).max(10).optional(), + mobile: z.boolean().optional(), + everyNthFrame: z.number().int().min(1).max(60), + minFrameIntervalMs: z.number().int().min(16).max(10_000) + }), + handler: async (params, context, emit) => { + const subscriptionId = randomUUID() + const key = subscriptionKey(context.connectionId, subscriptionId) + const inner = new AbortController() + let closed = false + const close = (event?: MobileWebBrowserEvent): void => { + if (closed) { + return + } + closed = true + if (event) { + emit(event) + } + inner.abort() + } + const end = (): void => close({ type: 'end' }) + context.runtime.registerSubscriptionCleanup(key, end, context.connectionId) + context.signal?.addEventListener('abort', end, { once: true }) + if (context.signal?.aborted) { + end() + return + } + // The shell learns the cancel id from this frame, so it precedes anything the stream sends. + emit({ type: 'ready', subscriptionId }) + const deliverFrame = (bytes: Uint8Array): boolean => { + if (closed) { + return true + } + const frame = decodeBrowserScreencastFrame(bytes) + if (!frame) { + return true + } + const chunks = mobileWebBrowserFrameChunks(frame) + if (!chunks) { + emit({ type: 'error', message: 'Browser frame is too large to display safely.' }) + return true + } + for (const chunk of chunks) { + emit(chunk) + } + return true + } + try { + await stream.handler( + stream.params!.parse(params), + { ...context, signal: inner.signal, sendBinary: deliverFrame }, + (event) => { + if (closed) { + return + } + const projected = mobileWebBrowserPageEvent(event) + if (!projected) { + return + } + if (projected.type === 'end' || projected.type === 'error') { + close(projected) + return + } + emit(projected) + } + ) + } finally { + context.signal?.removeEventListener('abort', end) + context.runtime.cleanupSubscription(key) + } + } + }), + defineMethod({ + name: 'mobileWeb.browser.unsubscribe', + params: z.object({ subscriptionId: z.string().uuid() }), + handler: (params, context) => { + context.runtime.cleanupSubscription( + subscriptionKey(context.connectionId, params.subscriptionId) + ) + return { unsubscribed: true } + } + }) +] diff --git a/src/main/runtime/rpc/methods/mobile-web-host-rpc-allowlist.ts b/src/main/runtime/rpc/methods/mobile-web-host-rpc-allowlist.ts index 73535dccf59..74b42435126 100644 --- a/src/main/runtime/rpc/methods/mobile-web-host-rpc-allowlist.ts +++ b/src/main/runtime/rpc/methods/mobile-web-host-rpc-allowlist.ts @@ -21,6 +21,12 @@ export const MOBILE_WEB_HOST_RPC_METHODS = new Set([ 'mobileWeb.session.activate', 'mobileWeb.session.close', 'mobileWeb.session.createBrowser', + 'mobileWeb.browser.subscribe', + 'mobileWeb.browser.navigate', + 'mobileWeb.browser.history', + 'mobileWeb.browser.pointer', + 'mobileWeb.browser.keyboard', + 'mobileWeb.browser.dialog', 'mobileWeb.session.quickCommands', 'mobileWeb.session.quickCommandMutate', 'mobileWeb.session.createQuickCommand', diff --git a/src/main/runtime/runtime-rpc-mobile-web-host-allowlist.test.ts b/src/main/runtime/runtime-rpc-mobile-web-host-allowlist.test.ts index f13baa3c1c1..45431dad8e8 100644 --- a/src/main/runtime/runtime-rpc-mobile-web-host-allowlist.test.ts +++ b/src/main/runtime/runtime-rpc-mobile-web-host-allowlist.test.ts @@ -53,6 +53,7 @@ it('admits every allowlisted method and cancel through authenticated mobile disp } expect(MOBILE_WEB_HOST_RPC_CANCEL_METHODS).toEqual( new Set([ + 'mobileWeb.browser.unsubscribe', 'mobileWeb.files.unwatch', 'mobileWeb.nativeChat.unsubscribe', 'mobileWeb.session.unsubscribe' From f6d32ea7cf068bfe1783765900ca3360a104e384 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 7 Sep 2026 15:07:41 -0400 Subject: [PATCH 2/4] refactor(mobile): move browser input and the screencast onto the host lane The page addresses the browser page id directly through requestMobileWebHost and parses each result with the shared contract, so no browser translator remains between it and the desktop. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- ...mobile-web-browser-event-sanitizer.test.ts | 27 --- .../mobile-web-browser-event-sanitizer.ts | 49 ----- .../mobile-web-browser-operations.test.ts | 108 ---------- .../mobile-web-browser-operations.ts | 160 --------------- .../mobile-web-browser-streams.test.ts | 126 ------------ .../mobile-web/mobile-web-browser-streams.ts | 189 ------------------ .../mobile-web-production-browser-grants.ts | 12 -- .../methods/mobile-web-browser-input.test.ts | 2 +- .../methods/mobile-web-browser-stream.test.ts | 9 +- .../src/mobile-web-bridge-client.ts | 3 +- .../mobile-web-bridge-subscription-client.ts | 5 - .../mobile-web-bridge-subscription-setup.ts | 21 -- .../mobile-web-browser-request-client.test.ts | 137 +++++++++++++ .../src/mobile-web-browser-request-client.ts | 105 +++++----- ...bile-web-host-browser-subscription.test.ts | 100 +++++++++ .../mobile-web-host-browser-subscription.ts | 70 +++++++ .../mobile-web/browser-operation-contract.ts | 13 +- 17 files changed, 382 insertions(+), 754 deletions(-) delete mode 100644 mobile/src/mobile-web/mobile-web-browser-event-sanitizer.test.ts delete mode 100644 mobile/src/mobile-web/mobile-web-browser-event-sanitizer.ts delete mode 100644 mobile/src/mobile-web/mobile-web-browser-operations.test.ts delete mode 100644 mobile/src/mobile-web/mobile-web-browser-operations.ts delete mode 100644 mobile/src/mobile-web/mobile-web-browser-streams.test.ts delete mode 100644 mobile/src/mobile-web/mobile-web-browser-streams.ts delete mode 100644 mobile/src/mobile-web/mobile-web-production-browser-grants.ts create mode 100644 src/mobile-web/src/mobile-web-browser-request-client.test.ts create mode 100644 src/mobile-web/src/mobile-web-host-browser-subscription.test.ts create mode 100644 src/mobile-web/src/mobile-web-host-browser-subscription.ts diff --git a/mobile/src/mobile-web/mobile-web-browser-event-sanitizer.test.ts b/mobile/src/mobile-web/mobile-web-browser-event-sanitizer.test.ts deleted file mode 100644 index fa450d6b05b..00000000000 --- a/mobile/src/mobile-web/mobile-web-browser-event-sanitizer.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { sanitizeMobileWebBrowserEvent } from './mobile-web-browser-event-sanitizer' - -describe('sanitizeMobileWebBrowserEvent', () => { - it('removes URL credentials from browser events', () => { - const event = sanitizeMobileWebBrowserEvent({ - type: 'navigation', - tab: { - url: 'https://user:password@example.com/callback?access_token=secret&tab=review', - title: 'Review', - canGoBack: true, - canGoForward: false - } - }) - - expect(event).toEqual({ - type: 'navigation', - tab: { - url: 'https://example.com/callback?tab=review', - title: 'Review', - canGoBack: true, - canGoForward: false - } - }) - expect(JSON.stringify(event)).not.toMatch(/password|access_token|secret/) - }) -}) diff --git a/mobile/src/mobile-web/mobile-web-browser-event-sanitizer.ts b/mobile/src/mobile-web/mobile-web-browser-event-sanitizer.ts deleted file mode 100644 index 6e7bb7eeff4..00000000000 --- a/mobile/src/mobile-web/mobile-web-browser-event-sanitizer.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - MobileWebBrowserEventSchema, - type MobileWebBrowserEvent -} from '../../../src/shared/mobile-web/browser-operation-contract' -import { mobileWebPageBrowserUrl } from '../../../src/shared/mobile-web/browser-url-privacy' - -export function sanitizeMobileWebBrowserEvent(value: unknown): MobileWebBrowserEvent | null { - if (!isRecord(value)) { - return null - } - if (value.type === 'ready' || value.type === 'navigation') { - const tab = isRecord(value.tab) ? value.tab : {} - return MobileWebBrowserEventSchema.parse({ - type: value.type, - tab: { - url: mobileWebPageBrowserUrl(tab.url), - title: boundedText(tab.title, 240, ''), - canGoBack: tab.canGoBack === true, - canGoForward: tab.canGoForward === true - } - }) - } - if (value.type === 'end' || value.type === 'dialogClosed') { - return { type: value.type } - } - if (value.type === 'dialog') { - return MobileWebBrowserEventSchema.parse({ - type: 'dialog', - dialogType: isDialogType(value.dialogType) ? value.dialogType : 'alert', - message: boundedText(value.message, 8192, 'Browser dialog') - }) - } - if (value.type === 'error') { - return { type: 'error', message: 'Browser stream failed.' } - } - return null -} - -function boundedText(value: unknown, maximum: number, fallback: string): string { - return typeof value === 'string' ? value.slice(0, maximum) : fallback -} - -function isDialogType(value: unknown): value is 'alert' | 'confirm' | 'prompt' | 'beforeunload' { - return value === 'alert' || value === 'confirm' || value === 'prompt' || value === 'beforeunload' -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} diff --git a/mobile/src/mobile-web/mobile-web-browser-operations.test.ts b/mobile/src/mobile-web/mobile-web-browser-operations.test.ts deleted file mode 100644 index ca169f0d701..00000000000 --- a/mobile/src/mobile-web/mobile-web-browser-operations.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import type { RpcClient } from '../transport/rpc-client' -import { executeMobileWebBrowserOperation } from './mobile-web-browser-operations' -import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' - -describe('mobile web browser operations', () => { - it('resolves opaque workspace authority and forwards the host page id for navigation', async () => { - const { workspaceAuthority, workspaceId, pageId } = authorities() - const sendRequest = vi.fn().mockResolvedValue({ - ok: true, - result: { url: 'https://example.com/', title: 'Example', rawPageId: 'raw-page' } - }) - - await expect( - executeMobileWebBrowserOperation({ - operation: 'navigate', - payload: { workspaceId, pageId, url: 'https://example.com' }, - client: { sendRequest } as unknown as RpcClient, - workspaceAuthority - }) - ).resolves.toEqual({ url: 'https://example.com/' }) - - expect(sendRequest).toHaveBeenCalledWith( - 'browser.goto', - { - worktree: 'id:host-workspace', - page: 'raw-page', - url: 'https://example.com' - }, - { timeoutMs: 30_000 } - ) - }) - - it('removes credentials from the authoritative navigation result', async () => { - const { workspaceAuthority, workspaceId, pageId } = authorities() - const sendRequest = vi.fn().mockResolvedValue({ - ok: true, - result: { - url: 'https://user:password@example.com/callback?code=secret&tab=review#access_token=secret' - } - }) - - await expect( - executeMobileWebBrowserOperation({ - operation: 'navigate', - payload: { workspaceId, pageId, url: 'https://example.com' }, - client: { sendRequest } as unknown as RpcClient, - workspaceAuthority - }) - ).resolves.toEqual({ url: 'https://example.com/callback?tab=review' }) - }) - - it('keeps pointer fallback native and refuses an unknown page workspace', async () => { - const { workspaceAuthority, workspaceId, pageId } = authorities() - const sendRequest = vi - .fn() - .mockResolvedValueOnce({ ok: false, error: { code: 'unsupported', message: 'unsupported' } }) - .mockResolvedValue({ ok: true, result: null }) - const client = { sendRequest } as unknown as RpcClient - - await executeMobileWebBrowserOperation({ - operation: 'pointer', - payload: { - workspaceId, - pageId, - action: 'click', - x: 20, - y: 30, - button: 'left', - modifiers: [] - }, - client, - workspaceAuthority - }) - - expect(sendRequest.mock.calls.map(([method]) => method)).toEqual([ - 'browser.mouseClick', - 'browser.mouseMove', - 'browser.mouseDown', - 'browser.mouseUp' - ]) - - workspaceAuthority.synchronize([]) - await expect( - executeMobileWebBrowserOperation({ - operation: 'reload', - payload: { workspaceId, pageId }, - client, - workspaceAuthority - }) - ).rejects.toMatchObject({ code: 'not_found' }) - }) -}) - -function authorities(): { - workspaceAuthority: MobileWebWorkspaceAuthority - workspaceId: string - pageId: string -} { - const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(3) - const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes) - workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }]) - return { - workspaceAuthority, - workspaceId: workspaceAuthority.pageWorkspaceId('host-workspace'), - pageId: 'raw-page' - } -} diff --git a/mobile/src/mobile-web/mobile-web-browser-operations.ts b/mobile/src/mobile-web/mobile-web-browser-operations.ts deleted file mode 100644 index a3123b74a90..00000000000 --- a/mobile/src/mobile-web/mobile-web-browser-operations.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { - MobileWebBrowserCommandResultSchema, - MobileWebBrowserDialogPayloadSchema, - MobileWebBrowserKeyboardPayloadSchema, - MobileWebBrowserNavigatePayloadSchema, - MobileWebBrowserNavigateResultSchema, - MobileWebBrowserPointerPayloadSchema, - MobileWebBrowserTargetPayloadSchema -} from '../../../src/shared/mobile-web/browser-operation-contract' -import { mobileWebPageBrowserUrl } from '../../../src/shared/mobile-web/browser-url-privacy' -import type { RpcClient } from '../transport/rpc-client' -import { MobileWebBrokerError, mobileWebBrokerHostRpcError } from './mobile-web-broker-error' -import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' - -export async function executeMobileWebBrowserOperation(args: { - operation: string - payload: unknown - client: RpcClient - workspaceAuthority: MobileWebWorkspaceAuthority -}): Promise { - if (args.operation === 'navigate') { - const payload = MobileWebBrowserNavigatePayloadSchema.parse(args.payload) - const target = resolveTarget(payload, args.workspaceAuthority) - const response = await args.client.sendRequest( - 'browser.goto', - { ...target, url: payload.url }, - { timeoutMs: 30_000 } - ) - const result = requireResult(response) - const parsed = MobileWebBrowserNavigateResultSchema.safeParse({ - url: isRecord(result) ? mobileWebPageBrowserUrl(result.url) : 'about:blank' - }) - if (!parsed.success) { - throw new MobileWebBrokerError('host_error') - } - return parsed.data - } - if (args.operation === 'pointer') { - const payload = MobileWebBrowserPointerPayloadSchema.parse(args.payload) - const target = resolveTarget(payload, args.workspaceAuthority) - if (payload.action === 'scroll') { - await requireRequest(args.client, 'browser.mouseMove', { - ...target, - x: payload.x, - y: payload.y - }) - assertTarget(payload, target, args.workspaceAuthority) - await requireRequest(args.client, 'browser.mouseWheel', { - ...target, - dx: payload.dx, - dy: payload.dy - }) - return MobileWebBrowserCommandResultSchema.parse(null) - } - const click = await args.client.sendRequest( - 'browser.mouseClick', - { - ...target, - x: payload.x, - y: payload.y, - button: payload.button, - modifiers: payload.modifiers, - ...(payload.radius === undefined ? {} : { radius: payload.radius }) - }, - { timeoutMs: 5_000 } - ) - if (!click.ok && payload.modifiers.length === 0) { - assertTarget(payload, target, args.workspaceAuthority) - await requireRequest(args.client, 'browser.mouseMove', { - ...target, - x: payload.x, - y: payload.y - }) - assertTarget(payload, target, args.workspaceAuthority) - await requireRequest(args.client, 'browser.mouseDown', { - ...target, - button: payload.button - }) - assertTarget(payload, target, args.workspaceAuthority) - await requireRequest(args.client, 'browser.mouseUp', { - ...target, - button: payload.button - }) - } - return MobileWebBrowserCommandResultSchema.parse(null) - } - if (args.operation === 'keyboard') { - const payload = MobileWebBrowserKeyboardPayloadSchema.parse(args.payload) - const target = resolveTarget(payload, args.workspaceAuthority) - await requireRequest( - args.client, - payload.action === 'insertText' ? 'browser.keyboardInsertText' : 'browser.keypress', - payload.action === 'insertText' - ? { ...target, text: payload.text } - : { ...target, key: payload.key }, - 5_000 - ) - return MobileWebBrowserCommandResultSchema.parse(null) - } - if (args.operation === 'dialog') { - const payload = MobileWebBrowserDialogPayloadSchema.parse(args.payload) - const target = resolveTarget(payload, args.workspaceAuthority) - await requireRequest( - args.client, - payload.action === 'accept' ? 'browser.dialogAccept' : 'browser.dialogDismiss', - target, - 5_000 - ) - return MobileWebBrowserCommandResultSchema.parse(null) - } - if (args.operation === 'back' || args.operation === 'forward' || args.operation === 'reload') { - const payload = MobileWebBrowserTargetPayloadSchema.parse(args.payload) - const target = resolveTarget(payload, args.workspaceAuthority) - await requireRequest(args.client, `browser.${args.operation}`, target) - return MobileWebBrowserCommandResultSchema.parse(null) - } - throw new MobileWebBrokerError('unsupported_capability') -} - -// Re-reads the workspace binding mid-operation: a workspace switch must not land later frames. -function assertTarget( - payload: { workspaceId: string; pageId: string }, - expected: { worktree: string; page: string }, - workspaceAuthority: MobileWebWorkspaceAuthority -): void { - const current = resolveTarget(payload, workspaceAuthority) - if (current.worktree !== expected.worktree || current.page !== expected.page) { - throw new MobileWebBrokerError('conflict') - } -} - -function resolveTarget( - payload: { workspaceId: string; pageId: string }, - workspaceAuthority: MobileWebWorkspaceAuthority -): { worktree: string; page: string } { - return { - worktree: `id:${workspaceAuthority.hostWorkspaceId(payload.workspaceId)}`, - page: payload.pageId - } -} - -async function requireRequest( - client: RpcClient, - method: string, - payload: unknown, - timeoutMs = 15_000 -): Promise { - requireResult(await client.sendRequest(method, payload, { timeoutMs })) -} - -function requireResult(response: Awaited>): unknown { - if (!response.ok) { - throw mobileWebBrokerHostRpcError(response.error) - } - return response.result -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} diff --git a/mobile/src/mobile-web/mobile-web-browser-streams.test.ts b/mobile/src/mobile-web/mobile-web-browser-streams.test.ts deleted file mode 100644 index b8c06ad9f88..00000000000 --- a/mobile/src/mobile-web/mobile-web-browser-streams.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import { - BrowserScreencastOpcode, - type BrowserScreencastFrame -} from '../transport/browser-screencast-protocol' -import type { RpcClient } from '../transport/rpc-client' -import { MobileWebBrowserStreams } from './mobile-web-browser-streams' -import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' - -describe('mobile web browser streams', () => { - it('chunks frames below the bridge limit and cleans up', async () => { - const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(4) - const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes) - workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }]) - const workspaceId = workspaceAuthority.pageWorkspaceId('host-workspace') - const pageId = 'raw-page' - const postEvent = vi.fn(async () => {}) - let onEvent: ((event: unknown) => void) | undefined - let onFrame: ((frame: BrowserScreencastFrame) => void) | undefined - const unsubscribe = vi.fn() - const subscribe = vi - .fn() - .mockImplementation((_method, _payload, eventListener, options) => { - onEvent = eventListener - onFrame = options?.onBinaryFrame - return unsubscribe - }) - const streams = new MobileWebBrowserStreams({ - isActive: () => true, - workspaceAuthority, - postEvent, - postClosed: vi.fn() - }) - - streams.start({ - requestId: 'request-1', - subscriptionId: 'subscription-1', - payload: { - workspaceId, - pageId, - format: 'jpeg', - quality: 72, - maxWidth: 800, - maxHeight: 600, - everyNthFrame: 1, - minFrameIntervalMs: 100 - }, - client: { subscribe } as unknown as RpcClient - }) - - expect(subscribe).toHaveBeenCalledWith( - 'browser.screencast', - { - worktree: 'id:host-workspace', - page: 'raw-page', - format: 'jpeg', - quality: 72, - maxWidth: 800, - maxHeight: 600, - everyNthFrame: 1, - minFrameIntervalMs: 100 - }, - expect.any(Function), - { onBinaryFrame: expect.any(Function) } - ) - - onEvent?.({ - type: 'ready', - browserPageId: 'raw-page', - tab: { - url: 'https://example.com', - title: 'Example', - canGoBack: true, - canGoForward: false, - rawSecret: 'must-not-cross' - } - }) - onEvent?.({ - type: 'navigation', - tab: { - url: 'https://www.iana.org/help/example-domains', - title: 'IANA-managed Reserved Domains', - canGoBack: true, - canGoForward: false - } - }) - const image = new Uint8Array(200_000).map((_, index) => index % 251) - onFrame?.({ - opcode: BrowserScreencastOpcode.Frame, - seq: 7, - format: 'jpeg', - metadata: { deviceWidth: 800, deviceHeight: 600 }, - image - }) - - await vi.waitFor(() => expect(postEvent).toHaveBeenCalledTimes(4)) - expect(postEvent.mock.calls.map(([, sequence]) => sequence)).toEqual([0, 1, 2, 3]) - expect(postEvent.mock.calls[0]?.[2]).toEqual({ - type: 'ready', - tab: { - url: 'https://example.com', - title: 'Example', - canGoBack: true, - canGoForward: false - } - }) - expect(postEvent.mock.calls[1]?.[2]).toEqual({ - type: 'navigation', - tab: { - url: 'https://www.iana.org/help/example-domains', - title: 'IANA-managed Reserved Domains', - canGoBack: true, - canGoForward: false - } - }) - const chunks = postEvent.mock.calls.slice(2).map((call) => call[2]) - expect(chunks).toMatchObject([ - { type: 'frameChunk', frameSequence: 7, chunkIndex: 0, chunkCount: 2 }, - { type: 'frameChunk', frameSequence: 7, chunkIndex: 1, chunkCount: 2 } - ]) - expect(JSON.stringify(chunks)).not.toContain('raw-page') - - expect(streams.cancel('subscription-1')).toBe('request-1') - expect(unsubscribe).toHaveBeenCalledOnce() - }) -}) diff --git a/mobile/src/mobile-web/mobile-web-browser-streams.ts b/mobile/src/mobile-web/mobile-web-browser-streams.ts deleted file mode 100644 index f56fd5a4258..00000000000 --- a/mobile/src/mobile-web/mobile-web-browser-streams.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { - MobileWebSubscriptionLedger, - type MobileWebSubscriptionLedgerConfig, - type MobileWebSubscriptionRecord -} from './mobile-web-subscription-ledger' -import { Buffer } from 'buffer/' -import { - MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES, - MOBILE_WEB_BROWSER_FRAME_MAX_IMAGE_BYTES, - MobileWebBrowserEventSchema, - MobileWebBrowserStreamPayloadSchema, - type MobileWebBrowserEvent -} from '../../../src/shared/mobile-web/browser-operation-contract' -import type { - BrowserScreencastFrame, - BrowserScreencastFrameMetadata -} from '../transport/browser-screencast-protocol' -import type { RpcClient } from '../transport/rpc-client' -import { sanitizeMobileWebBrowserEvent } from './mobile-web-browser-event-sanitizer' -import { MobileWebBrokerError } from './mobile-web-broker-error' -import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' - -type ScreencastRecord = MobileWebSubscriptionRecord & { - frameQueued: boolean - pendingFrame: BrowserScreencastFrame | null -} - -type BrowserLedgerConfig = MobileWebSubscriptionLedgerConfig & { - workspaceAuthority: MobileWebWorkspaceAuthority -} - -export class MobileWebBrowserStreams extends MobileWebSubscriptionLedger< - MobileWebBrowserEvent, - ScreencastRecord -> { - constructor(private readonly config: BrowserLedgerConfig) { - super({ ...config, operationKey: 'browser.subscribe' }) - } - - start(args: { - requestId: string - subscriptionId: string - payload: unknown - client: RpcClient - }): void { - this.admit(args.subscriptionId) - const payload = MobileWebBrowserStreamPayloadSchema.parse(args.payload) - const hostWorkspaceId = this.config.workspaceAuthority.hostWorkspaceId(payload.workspaceId) - const record: ScreencastRecord = { - ...this.newRecord(args.requestId), - frameQueued: false, - pendingFrame: null - } - this.open(args.subscriptionId, record, () => - args.client.subscribe( - 'browser.screencast', - { - worktree: `id:${hostWorkspaceId}`, - page: payload.pageId, - format: payload.format, - quality: payload.quality, - maxWidth: payload.maxWidth, - maxHeight: payload.maxHeight, - everyNthFrame: payload.everyNthFrame, - minFrameIntervalMs: payload.minFrameIntervalMs, - ...(payload.viewportWidth === undefined ? {} : { viewportWidth: payload.viewportWidth }), - ...(payload.viewportHeight === undefined - ? {} - : { viewportHeight: payload.viewportHeight }), - ...(payload.deviceScaleFactor === undefined - ? {} - : { deviceScaleFactor: payload.deviceScaleFactor }), - ...(payload.mobile === undefined ? {} : { mobile: payload.mobile }) - }, - (event) => this.receiveEvent(args.subscriptionId, record, event), - { - onBinaryFrame: (frame) => this.receiveFrame(args.subscriptionId, record, frame) - } - ) - ) - } - - // Drops the parked image so a retired stream cannot pin a multi-megabyte frame. - protected override retire(record: ScreencastRecord): void { - record.pendingFrame = null - } - - private receiveEvent(subscriptionId: string, record: ScreencastRecord, value: unknown): void { - if (!this.isCurrent(subscriptionId, record)) { - return - } - const event = sanitizeMobileWebBrowserEvent(value) - if (event) { - this.enqueueTask(subscriptionId, record, () => this.deliver(subscriptionId, record, event)) - } - } - - private receiveFrame( - subscriptionId: string, - record: ScreencastRecord, - frame: BrowserScreencastFrame - ): void { - if (!this.isCurrent(subscriptionId, record)) { - return - } - if (frame.image.byteLength > MOBILE_WEB_BROWSER_FRAME_MAX_IMAGE_BYTES) { - this.enqueueTask(subscriptionId, record, () => - this.deliver(subscriptionId, record, { - type: 'error', - message: 'Browser frame is too large to display safely.' - }) - ) - return - } - record.pendingFrame = frame - if (record.frameQueued) { - return - } - record.frameQueued = true - this.enqueueTask(subscriptionId, record, async () => { - const latest = record.pendingFrame - record.pendingFrame = null - if (latest) { - await this.deliverFrame(subscriptionId, record, latest) - } - record.frameQueued = false - const pending = record.pendingFrame - if (pending && this.isCurrent(subscriptionId, record)) { - record.pendingFrame = null - this.receiveFrame(subscriptionId, record, pending) - } - }) - } - - private async deliverFrame( - subscriptionId: string, - record: ScreencastRecord, - frame: BrowserScreencastFrame - ): Promise { - const chunkCount = Math.ceil(frame.image.byteLength / MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES) - for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex += 1) { - if (!this.isCurrent(subscriptionId, record)) { - return - } - const start = chunkIndex * MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES - const end = Math.min(frame.image.byteLength, start + MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES) - await this.deliver(subscriptionId, record, { - type: 'frameChunk', - frameSequence: frame.seq, - format: frame.format, - metadata: boundedMetadata(frame.metadata), - imageBytes: frame.image.byteLength, - chunkIndex, - chunkCount, - data: Buffer.from(frame.image.subarray(start, end)).toString('base64') - }) - } - } - - // Why the sequence is claimed here and not at enqueue time: one queued frame task posts a whole - // chunk run, so the numbers must be handed out per post, in post order. - private async deliver( - subscriptionId: string, - record: ScreencastRecord, - value: MobileWebBrowserEvent - ): Promise { - const event = MobileWebBrowserEventSchema.parse(value) - const sequence = record.sequence - record.sequence += 1 - await this.options.postEvent(subscriptionId, sequence, event) - } -} - -function boundedMetadata(metadata: BrowserScreencastFrameMetadata): BrowserScreencastFrameMetadata { - const parsed = MobileWebBrowserEventSchema.parse({ - type: 'frameChunk', - frameSequence: 0, - format: 'jpeg', - metadata, - imageBytes: 1, - chunkIndex: 0, - chunkCount: 1, - data: 'AA==' - }) - if (parsed.type !== 'frameChunk') { - throw new MobileWebBrokerError('host_error') - } - return parsed.metadata -} diff --git a/mobile/src/mobile-web/mobile-web-production-browser-grants.ts b/mobile/src/mobile-web/mobile-web-production-browser-grants.ts deleted file mode 100644 index a2595733af7..00000000000 --- a/mobile/src/mobile-web/mobile-web-production-browser-grants.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { capabilityGrants, grantLimits } from './mobile-web-production-grant-table' - -export const MOBILE_WEB_PRODUCTION_BROWSER_GRANTS = capabilityGrants('browser', { - subscribe: grantLimits(4 * 1024, 1 * 1024, 1, 4, 1), - navigate: grantLimits(8 * 1024, 8 * 1024, 1, 8, 2), - back: grantLimits(2 * 1024, 256, 2, 12, 4), - forward: grantLimits(2 * 1024, 256, 2, 12, 4), - reload: grantLimits(2 * 1024, 256, 2, 12, 4), - dialog: grantLimits(2 * 1024, 256, 2, 12, 4), - pointer: grantLimits(4 * 1024, 256, 2, 40, 20), - keyboard: grantLimits(40 * 1024, 256, 2, 20, 10) -}) diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts b/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts index 576568f5285..f81ff0a0314 100644 --- a/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts +++ b/src/main/runtime/rpc/methods/mobile-web-browser-input.test.ts @@ -138,7 +138,7 @@ describe('host-owned browser input', () => { let rejected = 0 for (let attempt = 0; attempt < 60; attempt += 1) { - await pointer.handler(move, f.context).catch(() => { + await Promise.resolve(pointer.handler(move, f.context)).catch(() => { rejected += 1 }) } diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts b/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts index a075693e234..c51560fe59a 100644 --- a/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts +++ b/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts @@ -3,13 +3,14 @@ import { BrowserScreencastOpcode, encodeBrowserScreencastFrame } from '../../../../shared/browser-screencast-protocol' -import type { RpcContext } from '../core' +import { isStreamingMethod, type RpcContext, type RpcMethod } from '../core' import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist' import { MOBILE_WEB_BROWSER_STREAM_METHODS } from './mobile-web-browser-stream' -const methods = new Map(MOBILE_WEB_BROWSER_STREAM_METHODS.map((method) => [method.name, method])) -const subscribe = methods.get('mobileWeb.browser.subscribe')! -const unsubscribe = methods.get('mobileWeb.browser.unsubscribe')! +const subscribe = MOBILE_WEB_BROWSER_STREAM_METHODS.find(isStreamingMethod)! +const unsubscribe = MOBILE_WEB_BROWSER_STREAM_METHODS.find( + (method) => !isStreamingMethod(method) +) as RpcMethod const REQUEST = { worktree: 'id:workspace', diff --git a/src/mobile-web/src/mobile-web-bridge-client.ts b/src/mobile-web/src/mobile-web-bridge-client.ts index cba8b1ec43c..978c2e1eac5 100644 --- a/src/mobile-web/src/mobile-web-bridge-client.ts +++ b/src/mobile-web/src/mobile-web-bridge-client.ts @@ -25,6 +25,7 @@ import type { } from '../../shared/mobile-web/browser-operation-contract' import type { MobileWebBrowserRequestClient } from './mobile-web-browser-request-client' import { mobileWebBrowserNavigationClientBindings } from './mobile-web-browser-navigation-client-bindings' +import { subscribeMobileWebHostBrowser } from './mobile-web-host-browser-subscription' import { MobileWebAccountRequestClient } from './mobile-web-account-request-client' import { MobileWebAgentHistoryRequestClient } from './mobile-web-agent-history-request-client' import { mobileWebFileClientBindings } from './mobile-web-file-client-bindings' @@ -258,7 +259,7 @@ export class MobileWebBridgeClient { onEvent: (event: MobileWebBrowserEvent) => void, onError: (error: MobileWebBridgeClientError) => void ): MobileWebBridgeSubscription { - return this.subscriptions.subscribeBrowser(payload, onEvent, onError) + return subscribeMobileWebHostBrowser(this.subscriptions, payload, onEvent, onError) } receive(message: MobileWebBridgeShellMessage): void { diff --git a/src/mobile-web/src/mobile-web-bridge-subscription-client.ts b/src/mobile-web/src/mobile-web-bridge-subscription-client.ts index 25ea6873d36..b0c24990b0d 100644 --- a/src/mobile-web/src/mobile-web-bridge-subscription-client.ts +++ b/src/mobile-web/src/mobile-web-bridge-subscription-client.ts @@ -19,7 +19,6 @@ import { deliverMobileWebSubscriptionEvent } from './mobile-web-bridge-subscript import { accountSubscriptionSetup, terminalSubscriptionSetup, - browserSubscriptionSetup, speechSubscriptionSetup, workspaceSubscriptionSetup, type MobileWebBridgeSubscriptionSetup @@ -94,10 +93,6 @@ export class MobileWebBridgeSubscriptionClient { return this.subscribeWith(hostSubscriptionSetup(...args)) } - subscribeBrowser(...args: Parameters) { - return this.subscribeWith(browserSubscriptionSetup(...args)) - } - subscribeSpeech(...args: Parameters) { return this.subscribeWith(speechSubscriptionSetup(...args)) } diff --git a/src/mobile-web/src/mobile-web-bridge-subscription-setup.ts b/src/mobile-web/src/mobile-web-bridge-subscription-setup.ts index bedfd657c3b..284fa77fa49 100644 --- a/src/mobile-web/src/mobile-web-bridge-subscription-setup.ts +++ b/src/mobile-web/src/mobile-web-bridge-subscription-setup.ts @@ -17,12 +17,6 @@ import { type MobileWebWorkspaceChange } from '../../shared/mobile-web/bridge-operation-contract' import type { MobileWebBridgeClientError } from './mobile-web-bridge-client-error' -import { - MobileWebBrowserEventSchema, - MobileWebBrowserStreamPayloadSchema, - type MobileWebBrowserEvent, - type MobileWebBrowserStreamPayload -} from '../../shared/mobile-web/browser-operation-contract' import { MobileWebSpeechEventSchema, MobileWebSpeechSubscribePayloadSchema, @@ -67,21 +61,6 @@ export function speechSubscriptionSetup( } } -export function browserSubscriptionSetup( - payload: MobileWebBrowserStreamPayload, - onEvent: (event: MobileWebBrowserEvent) => void, - onError: (error: MobileWebBridgeClientError) => void -): MobileWebBridgeSubscriptionSetup { - return { - capability: 'browser', - payload, - payloadSchema: MobileWebBrowserStreamPayloadSchema, - eventSchema: MobileWebBrowserEventSchema, - onEvent: (value) => onEvent(value as MobileWebBrowserEvent), - onError - } -} - export function workspaceSubscriptionSetup( onEvent: (event: MobileWebWorkspaceChange) => void, onError: (error: MobileWebBridgeClientError) => void diff --git a/src/mobile-web/src/mobile-web-browser-request-client.test.ts b/src/mobile-web/src/mobile-web-browser-request-client.test.ts new file mode 100644 index 00000000000..dd160801a80 --- /dev/null +++ b/src/mobile-web/src/mobile-web-browser-request-client.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it, vi } from 'vitest' +import { MobileWebBrowserRequestClient } from './mobile-web-browser-request-client' +import type { MobileWebOneShotRequestClient } from './mobile-web-one-shot-request-client' + +const TARGET = { workspaceId: 'workspace-1', pageId: 'browser-1' } as const + +function fixture(result: unknown = { applied: true }) { + const request = vi.fn(async (..._args: unknown[]) => result) + const requests = { supports: () => true, request } as unknown as MobileWebOneShotRequestClient + return { request, client: new MobileWebBrowserRequestClient(requests) } +} + +function payloads( + request: ReturnType['request'] +): { method: string; workspaceId?: string; params: Record }[] { + return request.mock.calls.map( + (call) => call[2] as { method: string; workspaceId?: string; params: Record } + ) +} + +describe('page-owned browser commands', () => { + it('names the workspace in the envelope and the page in the params', async () => { + const f = fixture({ url: 'https://app.example/' }) + + await expect(f.client.navigate({ ...TARGET, url: 'https://app.example/' })).resolves.toEqual({ + url: 'https://app.example/' + }) + + expect(payloads(f.request)).toEqual([ + { + method: 'mobileWeb.browser.navigate', + workspaceId: 'workspace-1', + params: { page: 'browser-1', url: 'https://app.example/' } + } + ]) + }) + + it('sends one host request per pointer, keyboard and dialog action', async () => { + const f = fixture() + + await expect( + f.client.pointer({ ...TARGET, action: 'scroll', x: 1, y: 2, dx: 0, dy: -30 }) + ).resolves.toBeNull() + await f.client.pointer({ + ...TARGET, + action: 'click', + x: 3, + y: 4, + button: 'left', + modifiers: ['cmd'], + radius: 8 + }) + await f.client.keyboard({ ...TARGET, action: 'insertText', text: 'hi' }) + await f.client.keyboard({ ...TARGET, action: 'keypress', key: 'Escape' }) + await f.client.dialog({ ...TARGET, action: 'accept' }) + + expect(payloads(f.request)).toEqual([ + { + method: 'mobileWeb.browser.pointer', + workspaceId: 'workspace-1', + params: { page: 'browser-1', action: 'scroll', x: 1, y: 2, dx: 0, dy: -30 } + }, + { + method: 'mobileWeb.browser.pointer', + workspaceId: 'workspace-1', + params: { + page: 'browser-1', + action: 'click', + x: 3, + y: 4, + button: 'left', + modifiers: ['cmd'], + radius: 8 + } + }, + { + method: 'mobileWeb.browser.keyboard', + workspaceId: 'workspace-1', + params: { page: 'browser-1', action: 'insertText', text: 'hi' } + }, + { + method: 'mobileWeb.browser.keyboard', + workspaceId: 'workspace-1', + params: { page: 'browser-1', action: 'keypress', key: 'Escape' } + }, + { + method: 'mobileWeb.browser.dialog', + workspaceId: 'workspace-1', + params: { page: 'browser-1', action: 'accept' } + } + ]) + }) + + it('carries back, forward and reload as one history method', async () => { + const f = fixture() + + await f.client.back(TARGET) + await f.client.forward(TARGET) + await f.client.reload(TARGET) + + expect(payloads(f.request)).toEqual( + ['back', 'forward', 'reload'].map((action) => ({ + method: 'mobileWeb.browser.history', + workspaceId: 'workspace-1', + params: { page: 'browser-1', action } + })) + ) + }) + + it('rejects a command the host did not acknowledge as applied', async () => { + const f = fixture({ applied: false }) + + await expect(f.client.reload(TARGET)).rejects.toMatchObject({ code: 'invalid_message' }) + }) + + it('rejects a navigation result with no URL and tolerates an unknown field', async () => { + const f = fixture({ title: 'App' }) + await expect(f.client.navigate({ ...TARGET, url: 'https://a.example/' })).rejects.toMatchObject( + { + code: 'invalid_message' + } + ) + + const forwardCompatible = fixture({ url: 'https://a.example/', title: 'App' }) + await expect( + forwardCompatible.client.navigate({ ...TARGET, url: 'https://a.example/' }) + ).resolves.toEqual({ url: 'https://a.example/' }) + }) + + it('never forwards the opaque workspace handle inside the host params', async () => { + const f = fixture() + + await f.client.dialog({ ...TARGET, action: 'dismiss' }) + + expect(JSON.stringify(payloads(f.request)[0]!.params)).not.toContain('workspace-1') + }) +}) diff --git a/src/mobile-web/src/mobile-web-browser-request-client.ts b/src/mobile-web/src/mobile-web-browser-request-client.ts index dde0c6f9745..6aafb8b3693 100644 --- a/src/mobile-web/src/mobile-web-browser-request-client.ts +++ b/src/mobile-web/src/mobile-web-browser-request-client.ts @@ -1,32 +1,39 @@ +import type { z } from 'zod' import { - MobileWebBrowserCommandResultSchema, - MobileWebBrowserDialogPayloadSchema, - MobileWebBrowserKeyboardPayloadSchema, - MobileWebBrowserNavigatePayloadSchema, + MobileWebBrowserAckSchema, MobileWebBrowserNavigateResultSchema, - MobileWebBrowserPointerPayloadSchema, - MobileWebBrowserTargetPayloadSchema, type MobileWebBrowserDialogPayload, type MobileWebBrowserKeyboardPayload, type MobileWebBrowserNavigatePayload, type MobileWebBrowserPointerPayload, type MobileWebBrowserTargetPayload } from '../../shared/mobile-web/browser-operation-contract' +import { MobileWebBridgeClientError } from './mobile-web-bridge-client-error' import type { MobileWebBridgeRequestOptions } from './mobile-web-bridge-request-state' +import { requestMobileWebHost } from './mobile-web-host-request-client' import type { MobileWebOneShotRequestClient } from './mobile-web-one-shot-request-client' +/** The workspace handle rides in the host request envelope, and the browser page id is the only + * target field the desktop wrapper reads. */ +function hostParams({ + workspaceId: _workspaceId, + pageId, + ...fields +}: TPayload): Record { + return { page: pageId, ...fields } +} + export class MobileWebBrowserRequestClient { constructor(private readonly requests: MobileWebOneShotRequestClient) {} - navigate( + async navigate( payload: MobileWebBrowserNavigatePayload, options?: MobileWebBridgeRequestOptions ): Promise<{ url: string }> { - return this.requests.request( - 'browser', - 'navigate', + return this.send( + 'mobileWeb.browser.navigate', payload, - MobileWebBrowserNavigatePayloadSchema, + hostParams(payload), MobileWebBrowserNavigateResultSchema, options ) @@ -36,77 +43,85 @@ export class MobileWebBrowserRequestClient { payload: MobileWebBrowserPointerPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.requests.request( - 'browser', - 'pointer', - payload, - MobileWebBrowserPointerPayloadSchema, - MobileWebBrowserCommandResultSchema, - options - ) + return this.command('mobileWeb.browser.pointer', payload, hostParams(payload), options) } keyboard( payload: MobileWebBrowserKeyboardPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.requests.request( - 'browser', - 'keyboard', - payload, - MobileWebBrowserKeyboardPayloadSchema, - MobileWebBrowserCommandResultSchema, - options - ) + return this.command('mobileWeb.browser.keyboard', payload, hostParams(payload), options) } dialog( payload: MobileWebBrowserDialogPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.requests.request( - 'browser', - 'dialog', - payload, - MobileWebBrowserDialogPayloadSchema, - MobileWebBrowserCommandResultSchema, - options - ) + return this.command('mobileWeb.browser.dialog', payload, hostParams(payload), options) } back( payload: MobileWebBrowserTargetPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.command('back', payload, options) + return this.history('back', payload, options) } forward( payload: MobileWebBrowserTargetPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.command('forward', payload, options) + return this.history('forward', payload, options) } reload( payload: MobileWebBrowserTargetPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.command('reload', payload, options) + return this.history('reload', payload, options) } - private command( - operation: 'back' | 'forward' | 'reload', + private history( + action: 'back' | 'forward' | 'reload', payload: MobileWebBrowserTargetPayload, options?: MobileWebBridgeRequestOptions ): Promise { - return this.requests.request( - 'browser', - operation, + return this.command( + 'mobileWeb.browser.history', payload, - MobileWebBrowserTargetPayloadSchema, - MobileWebBrowserCommandResultSchema, + { ...hostParams(payload), action }, options ) } + + private async command( + method: string, + payload: MobileWebBrowserTargetPayload, + params: Record, + options?: MobileWebBridgeRequestOptions + ): Promise { + await this.send(method, payload, params, MobileWebBrowserAckSchema, options) + return null + } + + private async send( + method: string, + payload: MobileWebBrowserTargetPayload, + params: Record, + schema: z.ZodType, + options?: MobileWebBridgeRequestOptions + ): Promise { + const result = await requestMobileWebHost( + this.requests, + method, + payload.workspaceId, + params, + options + ) + const parsed = schema.safeParse(result) + if (!parsed.success) { + throw new MobileWebBridgeClientError('invalid_message', false) + } + return parsed.data + } } diff --git a/src/mobile-web/src/mobile-web-host-browser-subscription.test.ts b/src/mobile-web/src/mobile-web-host-browser-subscription.test.ts new file mode 100644 index 00000000000..3a8c481be9c --- /dev/null +++ b/src/mobile-web/src/mobile-web-host-browser-subscription.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from 'vitest' +import type { MobileWebBrowserEvent } from '../../shared/mobile-web/browser-operation-contract' +import type { MobileWebBridgeClientError } from './mobile-web-bridge-client-error' +import type { MobileWebBridgeSubscriptionClient } from './mobile-web-bridge-subscription-client' +import { subscribeMobileWebHostBrowser } from './mobile-web-host-browser-subscription' + +const PAYLOAD = { + workspaceId: 'workspace-1', + pageId: 'browser-1', + format: 'jpeg' as const, + quality: 72, + maxWidth: 800, + maxHeight: 600, + everyNthFrame: 1, + minFrameIntervalMs: 100 +} + +function fixture() { + const unsubscribe = vi.fn() + let emit: (event: unknown) => void = () => {} + const subscribeHost = vi.fn((_payload: unknown, onEvent: (event: unknown) => void) => { + emit = onEvent + return { ready: Promise.resolve(), unsubscribe } + }) + const subscriptions = { subscribeHost } as unknown as MobileWebBridgeSubscriptionClient + const events: MobileWebBrowserEvent[] = [] + const errors: MobileWebBridgeClientError[] = [] + const subscription = subscribeMobileWebHostBrowser( + subscriptions, + PAYLOAD, + (event) => events.push(event), + (error) => errors.push(error) + ) + return { emit: (event: unknown) => emit(event), errors, events, subscribeHost, subscription } +} + +describe('page-owned browser stream', () => { + it('opens the host stream with the page id and the screencast request', () => { + const f = fixture() + + expect(f.subscribeHost.mock.calls[0]![0]).toEqual({ + method: 'mobileWeb.browser.subscribe', + workspaceId: 'workspace-1', + params: { + page: 'browser-1', + format: 'jpeg', + quality: 72, + maxWidth: 800, + maxHeight: 600, + everyNthFrame: 1, + minFrameIntervalMs: 100 + } + }) + }) + + it('swallows the lane handshake and delivers the browser ready that carries tab state', () => { + const f = fixture() + const tab = { url: 'https://a.example/', title: 'A', canGoBack: false, canGoForward: false } + + f.emit({ type: 'ready', subscriptionId: 'lane-id' }) + f.emit({ type: 'ready', tab }) + + expect(f.events).toEqual([{ type: 'ready', tab }]) + expect(f.errors).toEqual([]) + }) + + it('reports an event the browser contract cannot describe', () => { + const f = fixture() + + f.emit({ type: 'frameChunk', chunkIndex: 4, chunkCount: 1 }) + + expect(f.events).toEqual([]) + expect(f.errors.map((error) => error.code)).toEqual(['invalid_message']) + }) + + it('stops delivering once the page unsubscribes', () => { + const f = fixture() + + f.subscription.unsubscribe() + f.emit({ type: 'dialogClosed' }) + + expect(f.events).toEqual([]) + }) + + it('refuses a screencast request the contract does not admit', async () => { + const errors: MobileWebBridgeClientError[] = [] + const subscribeHost = vi.fn() + const subscription = subscribeMobileWebHostBrowser( + { subscribeHost } as unknown as MobileWebBridgeSubscriptionClient, + { ...PAYLOAD, quality: 0 }, + () => {}, + (error) => errors.push(error) + ) + + await expect(subscription.ready).rejects.toMatchObject({ code: 'invalid_request' }) + await Promise.resolve() + expect(subscribeHost).not.toHaveBeenCalled() + expect(errors.map((error) => error.code)).toEqual(['invalid_request']) + }) +}) diff --git a/src/mobile-web/src/mobile-web-host-browser-subscription.ts b/src/mobile-web/src/mobile-web-host-browser-subscription.ts new file mode 100644 index 00000000000..f5a303950f1 --- /dev/null +++ b/src/mobile-web/src/mobile-web-host-browser-subscription.ts @@ -0,0 +1,70 @@ +import { + MobileWebBrowserEventSchema, + MobileWebBrowserStreamPayloadSchema, + type MobileWebBrowserEvent, + type MobileWebBrowserStreamPayload +} from '../../shared/mobile-web/browser-operation-contract' +import { MobileWebBridgeClientError } from './mobile-web-bridge-client-error' +import type { MobileWebBridgeSubscription } from './mobile-web-bridge-subscription' +import type { MobileWebBridgeSubscriptionClient } from './mobile-web-bridge-subscription-client' + +export type MobileWebBrowserSubscriptionArgs = [ + payload: MobileWebBrowserStreamPayload, + onEvent: (event: MobileWebBrowserEvent) => void, + onError: (error: MobileWebBridgeClientError) => void +] + +export function subscribeMobileWebHostBrowser( + subscriptions: MobileWebBridgeSubscriptionClient, + ...[payload, onEvent, onError]: MobileWebBrowserSubscriptionArgs +): MobileWebBridgeSubscription { + if (!MobileWebBrowserStreamPayloadSchema.safeParse(payload).success) { + const error = new MobileWebBridgeClientError('invalid_request', false) + queueMicrotask(() => onError(error)) + return { ready: Promise.reject(error), unsubscribe() {} } + } + const { workspaceId, pageId, ...request } = payload + let cancelled = false + const current = subscriptions.subscribeHost( + { + method: 'mobileWeb.browser.subscribe', + workspaceId, + params: { page: pageId, ...request } + }, + (event) => { + if (cancelled || isSubscriptionHandshake(event)) { + return + } + const parsed = MobileWebBrowserEventSchema.safeParse(event) + if (!parsed.success) { + onError(new MobileWebBridgeClientError('invalid_message', false)) + return + } + onEvent(parsed.data) + }, + (error) => { + if (!cancelled) { + onError(error) + } + } + ) + return { + ready: current.ready, + unsubscribe() { + cancelled = true + current.unsubscribe() + } + } +} + +/** The lane opens with a bare `ready` carrying the cancel id; the browser's own `ready` carries + * tab state, which is what the pane waits for. */ +function isSubscriptionHandshake(event: unknown): boolean { + return ( + typeof event === 'object' && + event !== null && + 'type' in event && + event.type === 'ready' && + !('tab' in event) + ) +} diff --git a/src/shared/mobile-web/browser-operation-contract.ts b/src/shared/mobile-web/browser-operation-contract.ts index d975a7452c0..2baa24d6181 100644 --- a/src/shared/mobile-web/browser-operation-contract.ts +++ b/src/shared/mobile-web/browser-operation-contract.ts @@ -92,13 +92,14 @@ export const MobileWebBrowserDialogPayloadSchema = MobileWebBrowserTargetSchema. action: z.enum(['accept', 'dismiss']) }).strict() -export const MobileWebBrowserNavigateResultSchema = z - .object({ - url: z.string().min(1).max(MOBILE_WEB_BROWSER_URL_MAX_LENGTH) - }) - .strict() +// Desktop-produced results stay open: a newer host may add a field a cached page has never seen. +export const MobileWebBrowserNavigateResultSchema = z.object({ + url: z.string().min(1).max(MOBILE_WEB_BROWSER_URL_MAX_LENGTH) +}) -export const MobileWebBrowserCommandResultSchema = z.null() +/** Every browser command the page issues answers with this acknowledgement; the host result + * carries the raw tab URL, which the desktop wrapper never forwards. */ +export const MobileWebBrowserAckSchema = z.object({ applied: z.literal(true) }) const MobileWebBrowserTabStateSchema = z .object({ From 4bb3f2e6e64b82389cc9764e4f32f4e0c0ea0f42 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 7 Sep 2026 15:07:47 -0400 Subject: [PATCH 3/4] refactor(mobile): delete the shell browser translators, streams and grants The browser capability is gone from the bridge registry, so the shell no longer names a browser RPC, sanitizes a tab URL or chunks a frame. hostRequest carries pointer traffic now, so its bucket takes over the retired pointer grant's rate. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .../mobile-web-capability-broker.ts | 1 - ...ile-web-capability-dispatch-census.test.ts | 2 +- .../mobile-web-capability-execution-arms.ts | 25 ---------- ...e-web-capability-execution-dependencies.ts | 2 - .../mobile-web-capability-subscriptions.ts | 8 +--- ...eb-mutation-reauthorization-census.test.ts | 11 +---- .../mobile-web-production-grants.ts | 4 +- .../mobile-web-request-accounting.ts | 1 - ...le-web-subscription-closure-frames.test.ts | 46 +------------------ ...mobile-web-subscription-error-code.test.ts | 34 -------------- .../mobile-web/bridge-operation-registry.ts | 10 ---- 11 files changed, 7 insertions(+), 137 deletions(-) diff --git a/mobile/src/mobile-web/mobile-web-capability-broker.ts b/mobile/src/mobile-web/mobile-web-capability-broker.ts index 605d3a9bba7..295379a0c09 100644 --- a/mobile/src/mobile-web/mobile-web-capability-broker.ts +++ b/mobile/src/mobile-web/mobile-web-capability-broker.ts @@ -244,7 +244,6 @@ export class MobileWebCapabilityBroker { agentHistoryPager: this.authorities.agentHistoryPager, agentHistoryResume: this.authorities.agentHistoryResume, accountSubscriptions: this.subscriptions.account, - browserStreams: this.subscriptions.browser, sourceControlBranchCompare: this.authorities.sourceControlBranchCompare, speechAuthority: this.speechAuthority, workspaceSubscriptions: this.subscriptions.workspace, diff --git a/mobile/src/mobile-web/mobile-web-capability-dispatch-census.test.ts b/mobile/src/mobile-web/mobile-web-capability-dispatch-census.test.ts index 699b55d83b3..900289e2cc0 100644 --- a/mobile/src/mobile-web/mobile-web-capability-dispatch-census.test.ts +++ b/mobile/src/mobile-web/mobile-web-capability-dispatch-census.test.ts @@ -49,7 +49,7 @@ describe('mobile web capability dispatch census', () => { }) expect(unresolved.map(({ capability, operation }) => `${capability}.${operation}`)).toEqual([]) - expect(registeredOperations()).toHaveLength(199) + expect(registeredOperations()).toHaveLength(191) }) it('carries a dispatch arm for exactly the capabilities that own operations of that mode', () => { diff --git a/mobile/src/mobile-web/mobile-web-capability-execution-arms.ts b/mobile/src/mobile-web/mobile-web-capability-execution-arms.ts index fc96f592881..48b72a06464 100644 --- a/mobile/src/mobile-web/mobile-web-capability-execution-arms.ts +++ b/mobile/src/mobile-web/mobile-web-capability-execution-arms.ts @@ -1,4 +1,3 @@ -import { MobileWebBrowserStreamPayloadSchema } from '../../../src/shared/mobile-web/browser-operation-contract' import { MobileWebWorkspaceSubscribePayloadSchema } from '../../../src/shared/mobile-web/bridge-operation-contract' import type { MobileWebBridgePageMessage } from '../../../src/shared/mobile-web/bridge-contract' import type { MobileWebBridgeCapability } from '../../../src/shared/mobile-web/bridge-operation-registry' @@ -7,7 +6,6 @@ import { executeWorkspace } from './mobile-web-workspace-capability' import { executeMobileWebAccountCapability } from './mobile-web-account-capability' import { executeMobileWebAgentHistoryOperation } from './mobile-web-agent-history-operations' import { MobileWebBrokerError } from './mobile-web-broker-error' -import { executeMobileWebBrowserOperation } from './mobile-web-browser-operations' import type { MobileWebCapabilityExecutionDependencies } from './mobile-web-capability-execution-dependencies' import { executeMobileWebFileOperation } from './mobile-web-file-operations' import { executeMobileWebMarkdownOperation } from './mobile-web-markdown-operations' @@ -53,15 +51,6 @@ async function executeNavigation(args: Deps, request: OnceRequest): Promise { - return executeMobileWebBrowserOperation({ - operation: request.operation, - payload: request.payload, - client: args.connectedClient(), - workspaceAuthority: args.workspaceAuthority - }) -} - async function executeTerminal(args: Deps, request: OnceRequest): Promise { return args.terminalStreams.handle(request.payload, args.connectedClient()) } @@ -158,7 +147,6 @@ export const MOBILE_WEB_ONCE_CAPABILITY_ARMS: Partial executeMobileWebAgentHistoryOperation(args), account: (args) => executeMobileWebAccountCapability(args), - browser: executeBrowser, workspace: executeWorkspace, settings: executeWorkspace, terminal: executeTerminal, @@ -169,18 +157,6 @@ export const MOBILE_WEB_ONCE_CAPABILITY_ARMS: Partial { - requireSubscribeOperation(request) - MobileWebBrowserStreamPayloadSchema.parse(request.payload) - args.browserStreams.start({ - requestId: request.requestId, - subscriptionId: request.subscriptionId, - payload: request.payload, - client: args.connectedClient() - }) - return null -} - async function subscribeWorkspace(args: Deps, request: SubscriptionRequest): Promise { if (request.operation === 'hostSubscribe') { args.hostSubscriptions.start({ @@ -228,7 +204,6 @@ export const MOBILE_WEB_SUBSCRIPTION_CAPABILITY_ARMS: Partial< Record > = { account: (args) => executeMobileWebAccountCapability(args), - browser: subscribeBrowser, workspace: subscribeWorkspace, terminal: subscribeTerminal, speech: subscribeSpeech diff --git a/mobile/src/mobile-web/mobile-web-capability-execution-dependencies.ts b/mobile/src/mobile-web/mobile-web-capability-execution-dependencies.ts index f1318338797..da72a658218 100644 --- a/mobile/src/mobile-web/mobile-web-capability-execution-dependencies.ts +++ b/mobile/src/mobile-web/mobile-web-capability-execution-dependencies.ts @@ -5,7 +5,6 @@ import type { MobileWebAccountSubscriptions } from './mobile-web-account-subscri import type { MobileWebAgentHistoryAuthority } from './mobile-web-agent-history-authority' import type { MobileWebAgentHistoryPager } from './mobile-web-agent-history-pager' import type { MobileWebAgentHistoryResume } from './mobile-web-agent-history-resume' -import type { MobileWebBrowserStreams } from './mobile-web-browser-streams' import type { MobileWebCommitMessageGeneration } from './mobile-web-commit-message-generation' import type { MobileWebNavigationAuthority } from './mobile-web-navigation-operations' import type { MobileWebNativeCapabilityAuthority } from './mobile-web-native-capability-authority' @@ -33,7 +32,6 @@ export type MobileWebCapabilityExecutionDependencies = { agentHistoryResume: MobileWebAgentHistoryResume hostSubscriptions: MobileWebHostSubscriptions accountSubscriptions: MobileWebAccountSubscriptions - browserStreams: MobileWebBrowserStreams sourceControlBranchCompare: MobileWebSourceControlBranchComparePager speechAuthority: MobileWebSpeechAuthority workspaceSubscriptions: MobileWebWorkspaceSubscriptions diff --git a/mobile/src/mobile-web/mobile-web-capability-subscriptions.ts b/mobile/src/mobile-web/mobile-web-capability-subscriptions.ts index 1b23a842e3e..aaaef8e3f8c 100644 --- a/mobile/src/mobile-web/mobile-web-capability-subscriptions.ts +++ b/mobile/src/mobile-web/mobile-web-capability-subscriptions.ts @@ -5,14 +5,12 @@ import type { MobileWebSubscriptionLedgerConfig, MobileWebSubscriptionLedgerHandle } from './mobile-web-subscription-ledger' -import { MobileWebBrowserStreams } from './mobile-web-browser-streams' import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' import { MobileWebWorkspaceSubscriptions } from './mobile-web-workspace-subscriptions' export class MobileWebCapabilitySubscriptions { readonly host: MobileWebHostSubscriptions readonly account: MobileWebAccountSubscriptions - readonly browser: MobileWebBrowserStreams readonly workspace: MobileWebWorkspaceSubscriptions private readonly ledgers: MobileWebSubscriptionLedgerHandle[] @@ -31,12 +29,8 @@ export class MobileWebCapabilitySubscriptions { workspaceAuthority: args.workspaceAuthority }) this.account = new MobileWebAccountSubscriptions(shared) - this.browser = new MobileWebBrowserStreams({ - ...shared, - workspaceAuthority: args.workspaceAuthority - }) this.workspace = new MobileWebWorkspaceSubscriptions(shared) - this.ledgers = [this.host, this.account, this.browser, this.workspace] + this.ledgers = [this.host, this.account, this.workspace] } countForOperation(operationKey: string): number { diff --git a/mobile/src/mobile-web/mobile-web-mutation-reauthorization-census.test.ts b/mobile/src/mobile-web/mobile-web-mutation-reauthorization-census.test.ts index 54863942db5..eb680a3cb4c 100644 --- a/mobile/src/mobile-web/mobile-web-mutation-reauthorization-census.test.ts +++ b/mobile/src/mobile-web/mobile-web-mutation-reauthorization-census.test.ts @@ -43,13 +43,6 @@ const REAUTHORIZATION_SITES: Record = { // Device-only mutations and handles consumed in one awaited call have no reauthorization window. const NO_REAUTHORIZATION_WINDOW: readonly string[] = [ 'workspace.activate', - 'browser.back', - 'browser.dialog', - 'browser.forward', - 'browser.keyboard', - 'browser.navigate', - 'browser.pointer', - 'browser.reload', 'file.releaseTerminalArtifact', 'native.alert', 'native.clipboardWrite', @@ -167,7 +160,7 @@ describe('mobile web mutation reauthorization census', () => { } expect(unaccounted).toEqual([]) - expect(mutations()).toHaveLength(113) + expect(mutations()).toHaveLength(106) }) it('exempts only registered mutations', () => { @@ -185,6 +178,6 @@ describe('mobile web mutation reauthorization census', () => { const capabilities = Object.keys(MOBILE_WEB_BRIDGE_OPERATIONS) as MobileWebBridgeCapability[] expect([...kinds].sort()).toEqual(['mutation', 'read', 'subscription']) - expect(capabilities).toHaveLength(14) + expect(capabilities).toHaveLength(13) }) }) diff --git a/mobile/src/mobile-web/mobile-web-production-grants.ts b/mobile/src/mobile-web/mobile-web-production-grants.ts index 354e9a7ff39..6dda665d957 100644 --- a/mobile/src/mobile-web/mobile-web-production-grants.ts +++ b/mobile/src/mobile-web/mobile-web-production-grants.ts @@ -1,4 +1,3 @@ -import { MOBILE_WEB_PRODUCTION_BROWSER_GRANTS } from './mobile-web-production-browser-grants' import { MOBILE_WEB_PRODUCTION_FILE_GRANTS } from './mobile-web-production-file-grants' import { capabilityGrants, grantLimits, indexGrants } from './mobile-web-production-grant-table' import { MOBILE_WEB_PRODUCTION_NAVIGATION_GRANTS } from './mobile-web-production-navigation-grants' @@ -16,7 +15,7 @@ export type { MobileWebOperationGrant } from './mobile-web-production-grant-tabl export const MOBILE_WEB_PRODUCTION_GRANTS = [ ...capabilityGrants('workspace', { hostSubscribe: grantLimits(600 * 1024, 1024, 8, 8, 2), - hostRequest: grantLimits(600 * 1024, 600 * 1024, 16, 32, 4), + hostRequest: grantLimits(600 * 1024, 600 * 1024, 16, 48, 24), snapshot: grantLimits(1 * 1024, 128 * 1024, 2, 4, 1), repositories: grantLimits(256, 128 * 1024, 2, 4, 1), subscribe: grantLimits(256, 1 * 1024, 1, 4, 1), @@ -40,7 +39,6 @@ export const MOBILE_WEB_PRODUCTION_GRANTS = [ }), ...MOBILE_WEB_PRODUCTION_SESSION_GRANTS, ...MOBILE_WEB_PRODUCTION_TERMINAL_GRANTS, - ...MOBILE_WEB_PRODUCTION_BROWSER_GRANTS, ...MOBILE_WEB_PRODUCTION_FILE_GRANTS, ...MOBILE_WEB_PRODUCTION_SOURCE_CONTROL_GRANTS, ...MOBILE_WEB_PRODUCTION_SPEECH_GRANTS, diff --git a/mobile/src/mobile-web/mobile-web-request-accounting.ts b/mobile/src/mobile-web/mobile-web-request-accounting.ts index 64652b7a041..fb50a84817d 100644 --- a/mobile/src/mobile-web/mobile-web-request-accounting.ts +++ b/mobile/src/mobile-web/mobile-web-request-accounting.ts @@ -18,7 +18,6 @@ export function mobileWebRequestExpectsSubscription(request: { request.capability === 'session' || request.capability === 'sourceControl' || request.capability === 'terminal' || - request.capability === 'browser' || request.capability === 'nativeChat' || request.capability === 'speech') && request.operation === 'subscribe') diff --git a/mobile/src/mobile-web/mobile-web-subscription-closure-frames.test.ts b/mobile/src/mobile-web/mobile-web-subscription-closure-frames.test.ts index a29565a3b00..f383e03c817 100644 --- a/mobile/src/mobile-web/mobile-web-subscription-closure-frames.test.ts +++ b/mobile/src/mobile-web/mobile-web-subscription-closure-frames.test.ts @@ -2,11 +2,9 @@ import { describe, expect, it, vi } from 'vitest' import type { MobileWebSubscriptionClosure } from './mobile-web-subscription-closure' import type { RpcClient } from '../transport/rpc-client' import { MobileWebAccountSubscriptions } from './mobile-web-account-subscriptions' -import { MobileWebBrowserStreams } from './mobile-web-browser-streams' import { MobileWebSpeechSubscriptions } from './mobile-web-speech-subscriptions' import type { MobileWebSpeechEvent } from '../../../src/shared/mobile-web/speech-operation-contract' import { MobileWebWorkspaceSubscriptions } from './mobile-web-workspace-subscriptions' -import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' const SUBSCRIPTION_ID = 'subscription-1' @@ -18,17 +16,13 @@ type Posts = { type LedgerCase = { name: string - // Browser drops an unparseable frame instead of retiring, so it has no invalid-message closure. + // A push-driven ledger has no host frame to reject, so it has no invalid-message closure. invalidCode: 'invalid_message' | null invalid: unknown valid: unknown open: (posts: Posts) => Promise<(value: unknown) => void> } -function randomBytes(length: number): Uint8Array { - return new Uint8Array(length).fill(4) -} - function hostClient(): { client: RpcClient; emit: (value: unknown) => void } { let listener: ((value: unknown) => void) | undefined const client = { @@ -54,12 +48,6 @@ function hostClient(): { client: RpcClient; emit: (value: unknown) => void } { return { client, emit: (value) => listener?.(value) } } -function pageWorkspace(): { authority: MobileWebWorkspaceAuthority; pageWorkspaceId: string } { - const authority = new MobileWebWorkspaceAuthority(randomBytes) - authority.synchronize([{ workspaceId: 'workspace-1', repoId: 'repo-1' }]) - return { authority, pageWorkspaceId: authority.pageWorkspaceId('workspace-1') } -} - const LEDGER_CASES: LedgerCase[] = [ { name: 'account', @@ -91,36 +79,6 @@ const LEDGER_CASES: LedgerCase[] = [ return host.emit } }, - { - name: 'browser', - invalidCode: null, - invalid: { type: 'bogus' }, - valid: { - type: 'ready', - browserPageId: 'raw-page', - tab: { url: 'https://example.com', title: 'Example', canGoBack: false, canGoForward: false } - }, - open: async (posts) => { - const host = hostClient() - const { authority, pageWorkspaceId } = pageWorkspace() - new MobileWebBrowserStreams({ ...posts, workspaceAuthority: authority }).start({ - requestId: 'request-1', - subscriptionId: SUBSCRIPTION_ID, - payload: { - workspaceId: pageWorkspaceId, - pageId: 'raw-page', - format: 'jpeg', - quality: 72, - maxWidth: 800, - maxHeight: 600, - everyNthFrame: 1, - minFrameIntervalMs: 100 - }, - client: host.client - }) - return host.emit - } - }, { name: 'speech', // Push-driven from the shell's dictation runtime, so no host frame can be unusable. @@ -135,7 +93,7 @@ const LEDGER_CASES: LedgerCase[] = [ } ] -// Ledgers that retire on an unusable host message; browser drops the frame instead, so it is absent. +// Ledgers that retire on an unusable host message; a push-driven ledger has none, so it is absent. const RETIRING_LEDGER_CASES = LEDGER_CASES.filter( (ledger): ledger is LedgerCase & { invalidCode: 'invalid_message' } => ledger.invalidCode !== null ) diff --git a/mobile/src/mobile-web/mobile-web-subscription-error-code.test.ts b/mobile/src/mobile-web/mobile-web-subscription-error-code.test.ts index 82100c4a3a5..54d6b7edd07 100644 --- a/mobile/src/mobile-web/mobile-web-subscription-error-code.test.ts +++ b/mobile/src/mobile-web/mobile-web-subscription-error-code.test.ts @@ -6,7 +6,6 @@ import type { } from '../../../src/shared/mobile-web/bridge-contract' import type { RpcClient } from '../transport/rpc-client' import { MobileWebAccountSubscriptions } from './mobile-web-account-subscriptions' -import { MobileWebBrowserStreams } from './mobile-web-browser-streams' import { isRetryableMobileWebBridgeError, mobileWebBridgeErrorCode @@ -16,9 +15,6 @@ import { mobileWebBridgeRequestMessage } from './mobile-web-bridge-roundtrip-fixture' import { MobileWebWorkspaceSubscriptions } from './mobile-web-workspace-subscriptions' -import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority' - -const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(4) function stubClient(): RpcClient { return { subscribe: vi.fn(() => () => {}) } as unknown as RpcClient @@ -29,20 +25,9 @@ function ledgerStarters(): { name: string; start: (subscriptionId: string) => vo const postEvent = async (): Promise => {} const isActive = (): boolean => true const client = stubClient() - const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes) - workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }]) - const pageWorkspaceId = workspaceAuthority.pageWorkspaceId('host-workspace') - const pageId = 'raw-page' - const postClosed = (): void => {} const account = new MobileWebAccountSubscriptions({ isActive, postEvent, postClosed }) const workspace = new MobileWebWorkspaceSubscriptions({ isActive, postEvent, postClosed }) - const browser = new MobileWebBrowserStreams({ - isActive, - workspaceAuthority, - postEvent, - postClosed - }) return [ { name: 'account', @@ -51,25 +36,6 @@ function ledgerStarters(): { name: string; start: (subscriptionId: string) => vo { name: 'workspace', start: (subscriptionId) => workspace.start({ requestId: 'r', subscriptionId, client }) - }, - { - name: 'browser', - start: (subscriptionId) => - browser.start({ - requestId: 'r', - subscriptionId, - payload: { - workspaceId: pageWorkspaceId, - pageId, - format: 'jpeg', - quality: 72, - maxWidth: 800, - maxHeight: 600, - everyNthFrame: 1, - minFrameIntervalMs: 100 - }, - client - }) } ] } diff --git a/src/shared/mobile-web/bridge-operation-registry.ts b/src/shared/mobile-web/bridge-operation-registry.ts index bbc58ae2b59..0f05197b9c5 100644 --- a/src/shared/mobile-web/bridge-operation-registry.ts +++ b/src/shared/mobile-web/bridge-operation-registry.ts @@ -167,16 +167,6 @@ export const MOBILE_WEB_BRIDGE_OPERATIONS = { manageReview: 'mutation', submitReview: 'mutation' }, - browser: { - subscribe: 'subscription', - navigate: 'mutation', - back: 'mutation', - forward: 'mutation', - reload: 'mutation', - dialog: 'mutation', - pointer: 'mutation', - keyboard: 'mutation' - }, account: { snapshot: 'read', select: 'mutation', From 490c08669ef5413b907a558a0b6f0ec0becde189 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 7 Sep 2026 15:09:24 -0400 Subject: [PATCH 4/4] docs(mobile): record the desktop-owned browser lane Also ends the stream on a frame the page contract cannot describe, which is what an error event means on this lane. Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb --- .../mobile-hybrid-webview-architecture.md | 27 ++++++++++++++----- .../methods/mobile-web-browser-stream.test.ts | 6 ++--- .../rpc/methods/mobile-web-browser-stream.ts | 4 ++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/reference/mobile-hybrid-webview-architecture.md b/docs/reference/mobile-hybrid-webview-architecture.md index e606d1253e3..90287ad1a86 100644 --- a/docs/reference/mobile-hybrid-webview-architecture.md +++ b/docs/reference/mobile-hybrid-webview-architecture.md @@ -56,12 +56,12 @@ acceptance. ## Ownership Boundaries -| Owner | Responsibilities | -| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Owner | Responsibilities | +| ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Native mobile shell | Pairing and host selection; secure credential storage; authenticated encrypted transport; QR scanning; notifications and deep links; package verification, cache, private origin, and recovery; clipboard, haptics, audio, camera and file/photo pickers; native onboarding, pairing recovery, and privacy; diagnostics capture (transport log, reachability probes, report submission) | -| Desktop-served React Native Web application | Workspace list and creation; sessions and terminal presentation; files, previews, diffs, source control, reviews, tasks, accounts, browser presentation, Agent History, native-chat presentation, and the settings, troubleshooting and connection-log screens | -| Desktop runtime | Builds and ships the matching web package; serves its manifest and chunks through authenticated RPC; reauthorizes every workspace mutation; enforces host, workspace, provider, path, and resource limits | -| Typed native bridge | Connects the unprivileged page to explicitly granted Desktop operations and native capabilities; carries connection and route state without exposing transport credentials | +| Desktop-served React Native Web application | Workspace list and creation; sessions and terminal presentation; files, previews, diffs, source control, reviews, tasks, accounts, browser presentation, Agent History, native-chat presentation, and the settings, troubleshooting and connection-log screens | +| Desktop runtime | Builds and ships the matching web package; serves its manifest and chunks through authenticated RPC; reauthorizes every workspace mutation; enforces host, workspace, provider, path, and resource limits | +| Typed native bridge | Connects the unprivileged page to explicitly granted Desktop operations and native capabilities; carries connection and route state without exposing transport credentials | The page never receives the raw RPC client, pairing credential, host endpoint, private key, cache path, or unrestricted native module access. Native-owned @@ -209,8 +209,21 @@ edges still meet the device and keep their measured values. its host-work slot until the host call settles. The Desktop is trusted, so the page addresses host tabs, browser pages and provider sessions by their host ids. Generic subscriptions, native-chat domain actions, file reads, Source - Control reads/watch, session snapshot/feed/actions and terminal metadata use - this path. + Control reads/watch, session snapshot/feed/actions, terminal metadata and + browser input/navigation/screencast use this path. +- Browser control is desktop-owned end to end. `mobileWeb.browser.navigate`, + `.history`, `.pointer`, `.keyboard`, `.dialog` and the `.subscribe`/ + `.unsubscribe` screencast stream are the page's whole surface. The wrapper is + where the worktree is scoped, the tab URL is stripped + (`browser-url-privacy.ts`), the title and dialog text are bounded, the + press/release fallback for a failed plain click runs, and pointer and keyboard + traffic meet a per-connection token bucket + (`mobile-web-browser-input-rate-limit.ts`). The generic lane carries JSON only, + so a binary screencast frame crosses it as a run of base64 `frameChunk` + events the page reassembles. The stream opens with a bare `ready` carrying the + cancel id; the browser's own `ready` is the one that carries tab state. The + released native app keeps calling the raw `browser.*` methods and + `browser.screencast` unchanged. - Decisions behind the generic lane and its 2026-09-07 simplification are in [`plans/2026-09-07-long-lived-mobile-shell-decisions.md`](./plans/2026-09-07-long-lived-mobile-shell-decisions.md). Unmigrated domain operations keep their current adapters until moved. diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts b/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts index c51560fe59a..e1e6ef12143 100644 --- a/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts +++ b/src/main/runtime/rpc/methods/mobile-web-browser-stream.test.ts @@ -136,7 +136,7 @@ describe('host-owned browser screencast', () => { await f.done }) - it('reports an undisplayable frame instead of dropping the stream', async () => { + it('ends the stream on a frame the page contract cannot describe', async () => { const f = fixture() await f.started @@ -153,10 +153,10 @@ describe('host-owned browser screencast', () => { expect(f.events.at(-1)).toEqual({ type: 'error', - message: 'Browser frame is too large to display safely.' + message: 'Browser frame cannot be displayed safely.' }) - f.getHost().signal?.dispatchEvent(new Event('abort')) await f.done + expect(f.events.filter((event) => (event as { type: string }).type === 'end')).toEqual([]) }) it('ends the page stream when the host cleanup for this connection runs', async () => { diff --git a/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts b/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts index 194ded6bc12..63a4cc224e3 100644 --- a/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts +++ b/src/main/runtime/rpc/methods/mobile-web-browser-stream.ts @@ -66,8 +66,10 @@ export const MOBILE_WEB_BROWSER_STREAM_METHODS = [ return true } const chunks = mobileWebBrowserFrameChunks(frame) + // An error retires the stream on this lane, which is the honest end for a producer whose + // frames the page contract cannot describe at all. if (!chunks) { - emit({ type: 'error', message: 'Browser frame is too large to display safely.' }) + close({ type: 'error', message: 'Browser frame cannot be displayed safely.' }) return true } for (const chunk of chunks) {