refactor(mobile): move browser input and the screencast onto the host lane

The page addresses the browser page id directly through requestMobileWebHost and
parses each result with the shared contract, so no browser translator remains
between it and the desktop.

Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb
This commit is contained in:
Jinwoo-H
2026-09-07 15:07:41 -04:00
parent f9b5275683
commit f6d32ea7cf
17 changed files with 382 additions and 754 deletions
@@ -1,27 +0,0 @@
import { describe, expect, it } from 'vitest'
import { sanitizeMobileWebBrowserEvent } from './mobile-web-browser-event-sanitizer'
describe('sanitizeMobileWebBrowserEvent', () => {
it('removes URL credentials from browser events', () => {
const event = sanitizeMobileWebBrowserEvent({
type: 'navigation',
tab: {
url: 'https://user:password@example.com/callback?access_token=secret&tab=review',
title: 'Review',
canGoBack: true,
canGoForward: false
}
})
expect(event).toEqual({
type: 'navigation',
tab: {
url: 'https://example.com/callback?tab=review',
title: 'Review',
canGoBack: true,
canGoForward: false
}
})
expect(JSON.stringify(event)).not.toMatch(/password|access_token|secret/)
})
})
@@ -1,49 +0,0 @@
import {
MobileWebBrowserEventSchema,
type MobileWebBrowserEvent
} from '../../../src/shared/mobile-web/browser-operation-contract'
import { mobileWebPageBrowserUrl } from '../../../src/shared/mobile-web/browser-url-privacy'
export function sanitizeMobileWebBrowserEvent(value: unknown): MobileWebBrowserEvent | null {
if (!isRecord(value)) {
return null
}
if (value.type === 'ready' || value.type === 'navigation') {
const tab = isRecord(value.tab) ? value.tab : {}
return MobileWebBrowserEventSchema.parse({
type: value.type,
tab: {
url: mobileWebPageBrowserUrl(tab.url),
title: boundedText(tab.title, 240, ''),
canGoBack: tab.canGoBack === true,
canGoForward: tab.canGoForward === true
}
})
}
if (value.type === 'end' || value.type === 'dialogClosed') {
return { type: value.type }
}
if (value.type === 'dialog') {
return MobileWebBrowserEventSchema.parse({
type: 'dialog',
dialogType: isDialogType(value.dialogType) ? value.dialogType : 'alert',
message: boundedText(value.message, 8192, 'Browser dialog')
})
}
if (value.type === 'error') {
return { type: 'error', message: 'Browser stream failed.' }
}
return null
}
function boundedText(value: unknown, maximum: number, fallback: string): string {
return typeof value === 'string' ? value.slice(0, maximum) : fallback
}
function isDialogType(value: unknown): value is 'alert' | 'confirm' | 'prompt' | 'beforeunload' {
return value === 'alert' || value === 'confirm' || value === 'prompt' || value === 'beforeunload'
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -1,108 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import type { RpcClient } from '../transport/rpc-client'
import { executeMobileWebBrowserOperation } from './mobile-web-browser-operations'
import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
describe('mobile web browser operations', () => {
it('resolves opaque workspace authority and forwards the host page id for navigation', async () => {
const { workspaceAuthority, workspaceId, pageId } = authorities()
const sendRequest = vi.fn<RpcClient['sendRequest']>().mockResolvedValue({
ok: true,
result: { url: 'https://example.com/', title: 'Example', rawPageId: 'raw-page' }
})
await expect(
executeMobileWebBrowserOperation({
operation: 'navigate',
payload: { workspaceId, pageId, url: 'https://example.com' },
client: { sendRequest } as unknown as RpcClient,
workspaceAuthority
})
).resolves.toEqual({ url: 'https://example.com/' })
expect(sendRequest).toHaveBeenCalledWith(
'browser.goto',
{
worktree: 'id:host-workspace',
page: 'raw-page',
url: 'https://example.com'
},
{ timeoutMs: 30_000 }
)
})
it('removes credentials from the authoritative navigation result', async () => {
const { workspaceAuthority, workspaceId, pageId } = authorities()
const sendRequest = vi.fn<RpcClient['sendRequest']>().mockResolvedValue({
ok: true,
result: {
url: 'https://user:password@example.com/callback?code=secret&tab=review#access_token=secret'
}
})
await expect(
executeMobileWebBrowserOperation({
operation: 'navigate',
payload: { workspaceId, pageId, url: 'https://example.com' },
client: { sendRequest } as unknown as RpcClient,
workspaceAuthority
})
).resolves.toEqual({ url: 'https://example.com/callback?tab=review' })
})
it('keeps pointer fallback native and refuses an unknown page workspace', async () => {
const { workspaceAuthority, workspaceId, pageId } = authorities()
const sendRequest = vi
.fn<RpcClient['sendRequest']>()
.mockResolvedValueOnce({ ok: false, error: { code: 'unsupported', message: 'unsupported' } })
.mockResolvedValue({ ok: true, result: null })
const client = { sendRequest } as unknown as RpcClient
await executeMobileWebBrowserOperation({
operation: 'pointer',
payload: {
workspaceId,
pageId,
action: 'click',
x: 20,
y: 30,
button: 'left',
modifiers: []
},
client,
workspaceAuthority
})
expect(sendRequest.mock.calls.map(([method]) => method)).toEqual([
'browser.mouseClick',
'browser.mouseMove',
'browser.mouseDown',
'browser.mouseUp'
])
workspaceAuthority.synchronize([])
await expect(
executeMobileWebBrowserOperation({
operation: 'reload',
payload: { workspaceId, pageId },
client,
workspaceAuthority
})
).rejects.toMatchObject({ code: 'not_found' })
})
})
function authorities(): {
workspaceAuthority: MobileWebWorkspaceAuthority
workspaceId: string
pageId: string
} {
const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(3)
const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes)
workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }])
return {
workspaceAuthority,
workspaceId: workspaceAuthority.pageWorkspaceId('host-workspace'),
pageId: 'raw-page'
}
}
@@ -1,160 +0,0 @@
import {
MobileWebBrowserCommandResultSchema,
MobileWebBrowserDialogPayloadSchema,
MobileWebBrowserKeyboardPayloadSchema,
MobileWebBrowserNavigatePayloadSchema,
MobileWebBrowserNavigateResultSchema,
MobileWebBrowserPointerPayloadSchema,
MobileWebBrowserTargetPayloadSchema
} from '../../../src/shared/mobile-web/browser-operation-contract'
import { mobileWebPageBrowserUrl } from '../../../src/shared/mobile-web/browser-url-privacy'
import type { RpcClient } from '../transport/rpc-client'
import { MobileWebBrokerError, mobileWebBrokerHostRpcError } from './mobile-web-broker-error'
import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
export async function executeMobileWebBrowserOperation(args: {
operation: string
payload: unknown
client: RpcClient
workspaceAuthority: MobileWebWorkspaceAuthority
}): Promise<unknown> {
if (args.operation === 'navigate') {
const payload = MobileWebBrowserNavigatePayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
const response = await args.client.sendRequest(
'browser.goto',
{ ...target, url: payload.url },
{ timeoutMs: 30_000 }
)
const result = requireResult(response)
const parsed = MobileWebBrowserNavigateResultSchema.safeParse({
url: isRecord(result) ? mobileWebPageBrowserUrl(result.url) : 'about:blank'
})
if (!parsed.success) {
throw new MobileWebBrokerError('host_error')
}
return parsed.data
}
if (args.operation === 'pointer') {
const payload = MobileWebBrowserPointerPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
if (payload.action === 'scroll') {
await requireRequest(args.client, 'browser.mouseMove', {
...target,
x: payload.x,
y: payload.y
})
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseWheel', {
...target,
dx: payload.dx,
dy: payload.dy
})
return MobileWebBrowserCommandResultSchema.parse(null)
}
const click = await args.client.sendRequest(
'browser.mouseClick',
{
...target,
x: payload.x,
y: payload.y,
button: payload.button,
modifiers: payload.modifiers,
...(payload.radius === undefined ? {} : { radius: payload.radius })
},
{ timeoutMs: 5_000 }
)
if (!click.ok && payload.modifiers.length === 0) {
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseMove', {
...target,
x: payload.x,
y: payload.y
})
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseDown', {
...target,
button: payload.button
})
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseUp', {
...target,
button: payload.button
})
}
return MobileWebBrowserCommandResultSchema.parse(null)
}
if (args.operation === 'keyboard') {
const payload = MobileWebBrowserKeyboardPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
await requireRequest(
args.client,
payload.action === 'insertText' ? 'browser.keyboardInsertText' : 'browser.keypress',
payload.action === 'insertText'
? { ...target, text: payload.text }
: { ...target, key: payload.key },
5_000
)
return MobileWebBrowserCommandResultSchema.parse(null)
}
if (args.operation === 'dialog') {
const payload = MobileWebBrowserDialogPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
await requireRequest(
args.client,
payload.action === 'accept' ? 'browser.dialogAccept' : 'browser.dialogDismiss',
target,
5_000
)
return MobileWebBrowserCommandResultSchema.parse(null)
}
if (args.operation === 'back' || args.operation === 'forward' || args.operation === 'reload') {
const payload = MobileWebBrowserTargetPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
await requireRequest(args.client, `browser.${args.operation}`, target)
return MobileWebBrowserCommandResultSchema.parse(null)
}
throw new MobileWebBrokerError('unsupported_capability')
}
// Re-reads the workspace binding mid-operation: a workspace switch must not land later frames.
function assertTarget(
payload: { workspaceId: string; pageId: string },
expected: { worktree: string; page: string },
workspaceAuthority: MobileWebWorkspaceAuthority
): void {
const current = resolveTarget(payload, workspaceAuthority)
if (current.worktree !== expected.worktree || current.page !== expected.page) {
throw new MobileWebBrokerError('conflict')
}
}
function resolveTarget(
payload: { workspaceId: string; pageId: string },
workspaceAuthority: MobileWebWorkspaceAuthority
): { worktree: string; page: string } {
return {
worktree: `id:${workspaceAuthority.hostWorkspaceId(payload.workspaceId)}`,
page: payload.pageId
}
}
async function requireRequest(
client: RpcClient,
method: string,
payload: unknown,
timeoutMs = 15_000
): Promise<void> {
requireResult(await client.sendRequest(method, payload, { timeoutMs }))
}
function requireResult(response: Awaited<ReturnType<RpcClient['sendRequest']>>): unknown {
if (!response.ok) {
throw mobileWebBrokerHostRpcError(response.error)
}
return response.result
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -1,126 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import {
BrowserScreencastOpcode,
type BrowserScreencastFrame
} from '../transport/browser-screencast-protocol'
import type { RpcClient } from '../transport/rpc-client'
import { MobileWebBrowserStreams } from './mobile-web-browser-streams'
import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
describe('mobile web browser streams', () => {
it('chunks frames below the bridge limit and cleans up', async () => {
const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(4)
const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes)
workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }])
const workspaceId = workspaceAuthority.pageWorkspaceId('host-workspace')
const pageId = 'raw-page'
const postEvent = vi.fn(async () => {})
let onEvent: ((event: unknown) => void) | undefined
let onFrame: ((frame: BrowserScreencastFrame) => void) | undefined
const unsubscribe = vi.fn()
const subscribe = vi
.fn<RpcClient['subscribe']>()
.mockImplementation((_method, _payload, eventListener, options) => {
onEvent = eventListener
onFrame = options?.onBinaryFrame
return unsubscribe
})
const streams = new MobileWebBrowserStreams({
isActive: () => true,
workspaceAuthority,
postEvent,
postClosed: vi.fn()
})
streams.start({
requestId: 'request-1',
subscriptionId: 'subscription-1',
payload: {
workspaceId,
pageId,
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
},
client: { subscribe } as unknown as RpcClient
})
expect(subscribe).toHaveBeenCalledWith(
'browser.screencast',
{
worktree: 'id:host-workspace',
page: 'raw-page',
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
},
expect.any(Function),
{ onBinaryFrame: expect.any(Function) }
)
onEvent?.({
type: 'ready',
browserPageId: 'raw-page',
tab: {
url: 'https://example.com',
title: 'Example',
canGoBack: true,
canGoForward: false,
rawSecret: 'must-not-cross'
}
})
onEvent?.({
type: 'navigation',
tab: {
url: 'https://www.iana.org/help/example-domains',
title: 'IANA-managed Reserved Domains',
canGoBack: true,
canGoForward: false
}
})
const image = new Uint8Array(200_000).map((_, index) => index % 251)
onFrame?.({
opcode: BrowserScreencastOpcode.Frame,
seq: 7,
format: 'jpeg',
metadata: { deviceWidth: 800, deviceHeight: 600 },
image
})
await vi.waitFor(() => expect(postEvent).toHaveBeenCalledTimes(4))
expect(postEvent.mock.calls.map(([, sequence]) => sequence)).toEqual([0, 1, 2, 3])
expect(postEvent.mock.calls[0]?.[2]).toEqual({
type: 'ready',
tab: {
url: 'https://example.com',
title: 'Example',
canGoBack: true,
canGoForward: false
}
})
expect(postEvent.mock.calls[1]?.[2]).toEqual({
type: 'navigation',
tab: {
url: 'https://www.iana.org/help/example-domains',
title: 'IANA-managed Reserved Domains',
canGoBack: true,
canGoForward: false
}
})
const chunks = postEvent.mock.calls.slice(2).map((call) => call[2])
expect(chunks).toMatchObject([
{ type: 'frameChunk', frameSequence: 7, chunkIndex: 0, chunkCount: 2 },
{ type: 'frameChunk', frameSequence: 7, chunkIndex: 1, chunkCount: 2 }
])
expect(JSON.stringify(chunks)).not.toContain('raw-page')
expect(streams.cancel('subscription-1')).toBe('request-1')
expect(unsubscribe).toHaveBeenCalledOnce()
})
})
@@ -1,189 +0,0 @@
import {
MobileWebSubscriptionLedger,
type MobileWebSubscriptionLedgerConfig,
type MobileWebSubscriptionRecord
} from './mobile-web-subscription-ledger'
import { Buffer } from 'buffer/'
import {
MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES,
MOBILE_WEB_BROWSER_FRAME_MAX_IMAGE_BYTES,
MobileWebBrowserEventSchema,
MobileWebBrowserStreamPayloadSchema,
type MobileWebBrowserEvent
} from '../../../src/shared/mobile-web/browser-operation-contract'
import type {
BrowserScreencastFrame,
BrowserScreencastFrameMetadata
} from '../transport/browser-screencast-protocol'
import type { RpcClient } from '../transport/rpc-client'
import { sanitizeMobileWebBrowserEvent } from './mobile-web-browser-event-sanitizer'
import { MobileWebBrokerError } from './mobile-web-broker-error'
import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
type ScreencastRecord = MobileWebSubscriptionRecord & {
frameQueued: boolean
pendingFrame: BrowserScreencastFrame | null
}
type BrowserLedgerConfig = MobileWebSubscriptionLedgerConfig<MobileWebBrowserEvent> & {
workspaceAuthority: MobileWebWorkspaceAuthority
}
export class MobileWebBrowserStreams extends MobileWebSubscriptionLedger<
MobileWebBrowserEvent,
ScreencastRecord
> {
constructor(private readonly config: BrowserLedgerConfig) {
super({ ...config, operationKey: 'browser.subscribe' })
}
start(args: {
requestId: string
subscriptionId: string
payload: unknown
client: RpcClient
}): void {
this.admit(args.subscriptionId)
const payload = MobileWebBrowserStreamPayloadSchema.parse(args.payload)
const hostWorkspaceId = this.config.workspaceAuthority.hostWorkspaceId(payload.workspaceId)
const record: ScreencastRecord = {
...this.newRecord(args.requestId),
frameQueued: false,
pendingFrame: null
}
this.open(args.subscriptionId, record, () =>
args.client.subscribe(
'browser.screencast',
{
worktree: `id:${hostWorkspaceId}`,
page: payload.pageId,
format: payload.format,
quality: payload.quality,
maxWidth: payload.maxWidth,
maxHeight: payload.maxHeight,
everyNthFrame: payload.everyNthFrame,
minFrameIntervalMs: payload.minFrameIntervalMs,
...(payload.viewportWidth === undefined ? {} : { viewportWidth: payload.viewportWidth }),
...(payload.viewportHeight === undefined
? {}
: { viewportHeight: payload.viewportHeight }),
...(payload.deviceScaleFactor === undefined
? {}
: { deviceScaleFactor: payload.deviceScaleFactor }),
...(payload.mobile === undefined ? {} : { mobile: payload.mobile })
},
(event) => this.receiveEvent(args.subscriptionId, record, event),
{
onBinaryFrame: (frame) => this.receiveFrame(args.subscriptionId, record, frame)
}
)
)
}
// Drops the parked image so a retired stream cannot pin a multi-megabyte frame.
protected override retire(record: ScreencastRecord): void {
record.pendingFrame = null
}
private receiveEvent(subscriptionId: string, record: ScreencastRecord, value: unknown): void {
if (!this.isCurrent(subscriptionId, record)) {
return
}
const event = sanitizeMobileWebBrowserEvent(value)
if (event) {
this.enqueueTask(subscriptionId, record, () => this.deliver(subscriptionId, record, event))
}
}
private receiveFrame(
subscriptionId: string,
record: ScreencastRecord,
frame: BrowserScreencastFrame
): void {
if (!this.isCurrent(subscriptionId, record)) {
return
}
if (frame.image.byteLength > MOBILE_WEB_BROWSER_FRAME_MAX_IMAGE_BYTES) {
this.enqueueTask(subscriptionId, record, () =>
this.deliver(subscriptionId, record, {
type: 'error',
message: 'Browser frame is too large to display safely.'
})
)
return
}
record.pendingFrame = frame
if (record.frameQueued) {
return
}
record.frameQueued = true
this.enqueueTask(subscriptionId, record, async () => {
const latest = record.pendingFrame
record.pendingFrame = null
if (latest) {
await this.deliverFrame(subscriptionId, record, latest)
}
record.frameQueued = false
const pending = record.pendingFrame
if (pending && this.isCurrent(subscriptionId, record)) {
record.pendingFrame = null
this.receiveFrame(subscriptionId, record, pending)
}
})
}
private async deliverFrame(
subscriptionId: string,
record: ScreencastRecord,
frame: BrowserScreencastFrame
): Promise<void> {
const chunkCount = Math.ceil(frame.image.byteLength / MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES)
for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex += 1) {
if (!this.isCurrent(subscriptionId, record)) {
return
}
const start = chunkIndex * MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES
const end = Math.min(frame.image.byteLength, start + MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES)
await this.deliver(subscriptionId, record, {
type: 'frameChunk',
frameSequence: frame.seq,
format: frame.format,
metadata: boundedMetadata(frame.metadata),
imageBytes: frame.image.byteLength,
chunkIndex,
chunkCount,
data: Buffer.from(frame.image.subarray(start, end)).toString('base64')
})
}
}
// Why the sequence is claimed here and not at enqueue time: one queued frame task posts a whole
// chunk run, so the numbers must be handed out per post, in post order.
private async deliver(
subscriptionId: string,
record: ScreencastRecord,
value: MobileWebBrowserEvent
): Promise<void> {
const event = MobileWebBrowserEventSchema.parse(value)
const sequence = record.sequence
record.sequence += 1
await this.options.postEvent(subscriptionId, sequence, event)
}
}
function boundedMetadata(metadata: BrowserScreencastFrameMetadata): BrowserScreencastFrameMetadata {
const parsed = MobileWebBrowserEventSchema.parse({
type: 'frameChunk',
frameSequence: 0,
format: 'jpeg',
metadata,
imageBytes: 1,
chunkIndex: 0,
chunkCount: 1,
data: 'AA=='
})
if (parsed.type !== 'frameChunk') {
throw new MobileWebBrokerError('host_error')
}
return parsed.metadata
}
@@ -1,12 +0,0 @@
import { capabilityGrants, grantLimits } from './mobile-web-production-grant-table'
export const MOBILE_WEB_PRODUCTION_BROWSER_GRANTS = capabilityGrants('browser', {
subscribe: grantLimits(4 * 1024, 1 * 1024, 1, 4, 1),
navigate: grantLimits(8 * 1024, 8 * 1024, 1, 8, 2),
back: grantLimits(2 * 1024, 256, 2, 12, 4),
forward: grantLimits(2 * 1024, 256, 2, 12, 4),
reload: grantLimits(2 * 1024, 256, 2, 12, 4),
dialog: grantLimits(2 * 1024, 256, 2, 12, 4),
pointer: grantLimits(4 * 1024, 256, 2, 40, 20),
keyboard: grantLimits(40 * 1024, 256, 2, 20, 10)
})
@@ -138,7 +138,7 @@ describe('host-owned browser input', () => {
let rejected = 0
for (let attempt = 0; attempt < 60; attempt += 1) {
await pointer.handler(move, f.context).catch(() => {
await Promise.resolve(pointer.handler(move, f.context)).catch(() => {
rejected += 1
})
}
@@ -3,13 +3,14 @@ import {
BrowserScreencastOpcode,
encodeBrowserScreencastFrame
} from '../../../../shared/browser-screencast-protocol'
import type { RpcContext } from '../core'
import { isStreamingMethod, type RpcContext, type RpcMethod } from '../core'
import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist'
import { MOBILE_WEB_BROWSER_STREAM_METHODS } from './mobile-web-browser-stream'
const methods = new Map(MOBILE_WEB_BROWSER_STREAM_METHODS.map((method) => [method.name, method]))
const subscribe = methods.get('mobileWeb.browser.subscribe')!
const unsubscribe = methods.get('mobileWeb.browser.unsubscribe')!
const subscribe = MOBILE_WEB_BROWSER_STREAM_METHODS.find(isStreamingMethod)!
const unsubscribe = MOBILE_WEB_BROWSER_STREAM_METHODS.find(
(method) => !isStreamingMethod(method)
) as RpcMethod
const REQUEST = {
worktree: 'id:workspace',
@@ -25,6 +25,7 @@ import type {
} from '../../shared/mobile-web/browser-operation-contract'
import type { MobileWebBrowserRequestClient } from './mobile-web-browser-request-client'
import { mobileWebBrowserNavigationClientBindings } from './mobile-web-browser-navigation-client-bindings'
import { subscribeMobileWebHostBrowser } from './mobile-web-host-browser-subscription'
import { MobileWebAccountRequestClient } from './mobile-web-account-request-client'
import { MobileWebAgentHistoryRequestClient } from './mobile-web-agent-history-request-client'
import { mobileWebFileClientBindings } from './mobile-web-file-client-bindings'
@@ -258,7 +259,7 @@ export class MobileWebBridgeClient {
onEvent: (event: MobileWebBrowserEvent) => void,
onError: (error: MobileWebBridgeClientError) => void
): MobileWebBridgeSubscription {
return this.subscriptions.subscribeBrowser(payload, onEvent, onError)
return subscribeMobileWebHostBrowser(this.subscriptions, payload, onEvent, onError)
}
receive(message: MobileWebBridgeShellMessage): void {
@@ -19,7 +19,6 @@ import { deliverMobileWebSubscriptionEvent } from './mobile-web-bridge-subscript
import {
accountSubscriptionSetup,
terminalSubscriptionSetup,
browserSubscriptionSetup,
speechSubscriptionSetup,
workspaceSubscriptionSetup,
type MobileWebBridgeSubscriptionSetup
@@ -94,10 +93,6 @@ export class MobileWebBridgeSubscriptionClient {
return this.subscribeWith(hostSubscriptionSetup(...args))
}
subscribeBrowser(...args: Parameters<typeof browserSubscriptionSetup>) {
return this.subscribeWith(browserSubscriptionSetup(...args))
}
subscribeSpeech(...args: Parameters<typeof speechSubscriptionSetup>) {
return this.subscribeWith(speechSubscriptionSetup(...args))
}
@@ -17,12 +17,6 @@ import {
type MobileWebWorkspaceChange
} from '../../shared/mobile-web/bridge-operation-contract'
import type { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import {
MobileWebBrowserEventSchema,
MobileWebBrowserStreamPayloadSchema,
type MobileWebBrowserEvent,
type MobileWebBrowserStreamPayload
} from '../../shared/mobile-web/browser-operation-contract'
import {
MobileWebSpeechEventSchema,
MobileWebSpeechSubscribePayloadSchema,
@@ -67,21 +61,6 @@ export function speechSubscriptionSetup(
}
}
export function browserSubscriptionSetup(
payload: MobileWebBrowserStreamPayload,
onEvent: (event: MobileWebBrowserEvent) => void,
onError: (error: MobileWebBridgeClientError) => void
): MobileWebBridgeSubscriptionSetup {
return {
capability: 'browser',
payload,
payloadSchema: MobileWebBrowserStreamPayloadSchema,
eventSchema: MobileWebBrowserEventSchema,
onEvent: (value) => onEvent(value as MobileWebBrowserEvent),
onError
}
}
export function workspaceSubscriptionSetup(
onEvent: (event: MobileWebWorkspaceChange) => void,
onError: (error: MobileWebBridgeClientError) => void
@@ -0,0 +1,137 @@
import { describe, expect, it, vi } from 'vitest'
import { MobileWebBrowserRequestClient } from './mobile-web-browser-request-client'
import type { MobileWebOneShotRequestClient } from './mobile-web-one-shot-request-client'
const TARGET = { workspaceId: 'workspace-1', pageId: 'browser-1' } as const
function fixture(result: unknown = { applied: true }) {
const request = vi.fn(async (..._args: unknown[]) => result)
const requests = { supports: () => true, request } as unknown as MobileWebOneShotRequestClient
return { request, client: new MobileWebBrowserRequestClient(requests) }
}
function payloads(
request: ReturnType<typeof fixture>['request']
): { method: string; workspaceId?: string; params: Record<string, unknown> }[] {
return request.mock.calls.map(
(call) => call[2] as { method: string; workspaceId?: string; params: Record<string, unknown> }
)
}
describe('page-owned browser commands', () => {
it('names the workspace in the envelope and the page in the params', async () => {
const f = fixture({ url: 'https://app.example/' })
await expect(f.client.navigate({ ...TARGET, url: 'https://app.example/' })).resolves.toEqual({
url: 'https://app.example/'
})
expect(payloads(f.request)).toEqual([
{
method: 'mobileWeb.browser.navigate',
workspaceId: 'workspace-1',
params: { page: 'browser-1', url: 'https://app.example/' }
}
])
})
it('sends one host request per pointer, keyboard and dialog action', async () => {
const f = fixture()
await expect(
f.client.pointer({ ...TARGET, action: 'scroll', x: 1, y: 2, dx: 0, dy: -30 })
).resolves.toBeNull()
await f.client.pointer({
...TARGET,
action: 'click',
x: 3,
y: 4,
button: 'left',
modifiers: ['cmd'],
radius: 8
})
await f.client.keyboard({ ...TARGET, action: 'insertText', text: 'hi' })
await f.client.keyboard({ ...TARGET, action: 'keypress', key: 'Escape' })
await f.client.dialog({ ...TARGET, action: 'accept' })
expect(payloads(f.request)).toEqual([
{
method: 'mobileWeb.browser.pointer',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'scroll', x: 1, y: 2, dx: 0, dy: -30 }
},
{
method: 'mobileWeb.browser.pointer',
workspaceId: 'workspace-1',
params: {
page: 'browser-1',
action: 'click',
x: 3,
y: 4,
button: 'left',
modifiers: ['cmd'],
radius: 8
}
},
{
method: 'mobileWeb.browser.keyboard',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'insertText', text: 'hi' }
},
{
method: 'mobileWeb.browser.keyboard',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'keypress', key: 'Escape' }
},
{
method: 'mobileWeb.browser.dialog',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'accept' }
}
])
})
it('carries back, forward and reload as one history method', async () => {
const f = fixture()
await f.client.back(TARGET)
await f.client.forward(TARGET)
await f.client.reload(TARGET)
expect(payloads(f.request)).toEqual(
['back', 'forward', 'reload'].map((action) => ({
method: 'mobileWeb.browser.history',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action }
}))
)
})
it('rejects a command the host did not acknowledge as applied', async () => {
const f = fixture({ applied: false })
await expect(f.client.reload(TARGET)).rejects.toMatchObject({ code: 'invalid_message' })
})
it('rejects a navigation result with no URL and tolerates an unknown field', async () => {
const f = fixture({ title: 'App' })
await expect(f.client.navigate({ ...TARGET, url: 'https://a.example/' })).rejects.toMatchObject(
{
code: 'invalid_message'
}
)
const forwardCompatible = fixture({ url: 'https://a.example/', title: 'App' })
await expect(
forwardCompatible.client.navigate({ ...TARGET, url: 'https://a.example/' })
).resolves.toEqual({ url: 'https://a.example/' })
})
it('never forwards the opaque workspace handle inside the host params', async () => {
const f = fixture()
await f.client.dialog({ ...TARGET, action: 'dismiss' })
expect(JSON.stringify(payloads(f.request)[0]!.params)).not.toContain('workspace-1')
})
})
@@ -1,32 +1,39 @@
import type { z } from 'zod'
import {
MobileWebBrowserCommandResultSchema,
MobileWebBrowserDialogPayloadSchema,
MobileWebBrowserKeyboardPayloadSchema,
MobileWebBrowserNavigatePayloadSchema,
MobileWebBrowserAckSchema,
MobileWebBrowserNavigateResultSchema,
MobileWebBrowserPointerPayloadSchema,
MobileWebBrowserTargetPayloadSchema,
type MobileWebBrowserDialogPayload,
type MobileWebBrowserKeyboardPayload,
type MobileWebBrowserNavigatePayload,
type MobileWebBrowserPointerPayload,
type MobileWebBrowserTargetPayload
} from '../../shared/mobile-web/browser-operation-contract'
import { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import type { MobileWebBridgeRequestOptions } from './mobile-web-bridge-request-state'
import { requestMobileWebHost } from './mobile-web-host-request-client'
import type { MobileWebOneShotRequestClient } from './mobile-web-one-shot-request-client'
/** The workspace handle rides in the host request envelope, and the browser page id is the only
* target field the desktop wrapper reads. */
function hostParams<TPayload extends MobileWebBrowserTargetPayload>({
workspaceId: _workspaceId,
pageId,
...fields
}: TPayload): Record<string, unknown> {
return { page: pageId, ...fields }
}
export class MobileWebBrowserRequestClient {
constructor(private readonly requests: MobileWebOneShotRequestClient) {}
navigate(
async navigate(
payload: MobileWebBrowserNavigatePayload,
options?: MobileWebBridgeRequestOptions
): Promise<{ url: string }> {
return this.requests.request(
'browser',
'navigate',
return this.send(
'mobileWeb.browser.navigate',
payload,
MobileWebBrowserNavigatePayloadSchema,
hostParams(payload),
MobileWebBrowserNavigateResultSchema,
options
)
@@ -36,77 +43,85 @@ export class MobileWebBrowserRequestClient {
payload: MobileWebBrowserPointerPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
'pointer',
payload,
MobileWebBrowserPointerPayloadSchema,
MobileWebBrowserCommandResultSchema,
options
)
return this.command('mobileWeb.browser.pointer', payload, hostParams(payload), options)
}
keyboard(
payload: MobileWebBrowserKeyboardPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
'keyboard',
payload,
MobileWebBrowserKeyboardPayloadSchema,
MobileWebBrowserCommandResultSchema,
options
)
return this.command('mobileWeb.browser.keyboard', payload, hostParams(payload), options)
}
dialog(
payload: MobileWebBrowserDialogPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
'dialog',
payload,
MobileWebBrowserDialogPayloadSchema,
MobileWebBrowserCommandResultSchema,
options
)
return this.command('mobileWeb.browser.dialog', payload, hostParams(payload), options)
}
back(
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.command('back', payload, options)
return this.history('back', payload, options)
}
forward(
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.command('forward', payload, options)
return this.history('forward', payload, options)
}
reload(
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.command('reload', payload, options)
return this.history('reload', payload, options)
}
private command(
operation: 'back' | 'forward' | 'reload',
private history(
action: 'back' | 'forward' | 'reload',
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
operation,
return this.command(
'mobileWeb.browser.history',
payload,
MobileWebBrowserTargetPayloadSchema,
MobileWebBrowserCommandResultSchema,
{ ...hostParams(payload), action },
options
)
}
private async command(
method: string,
payload: MobileWebBrowserTargetPayload,
params: Record<string, unknown>,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
await this.send(method, payload, params, MobileWebBrowserAckSchema, options)
return null
}
private async send<TResult>(
method: string,
payload: MobileWebBrowserTargetPayload,
params: Record<string, unknown>,
schema: z.ZodType<TResult>,
options?: MobileWebBridgeRequestOptions
): Promise<TResult> {
const result = await requestMobileWebHost(
this.requests,
method,
payload.workspaceId,
params,
options
)
const parsed = schema.safeParse(result)
if (!parsed.success) {
throw new MobileWebBridgeClientError('invalid_message', false)
}
return parsed.data
}
}
@@ -0,0 +1,100 @@
import { describe, expect, it, vi } from 'vitest'
import type { MobileWebBrowserEvent } from '../../shared/mobile-web/browser-operation-contract'
import type { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import type { MobileWebBridgeSubscriptionClient } from './mobile-web-bridge-subscription-client'
import { subscribeMobileWebHostBrowser } from './mobile-web-host-browser-subscription'
const PAYLOAD = {
workspaceId: 'workspace-1',
pageId: 'browser-1',
format: 'jpeg' as const,
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
}
function fixture() {
const unsubscribe = vi.fn()
let emit: (event: unknown) => void = () => {}
const subscribeHost = vi.fn((_payload: unknown, onEvent: (event: unknown) => void) => {
emit = onEvent
return { ready: Promise.resolve(), unsubscribe }
})
const subscriptions = { subscribeHost } as unknown as MobileWebBridgeSubscriptionClient
const events: MobileWebBrowserEvent[] = []
const errors: MobileWebBridgeClientError[] = []
const subscription = subscribeMobileWebHostBrowser(
subscriptions,
PAYLOAD,
(event) => events.push(event),
(error) => errors.push(error)
)
return { emit: (event: unknown) => emit(event), errors, events, subscribeHost, subscription }
}
describe('page-owned browser stream', () => {
it('opens the host stream with the page id and the screencast request', () => {
const f = fixture()
expect(f.subscribeHost.mock.calls[0]![0]).toEqual({
method: 'mobileWeb.browser.subscribe',
workspaceId: 'workspace-1',
params: {
page: 'browser-1',
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
}
})
})
it('swallows the lane handshake and delivers the browser ready that carries tab state', () => {
const f = fixture()
const tab = { url: 'https://a.example/', title: 'A', canGoBack: false, canGoForward: false }
f.emit({ type: 'ready', subscriptionId: 'lane-id' })
f.emit({ type: 'ready', tab })
expect(f.events).toEqual([{ type: 'ready', tab }])
expect(f.errors).toEqual([])
})
it('reports an event the browser contract cannot describe', () => {
const f = fixture()
f.emit({ type: 'frameChunk', chunkIndex: 4, chunkCount: 1 })
expect(f.events).toEqual([])
expect(f.errors.map((error) => error.code)).toEqual(['invalid_message'])
})
it('stops delivering once the page unsubscribes', () => {
const f = fixture()
f.subscription.unsubscribe()
f.emit({ type: 'dialogClosed' })
expect(f.events).toEqual([])
})
it('refuses a screencast request the contract does not admit', async () => {
const errors: MobileWebBridgeClientError[] = []
const subscribeHost = vi.fn()
const subscription = subscribeMobileWebHostBrowser(
{ subscribeHost } as unknown as MobileWebBridgeSubscriptionClient,
{ ...PAYLOAD, quality: 0 },
() => {},
(error) => errors.push(error)
)
await expect(subscription.ready).rejects.toMatchObject({ code: 'invalid_request' })
await Promise.resolve()
expect(subscribeHost).not.toHaveBeenCalled()
expect(errors.map((error) => error.code)).toEqual(['invalid_request'])
})
})
@@ -0,0 +1,70 @@
import {
MobileWebBrowserEventSchema,
MobileWebBrowserStreamPayloadSchema,
type MobileWebBrowserEvent,
type MobileWebBrowserStreamPayload
} from '../../shared/mobile-web/browser-operation-contract'
import { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import type { MobileWebBridgeSubscription } from './mobile-web-bridge-subscription'
import type { MobileWebBridgeSubscriptionClient } from './mobile-web-bridge-subscription-client'
export type MobileWebBrowserSubscriptionArgs = [
payload: MobileWebBrowserStreamPayload,
onEvent: (event: MobileWebBrowserEvent) => void,
onError: (error: MobileWebBridgeClientError) => void
]
export function subscribeMobileWebHostBrowser(
subscriptions: MobileWebBridgeSubscriptionClient,
...[payload, onEvent, onError]: MobileWebBrowserSubscriptionArgs
): MobileWebBridgeSubscription {
if (!MobileWebBrowserStreamPayloadSchema.safeParse(payload).success) {
const error = new MobileWebBridgeClientError('invalid_request', false)
queueMicrotask(() => onError(error))
return { ready: Promise.reject(error), unsubscribe() {} }
}
const { workspaceId, pageId, ...request } = payload
let cancelled = false
const current = subscriptions.subscribeHost(
{
method: 'mobileWeb.browser.subscribe',
workspaceId,
params: { page: pageId, ...request }
},
(event) => {
if (cancelled || isSubscriptionHandshake(event)) {
return
}
const parsed = MobileWebBrowserEventSchema.safeParse(event)
if (!parsed.success) {
onError(new MobileWebBridgeClientError('invalid_message', false))
return
}
onEvent(parsed.data)
},
(error) => {
if (!cancelled) {
onError(error)
}
}
)
return {
ready: current.ready,
unsubscribe() {
cancelled = true
current.unsubscribe()
}
}
}
/** The lane opens with a bare `ready` carrying the cancel id; the browser's own `ready` carries
* tab state, which is what the pane waits for. */
function isSubscriptionHandshake(event: unknown): boolean {
return (
typeof event === 'object' &&
event !== null &&
'type' in event &&
event.type === 'ready' &&
!('tab' in event)
)
}
@@ -92,13 +92,14 @@ export const MobileWebBrowserDialogPayloadSchema = MobileWebBrowserTargetSchema.
action: z.enum(['accept', 'dismiss'])
}).strict()
export const MobileWebBrowserNavigateResultSchema = z
.object({
url: z.string().min(1).max(MOBILE_WEB_BROWSER_URL_MAX_LENGTH)
})
.strict()
// Desktop-produced results stay open: a newer host may add a field a cached page has never seen.
export const MobileWebBrowserNavigateResultSchema = z.object({
url: z.string().min(1).max(MOBILE_WEB_BROWSER_URL_MAX_LENGTH)
})
export const MobileWebBrowserCommandResultSchema = z.null()
/** Every browser command the page issues answers with this acknowledgement; the host result
* carries the raw tab URL, which the desktop wrapper never forwards. */
export const MobileWebBrowserAckSchema = z.object({ applied: z.literal(true) })
const MobileWebBrowserTabStateSchema = z
.object({