Merge branch 'ota-w-browser' of /tmp/orca-fix/browser into ota-wave-integrate

# Conflicts:
#	docs/reference/mobile-hybrid-webview-architecture.md
#	mobile/src/mobile-web/mobile-web-capability-broker.ts
#	mobile/src/mobile-web/mobile-web-capability-dispatch-census.test.ts
#	mobile/src/mobile-web/mobile-web-capability-execution-dependencies.ts
#	mobile/src/mobile-web/mobile-web-mutation-reauthorization-census.test.ts
#	src/shared/mobile-web/bridge-operation-registry.ts
This commit is contained in:
Jinwoo-H
2026-09-07 16:24:53 -04:00
39 changed files with 1249 additions and 844 deletions
@@ -56,12 +56,12 @@ acceptance.
## Ownership Boundaries
| Owner | Responsibilities |
| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Owner | Responsibilities |
| ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Native mobile shell | Pairing and host selection; secure credential storage; authenticated encrypted transport; QR scanning; notifications and deep links; package verification, cache, private origin, and recovery; clipboard, haptics, audio, camera and file/photo pickers; native onboarding, pairing recovery, and privacy; diagnostics capture (transport log, reachability probes, report submission) |
| Desktop-served React Native Web application | Workspace list and creation; sessions and terminal presentation; files, previews, diffs, source control, reviews, tasks, accounts, browser presentation, Agent History, native-chat presentation, and the settings, troubleshooting and connection-log screens |
| Desktop runtime | Builds and ships the matching web package; serves its manifest and chunks through authenticated RPC; reauthorizes every workspace mutation; enforces host, workspace, provider, path, and resource limits |
| Typed native bridge | Connects the unprivileged page to explicitly granted Desktop operations and native capabilities; carries connection and route state without exposing transport credentials |
| Desktop-served React Native Web application | Workspace list and creation; sessions and terminal presentation; files, previews, diffs, source control, reviews, tasks, accounts, browser presentation, Agent History, native-chat presentation, and the settings, troubleshooting and connection-log screens |
| Desktop runtime | Builds and ships the matching web package; serves its manifest and chunks through authenticated RPC; reauthorizes every workspace mutation; enforces host, workspace, provider, path, and resource limits |
| Typed native bridge | Connects the unprivileged page to explicitly granted Desktop operations and native capabilities; carries connection and route state without exposing transport credentials |
The page never receives the raw RPC client, pairing credential, host endpoint,
private key, cache path, or unrestricted native module access. Native-owned
@@ -229,6 +229,21 @@ edges still meet the device and keep their measured values.
GitLab differ only inside those handlers; the shell knows neither. Review
output the provider does not bound — check-run job logs and file diffs — is
clipped on the desktop so the page's schema bounds hold.
Control reads/watch, session snapshot/feed/actions, terminal metadata and
browser input/navigation/screencast use this path.
- Browser control is desktop-owned end to end. `mobileWeb.browser.navigate`,
`.history`, `.pointer`, `.keyboard`, `.dialog` and the `.subscribe`/
`.unsubscribe` screencast stream are the page's whole surface. The wrapper is
where the worktree is scoped, the tab URL is stripped
(`browser-url-privacy.ts`), the title and dialog text are bounded, the
press/release fallback for a failed plain click runs, and pointer and keyboard
traffic meet a per-connection token bucket
(`mobile-web-browser-input-rate-limit.ts`). The generic lane carries JSON only,
so a binary screencast frame crosses it as a run of base64 `frameChunk`
events the page reassembles. The stream opens with a bare `ready` carrying the
cancel id; the browser's own `ready` is the one that carries tab state. The
released native app keeps calling the raw `browser.*` methods and
`browser.screencast` unchanged.
- Decisions behind the generic lane and its 2026-09-07 simplification are in
[`plans/2026-09-07-long-lived-mobile-shell-decisions.md`](./plans/2026-09-07-long-lived-mobile-shell-decisions.md).
Unmigrated domain operations keep their current adapters until moved.
@@ -1,27 +0,0 @@
import { describe, expect, it } from 'vitest'
import { sanitizeMobileWebBrowserEvent } from './mobile-web-browser-event-sanitizer'
describe('sanitizeMobileWebBrowserEvent', () => {
it('removes URL credentials from browser events', () => {
const event = sanitizeMobileWebBrowserEvent({
type: 'navigation',
tab: {
url: 'https://user:password@example.com/callback?access_token=secret&tab=review',
title: 'Review',
canGoBack: true,
canGoForward: false
}
})
expect(event).toEqual({
type: 'navigation',
tab: {
url: 'https://example.com/callback?tab=review',
title: 'Review',
canGoBack: true,
canGoForward: false
}
})
expect(JSON.stringify(event)).not.toMatch(/password|access_token|secret/)
})
})
@@ -1,108 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import type { RpcClient } from '../transport/rpc-client'
import { executeMobileWebBrowserOperation } from './mobile-web-browser-operations'
import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
describe('mobile web browser operations', () => {
it('resolves opaque workspace authority and forwards the host page id for navigation', async () => {
const { workspaceAuthority, workspaceId, pageId } = authorities()
const sendRequest = vi.fn<RpcClient['sendRequest']>().mockResolvedValue({
ok: true,
result: { url: 'https://example.com/', title: 'Example', rawPageId: 'raw-page' }
})
await expect(
executeMobileWebBrowserOperation({
operation: 'navigate',
payload: { workspaceId, pageId, url: 'https://example.com' },
client: { sendRequest } as unknown as RpcClient,
workspaceAuthority
})
).resolves.toEqual({ url: 'https://example.com/' })
expect(sendRequest).toHaveBeenCalledWith(
'browser.goto',
{
worktree: 'id:host-workspace',
page: 'raw-page',
url: 'https://example.com'
},
{ timeoutMs: 30_000 }
)
})
it('removes credentials from the authoritative navigation result', async () => {
const { workspaceAuthority, workspaceId, pageId } = authorities()
const sendRequest = vi.fn<RpcClient['sendRequest']>().mockResolvedValue({
ok: true,
result: {
url: 'https://user:password@example.com/callback?code=secret&tab=review#access_token=secret'
}
})
await expect(
executeMobileWebBrowserOperation({
operation: 'navigate',
payload: { workspaceId, pageId, url: 'https://example.com' },
client: { sendRequest } as unknown as RpcClient,
workspaceAuthority
})
).resolves.toEqual({ url: 'https://example.com/callback?tab=review' })
})
it('keeps pointer fallback native and refuses an unknown page workspace', async () => {
const { workspaceAuthority, workspaceId, pageId } = authorities()
const sendRequest = vi
.fn<RpcClient['sendRequest']>()
.mockResolvedValueOnce({ ok: false, error: { code: 'unsupported', message: 'unsupported' } })
.mockResolvedValue({ ok: true, result: null })
const client = { sendRequest } as unknown as RpcClient
await executeMobileWebBrowserOperation({
operation: 'pointer',
payload: {
workspaceId,
pageId,
action: 'click',
x: 20,
y: 30,
button: 'left',
modifiers: []
},
client,
workspaceAuthority
})
expect(sendRequest.mock.calls.map(([method]) => method)).toEqual([
'browser.mouseClick',
'browser.mouseMove',
'browser.mouseDown',
'browser.mouseUp'
])
workspaceAuthority.synchronize([])
await expect(
executeMobileWebBrowserOperation({
operation: 'reload',
payload: { workspaceId, pageId },
client,
workspaceAuthority
})
).rejects.toMatchObject({ code: 'not_found' })
})
})
function authorities(): {
workspaceAuthority: MobileWebWorkspaceAuthority
workspaceId: string
pageId: string
} {
const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(3)
const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes)
workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }])
return {
workspaceAuthority,
workspaceId: workspaceAuthority.pageWorkspaceId('host-workspace'),
pageId: 'raw-page'
}
}
@@ -1,160 +0,0 @@
import {
MobileWebBrowserCommandResultSchema,
MobileWebBrowserDialogPayloadSchema,
MobileWebBrowserKeyboardPayloadSchema,
MobileWebBrowserNavigatePayloadSchema,
MobileWebBrowserNavigateResultSchema,
MobileWebBrowserPointerPayloadSchema,
MobileWebBrowserTargetPayloadSchema
} from '../../../src/shared/mobile-web/browser-operation-contract'
import { mobileWebPageBrowserUrl } from '../../../src/shared/mobile-web/browser-url-privacy'
import type { RpcClient } from '../transport/rpc-client'
import { MobileWebBrokerError, mobileWebBrokerHostRpcError } from './mobile-web-broker-error'
import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
export async function executeMobileWebBrowserOperation(args: {
operation: string
payload: unknown
client: RpcClient
workspaceAuthority: MobileWebWorkspaceAuthority
}): Promise<unknown> {
if (args.operation === 'navigate') {
const payload = MobileWebBrowserNavigatePayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
const response = await args.client.sendRequest(
'browser.goto',
{ ...target, url: payload.url },
{ timeoutMs: 30_000 }
)
const result = requireResult(response)
const parsed = MobileWebBrowserNavigateResultSchema.safeParse({
url: isRecord(result) ? mobileWebPageBrowserUrl(result.url) : 'about:blank'
})
if (!parsed.success) {
throw new MobileWebBrokerError('host_error')
}
return parsed.data
}
if (args.operation === 'pointer') {
const payload = MobileWebBrowserPointerPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
if (payload.action === 'scroll') {
await requireRequest(args.client, 'browser.mouseMove', {
...target,
x: payload.x,
y: payload.y
})
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseWheel', {
...target,
dx: payload.dx,
dy: payload.dy
})
return MobileWebBrowserCommandResultSchema.parse(null)
}
const click = await args.client.sendRequest(
'browser.mouseClick',
{
...target,
x: payload.x,
y: payload.y,
button: payload.button,
modifiers: payload.modifiers,
...(payload.radius === undefined ? {} : { radius: payload.radius })
},
{ timeoutMs: 5_000 }
)
if (!click.ok && payload.modifiers.length === 0) {
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseMove', {
...target,
x: payload.x,
y: payload.y
})
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseDown', {
...target,
button: payload.button
})
assertTarget(payload, target, args.workspaceAuthority)
await requireRequest(args.client, 'browser.mouseUp', {
...target,
button: payload.button
})
}
return MobileWebBrowserCommandResultSchema.parse(null)
}
if (args.operation === 'keyboard') {
const payload = MobileWebBrowserKeyboardPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
await requireRequest(
args.client,
payload.action === 'insertText' ? 'browser.keyboardInsertText' : 'browser.keypress',
payload.action === 'insertText'
? { ...target, text: payload.text }
: { ...target, key: payload.key },
5_000
)
return MobileWebBrowserCommandResultSchema.parse(null)
}
if (args.operation === 'dialog') {
const payload = MobileWebBrowserDialogPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
await requireRequest(
args.client,
payload.action === 'accept' ? 'browser.dialogAccept' : 'browser.dialogDismiss',
target,
5_000
)
return MobileWebBrowserCommandResultSchema.parse(null)
}
if (args.operation === 'back' || args.operation === 'forward' || args.operation === 'reload') {
const payload = MobileWebBrowserTargetPayloadSchema.parse(args.payload)
const target = resolveTarget(payload, args.workspaceAuthority)
await requireRequest(args.client, `browser.${args.operation}`, target)
return MobileWebBrowserCommandResultSchema.parse(null)
}
throw new MobileWebBrokerError('unsupported_capability')
}
// Re-reads the workspace binding mid-operation: a workspace switch must not land later frames.
function assertTarget(
payload: { workspaceId: string; pageId: string },
expected: { worktree: string; page: string },
workspaceAuthority: MobileWebWorkspaceAuthority
): void {
const current = resolveTarget(payload, workspaceAuthority)
if (current.worktree !== expected.worktree || current.page !== expected.page) {
throw new MobileWebBrokerError('conflict')
}
}
function resolveTarget(
payload: { workspaceId: string; pageId: string },
workspaceAuthority: MobileWebWorkspaceAuthority
): { worktree: string; page: string } {
return {
worktree: `id:${workspaceAuthority.hostWorkspaceId(payload.workspaceId)}`,
page: payload.pageId
}
}
async function requireRequest(
client: RpcClient,
method: string,
payload: unknown,
timeoutMs = 15_000
): Promise<void> {
requireResult(await client.sendRequest(method, payload, { timeoutMs }))
}
function requireResult(response: Awaited<ReturnType<RpcClient['sendRequest']>>): unknown {
if (!response.ok) {
throw mobileWebBrokerHostRpcError(response.error)
}
return response.result
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -1,126 +0,0 @@
import { describe, expect, it, vi } from 'vitest'
import {
BrowserScreencastOpcode,
type BrowserScreencastFrame
} from '../transport/browser-screencast-protocol'
import type { RpcClient } from '../transport/rpc-client'
import { MobileWebBrowserStreams } from './mobile-web-browser-streams'
import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
describe('mobile web browser streams', () => {
it('chunks frames below the bridge limit and cleans up', async () => {
const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(4)
const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes)
workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }])
const workspaceId = workspaceAuthority.pageWorkspaceId('host-workspace')
const pageId = 'raw-page'
const postEvent = vi.fn(async () => {})
let onEvent: ((event: unknown) => void) | undefined
let onFrame: ((frame: BrowserScreencastFrame) => void) | undefined
const unsubscribe = vi.fn()
const subscribe = vi
.fn<RpcClient['subscribe']>()
.mockImplementation((_method, _payload, eventListener, options) => {
onEvent = eventListener
onFrame = options?.onBinaryFrame
return unsubscribe
})
const streams = new MobileWebBrowserStreams({
isActive: () => true,
workspaceAuthority,
postEvent,
postClosed: vi.fn()
})
streams.start({
requestId: 'request-1',
subscriptionId: 'subscription-1',
payload: {
workspaceId,
pageId,
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
},
client: { subscribe } as unknown as RpcClient
})
expect(subscribe).toHaveBeenCalledWith(
'browser.screencast',
{
worktree: 'id:host-workspace',
page: 'raw-page',
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
},
expect.any(Function),
{ onBinaryFrame: expect.any(Function) }
)
onEvent?.({
type: 'ready',
browserPageId: 'raw-page',
tab: {
url: 'https://example.com',
title: 'Example',
canGoBack: true,
canGoForward: false,
rawSecret: 'must-not-cross'
}
})
onEvent?.({
type: 'navigation',
tab: {
url: 'https://www.iana.org/help/example-domains',
title: 'IANA-managed Reserved Domains',
canGoBack: true,
canGoForward: false
}
})
const image = new Uint8Array(200_000).map((_, index) => index % 251)
onFrame?.({
opcode: BrowserScreencastOpcode.Frame,
seq: 7,
format: 'jpeg',
metadata: { deviceWidth: 800, deviceHeight: 600 },
image
})
await vi.waitFor(() => expect(postEvent).toHaveBeenCalledTimes(4))
expect(postEvent.mock.calls.map(([, sequence]) => sequence)).toEqual([0, 1, 2, 3])
expect(postEvent.mock.calls[0]?.[2]).toEqual({
type: 'ready',
tab: {
url: 'https://example.com',
title: 'Example',
canGoBack: true,
canGoForward: false
}
})
expect(postEvent.mock.calls[1]?.[2]).toEqual({
type: 'navigation',
tab: {
url: 'https://www.iana.org/help/example-domains',
title: 'IANA-managed Reserved Domains',
canGoBack: true,
canGoForward: false
}
})
const chunks = postEvent.mock.calls.slice(2).map((call) => call[2])
expect(chunks).toMatchObject([
{ type: 'frameChunk', frameSequence: 7, chunkIndex: 0, chunkCount: 2 },
{ type: 'frameChunk', frameSequence: 7, chunkIndex: 1, chunkCount: 2 }
])
expect(JSON.stringify(chunks)).not.toContain('raw-page')
expect(streams.cancel('subscription-1')).toBe('request-1')
expect(unsubscribe).toHaveBeenCalledOnce()
})
})
@@ -1,189 +0,0 @@
import {
MobileWebSubscriptionLedger,
type MobileWebSubscriptionLedgerConfig,
type MobileWebSubscriptionRecord
} from './mobile-web-subscription-ledger'
import { Buffer } from 'buffer/'
import {
MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES,
MOBILE_WEB_BROWSER_FRAME_MAX_IMAGE_BYTES,
MobileWebBrowserEventSchema,
MobileWebBrowserStreamPayloadSchema,
type MobileWebBrowserEvent
} from '../../../src/shared/mobile-web/browser-operation-contract'
import type {
BrowserScreencastFrame,
BrowserScreencastFrameMetadata
} from '../transport/browser-screencast-protocol'
import type { RpcClient } from '../transport/rpc-client'
import { sanitizeMobileWebBrowserEvent } from './mobile-web-browser-event-sanitizer'
import { MobileWebBrokerError } from './mobile-web-broker-error'
import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
type ScreencastRecord = MobileWebSubscriptionRecord & {
frameQueued: boolean
pendingFrame: BrowserScreencastFrame | null
}
type BrowserLedgerConfig = MobileWebSubscriptionLedgerConfig<MobileWebBrowserEvent> & {
workspaceAuthority: MobileWebWorkspaceAuthority
}
export class MobileWebBrowserStreams extends MobileWebSubscriptionLedger<
MobileWebBrowserEvent,
ScreencastRecord
> {
constructor(private readonly config: BrowserLedgerConfig) {
super({ ...config, operationKey: 'browser.subscribe' })
}
start(args: {
requestId: string
subscriptionId: string
payload: unknown
client: RpcClient
}): void {
this.admit(args.subscriptionId)
const payload = MobileWebBrowserStreamPayloadSchema.parse(args.payload)
const hostWorkspaceId = this.config.workspaceAuthority.hostWorkspaceId(payload.workspaceId)
const record: ScreencastRecord = {
...this.newRecord(args.requestId),
frameQueued: false,
pendingFrame: null
}
this.open(args.subscriptionId, record, () =>
args.client.subscribe(
'browser.screencast',
{
worktree: `id:${hostWorkspaceId}`,
page: payload.pageId,
format: payload.format,
quality: payload.quality,
maxWidth: payload.maxWidth,
maxHeight: payload.maxHeight,
everyNthFrame: payload.everyNthFrame,
minFrameIntervalMs: payload.minFrameIntervalMs,
...(payload.viewportWidth === undefined ? {} : { viewportWidth: payload.viewportWidth }),
...(payload.viewportHeight === undefined
? {}
: { viewportHeight: payload.viewportHeight }),
...(payload.deviceScaleFactor === undefined
? {}
: { deviceScaleFactor: payload.deviceScaleFactor }),
...(payload.mobile === undefined ? {} : { mobile: payload.mobile })
},
(event) => this.receiveEvent(args.subscriptionId, record, event),
{
onBinaryFrame: (frame) => this.receiveFrame(args.subscriptionId, record, frame)
}
)
)
}
// Drops the parked image so a retired stream cannot pin a multi-megabyte frame.
protected override retire(record: ScreencastRecord): void {
record.pendingFrame = null
}
private receiveEvent(subscriptionId: string, record: ScreencastRecord, value: unknown): void {
if (!this.isCurrent(subscriptionId, record)) {
return
}
const event = sanitizeMobileWebBrowserEvent(value)
if (event) {
this.enqueueTask(subscriptionId, record, () => this.deliver(subscriptionId, record, event))
}
}
private receiveFrame(
subscriptionId: string,
record: ScreencastRecord,
frame: BrowserScreencastFrame
): void {
if (!this.isCurrent(subscriptionId, record)) {
return
}
if (frame.image.byteLength > MOBILE_WEB_BROWSER_FRAME_MAX_IMAGE_BYTES) {
this.enqueueTask(subscriptionId, record, () =>
this.deliver(subscriptionId, record, {
type: 'error',
message: 'Browser frame is too large to display safely.'
})
)
return
}
record.pendingFrame = frame
if (record.frameQueued) {
return
}
record.frameQueued = true
this.enqueueTask(subscriptionId, record, async () => {
const latest = record.pendingFrame
record.pendingFrame = null
if (latest) {
await this.deliverFrame(subscriptionId, record, latest)
}
record.frameQueued = false
const pending = record.pendingFrame
if (pending && this.isCurrent(subscriptionId, record)) {
record.pendingFrame = null
this.receiveFrame(subscriptionId, record, pending)
}
})
}
private async deliverFrame(
subscriptionId: string,
record: ScreencastRecord,
frame: BrowserScreencastFrame
): Promise<void> {
const chunkCount = Math.ceil(frame.image.byteLength / MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES)
for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex += 1) {
if (!this.isCurrent(subscriptionId, record)) {
return
}
const start = chunkIndex * MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES
const end = Math.min(frame.image.byteLength, start + MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES)
await this.deliver(subscriptionId, record, {
type: 'frameChunk',
frameSequence: frame.seq,
format: frame.format,
metadata: boundedMetadata(frame.metadata),
imageBytes: frame.image.byteLength,
chunkIndex,
chunkCount,
data: Buffer.from(frame.image.subarray(start, end)).toString('base64')
})
}
}
// Why the sequence is claimed here and not at enqueue time: one queued frame task posts a whole
// chunk run, so the numbers must be handed out per post, in post order.
private async deliver(
subscriptionId: string,
record: ScreencastRecord,
value: MobileWebBrowserEvent
): Promise<void> {
const event = MobileWebBrowserEventSchema.parse(value)
const sequence = record.sequence
record.sequence += 1
await this.options.postEvent(subscriptionId, sequence, event)
}
}
function boundedMetadata(metadata: BrowserScreencastFrameMetadata): BrowserScreencastFrameMetadata {
const parsed = MobileWebBrowserEventSchema.parse({
type: 'frameChunk',
frameSequence: 0,
format: 'jpeg',
metadata,
imageBytes: 1,
chunkIndex: 0,
chunkCount: 1,
data: 'AA=='
})
if (parsed.type !== 'frameChunk') {
throw new MobileWebBrokerError('host_error')
}
return parsed.metadata
}
@@ -238,7 +238,6 @@ export class MobileWebCapabilityBroker {
agentHistoryPager: this.authorities.agentHistoryPager,
agentHistoryResume: this.authorities.agentHistoryResume,
accountSubscriptions: this.subscriptions.account,
browserStreams: this.subscriptions.browser,
speechAuthority: this.speechAuthority,
workspaceSubscriptions: this.subscriptions.workspace,
hostSubscriptions: this.subscriptions.host,
@@ -49,7 +49,7 @@ describe('mobile web capability dispatch census', () => {
})
expect(unresolved.map(({ capability, operation }) => `${capability}.${operation}`)).toEqual([])
expect(registeredOperations()).toHaveLength(168)
expect(registeredOperations()).toHaveLength(0)
})
it('carries a dispatch arm for exactly the capabilities that own operations of that mode', () => {
@@ -1,4 +1,3 @@
import { MobileWebBrowserStreamPayloadSchema } from '../../../src/shared/mobile-web/browser-operation-contract'
import { MobileWebWorkspaceSubscribePayloadSchema } from '../../../src/shared/mobile-web/bridge-operation-contract'
import type { MobileWebBridgePageMessage } from '../../../src/shared/mobile-web/bridge-contract'
import type { MobileWebBridgeCapability } from '../../../src/shared/mobile-web/bridge-operation-registry'
@@ -7,7 +6,6 @@ import { executeWorkspace } from './mobile-web-workspace-capability'
import { executeMobileWebAccountCapability } from './mobile-web-account-capability'
import { executeMobileWebAgentHistoryOperation } from './mobile-web-agent-history-operations'
import { MobileWebBrokerError } from './mobile-web-broker-error'
import { executeMobileWebBrowserOperation } from './mobile-web-browser-operations'
import type { MobileWebCapabilityExecutionDependencies } from './mobile-web-capability-execution-dependencies'
import { executeMobileWebFileOperation } from './mobile-web-file-operations'
import { executeMobileWebMarkdownOperation } from './mobile-web-markdown-operations'
@@ -50,15 +48,6 @@ async function executeNavigation(args: Deps, request: OnceRequest): Promise<unkn
})
}
async function executeBrowser(args: Deps, request: OnceRequest): Promise<unknown> {
return executeMobileWebBrowserOperation({
operation: request.operation,
payload: request.payload,
client: args.connectedClient(),
workspaceAuthority: args.workspaceAuthority
})
}
async function executeTerminal(args: Deps, request: OnceRequest): Promise<unknown> {
return args.terminalStreams.handle(request.payload, args.connectedClient())
}
@@ -131,7 +120,6 @@ export const MOBILE_WEB_ONCE_CAPABILITY_ARMS: Partial<Record<MobileWebBridgeCapa
navigation: executeNavigation,
agentHistory: (args) => executeMobileWebAgentHistoryOperation(args),
account: (args) => executeMobileWebAccountCapability(args),
browser: executeBrowser,
workspace: executeWorkspace,
settings: executeWorkspace,
terminal: executeTerminal,
@@ -141,18 +129,6 @@ export const MOBILE_WEB_ONCE_CAPABILITY_ARMS: Partial<Record<MobileWebBridgeCapa
task: executeTask
}
async function subscribeBrowser(args: Deps, request: SubscriptionRequest): Promise<unknown> {
requireSubscribeOperation(request)
MobileWebBrowserStreamPayloadSchema.parse(request.payload)
args.browserStreams.start({
requestId: request.requestId,
subscriptionId: request.subscriptionId,
payload: request.payload,
client: args.connectedClient()
})
return null
}
async function subscribeWorkspace(args: Deps, request: SubscriptionRequest): Promise<unknown> {
if (request.operation === 'hostSubscribe') {
args.hostSubscriptions.start({
@@ -200,7 +176,6 @@ export const MOBILE_WEB_SUBSCRIPTION_CAPABILITY_ARMS: Partial<
Record<MobileWebBridgeCapability, SubscriptionArm>
> = {
account: (args) => executeMobileWebAccountCapability(args),
browser: subscribeBrowser,
workspace: subscribeWorkspace,
terminal: subscribeTerminal,
speech: subscribeSpeech
@@ -5,7 +5,6 @@ import type { MobileWebAccountSubscriptions } from './mobile-web-account-subscri
import type { MobileWebAgentHistoryAuthority } from './mobile-web-agent-history-authority'
import type { MobileWebAgentHistoryPager } from './mobile-web-agent-history-pager'
import type { MobileWebAgentHistoryResume } from './mobile-web-agent-history-resume'
import type { MobileWebBrowserStreams } from './mobile-web-browser-streams'
import type { MobileWebCommitMessageGeneration } from './mobile-web-commit-message-generation'
import type { MobileWebNavigationAuthority } from './mobile-web-navigation-operations'
import type { MobileWebNativeCapabilityAuthority } from './mobile-web-native-capability-authority'
@@ -32,7 +31,6 @@ export type MobileWebCapabilityExecutionDependencies = {
agentHistoryResume: MobileWebAgentHistoryResume
hostSubscriptions: MobileWebHostSubscriptions
accountSubscriptions: MobileWebAccountSubscriptions
browserStreams: MobileWebBrowserStreams
speechAuthority: MobileWebSpeechAuthority
workspaceSubscriptions: MobileWebWorkspaceSubscriptions
terminalStreams: MobileWebTerminalStreams
@@ -5,14 +5,12 @@ import type {
MobileWebSubscriptionLedgerConfig,
MobileWebSubscriptionLedgerHandle
} from './mobile-web-subscription-ledger'
import { MobileWebBrowserStreams } from './mobile-web-browser-streams'
import type { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
import { MobileWebWorkspaceSubscriptions } from './mobile-web-workspace-subscriptions'
export class MobileWebCapabilitySubscriptions {
readonly host: MobileWebHostSubscriptions
readonly account: MobileWebAccountSubscriptions
readonly browser: MobileWebBrowserStreams
readonly workspace: MobileWebWorkspaceSubscriptions
private readonly ledgers: MobileWebSubscriptionLedgerHandle[]
@@ -31,12 +29,8 @@ export class MobileWebCapabilitySubscriptions {
workspaceAuthority: args.workspaceAuthority
})
this.account = new MobileWebAccountSubscriptions(shared)
this.browser = new MobileWebBrowserStreams({
...shared,
workspaceAuthority: args.workspaceAuthority
})
this.workspace = new MobileWebWorkspaceSubscriptions(shared)
this.ledgers = [this.host, this.account, this.browser, this.workspace]
this.ledgers = [this.host, this.account, this.workspace]
}
countForOperation(operationKey: string): number {
@@ -35,13 +35,6 @@ const REAUTHORIZATION_SITES: Record<string, number> = {
// Device-only mutations and handles consumed in one awaited call have no reauthorization window.
const NO_REAUTHORIZATION_WINDOW: readonly string[] = [
'workspace.activate',
'browser.back',
'browser.dialog',
'browser.forward',
'browser.keyboard',
'browser.navigate',
'browser.pointer',
'browser.reload',
'file.releaseTerminalArtifact',
'native.alert',
'native.clipboardWrite',
@@ -159,7 +152,7 @@ describe('mobile web mutation reauthorization census', () => {
}
expect(unaccounted).toEqual([])
expect(mutations()).toHaveLength(94)
expect(mutations()).toHaveLength(0)
})
it('exempts only registered mutations', () => {
@@ -1,12 +0,0 @@
import { capabilityGrants, grantLimits } from './mobile-web-production-grant-table'
export const MOBILE_WEB_PRODUCTION_BROWSER_GRANTS = capabilityGrants('browser', {
subscribe: grantLimits(4 * 1024, 1 * 1024, 1, 4, 1),
navigate: grantLimits(8 * 1024, 8 * 1024, 1, 8, 2),
back: grantLimits(2 * 1024, 256, 2, 12, 4),
forward: grantLimits(2 * 1024, 256, 2, 12, 4),
reload: grantLimits(2 * 1024, 256, 2, 12, 4),
dialog: grantLimits(2 * 1024, 256, 2, 12, 4),
pointer: grantLimits(4 * 1024, 256, 2, 40, 20),
keyboard: grantLimits(40 * 1024, 256, 2, 20, 10)
})
@@ -1,4 +1,3 @@
import { MOBILE_WEB_PRODUCTION_BROWSER_GRANTS } from './mobile-web-production-browser-grants'
import { MOBILE_WEB_PRODUCTION_FILE_GRANTS } from './mobile-web-production-file-grants'
import { capabilityGrants, grantLimits, indexGrants } from './mobile-web-production-grant-table'
import { MOBILE_WEB_PRODUCTION_NAVIGATION_GRANTS } from './mobile-web-production-navigation-grants'
@@ -15,7 +14,7 @@ export type { MobileWebOperationGrant } from './mobile-web-production-grant-tabl
export const MOBILE_WEB_PRODUCTION_GRANTS = [
...capabilityGrants('workspace', {
hostSubscribe: grantLimits(600 * 1024, 1024, 8, 8, 2),
hostRequest: grantLimits(600 * 1024, 600 * 1024, 16, 32, 4),
hostRequest: grantLimits(600 * 1024, 600 * 1024, 16, 48, 24),
snapshot: grantLimits(1 * 1024, 128 * 1024, 2, 4, 1),
repositories: grantLimits(256, 128 * 1024, 2, 4, 1),
subscribe: grantLimits(256, 1 * 1024, 1, 4, 1),
@@ -39,7 +38,6 @@ export const MOBILE_WEB_PRODUCTION_GRANTS = [
}),
...MOBILE_WEB_PRODUCTION_SESSION_GRANTS,
...MOBILE_WEB_PRODUCTION_TERMINAL_GRANTS,
...MOBILE_WEB_PRODUCTION_BROWSER_GRANTS,
...MOBILE_WEB_PRODUCTION_FILE_GRANTS,
...capabilityGrants('sourceControl', {
generateCommitMessage: grantLimits(4 * 1024, 16 * 1024, 1, 4, 0.25),
@@ -18,7 +18,6 @@ export function mobileWebRequestExpectsSubscription(request: {
request.capability === 'session' ||
request.capability === 'sourceControl' ||
request.capability === 'terminal' ||
request.capability === 'browser' ||
request.capability === 'nativeChat' ||
request.capability === 'speech') &&
request.operation === 'subscribe')
@@ -2,11 +2,9 @@ import { describe, expect, it, vi } from 'vitest'
import type { MobileWebSubscriptionClosure } from './mobile-web-subscription-closure'
import type { RpcClient } from '../transport/rpc-client'
import { MobileWebAccountSubscriptions } from './mobile-web-account-subscriptions'
import { MobileWebBrowserStreams } from './mobile-web-browser-streams'
import { MobileWebSpeechSubscriptions } from './mobile-web-speech-subscriptions'
import type { MobileWebSpeechEvent } from '../../../src/shared/mobile-web/speech-operation-contract'
import { MobileWebWorkspaceSubscriptions } from './mobile-web-workspace-subscriptions'
import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
const SUBSCRIPTION_ID = 'subscription-1'
@@ -18,17 +16,13 @@ type Posts = {
type LedgerCase = {
name: string
// Browser drops an unparseable frame instead of retiring, so it has no invalid-message closure.
// A push-driven ledger has no host frame to reject, so it has no invalid-message closure.
invalidCode: 'invalid_message' | null
invalid: unknown
valid: unknown
open: (posts: Posts) => Promise<(value: unknown) => void>
}
function randomBytes(length: number): Uint8Array {
return new Uint8Array(length).fill(4)
}
function hostClient(): { client: RpcClient; emit: (value: unknown) => void } {
let listener: ((value: unknown) => void) | undefined
const client = {
@@ -54,12 +48,6 @@ function hostClient(): { client: RpcClient; emit: (value: unknown) => void } {
return { client, emit: (value) => listener?.(value) }
}
function pageWorkspace(): { authority: MobileWebWorkspaceAuthority; pageWorkspaceId: string } {
const authority = new MobileWebWorkspaceAuthority(randomBytes)
authority.synchronize([{ workspaceId: 'workspace-1', repoId: 'repo-1' }])
return { authority, pageWorkspaceId: authority.pageWorkspaceId('workspace-1') }
}
const LEDGER_CASES: LedgerCase[] = [
{
name: 'account',
@@ -91,36 +79,6 @@ const LEDGER_CASES: LedgerCase[] = [
return host.emit
}
},
{
name: 'browser',
invalidCode: null,
invalid: { type: 'bogus' },
valid: {
type: 'ready',
browserPageId: 'raw-page',
tab: { url: 'https://example.com', title: 'Example', canGoBack: false, canGoForward: false }
},
open: async (posts) => {
const host = hostClient()
const { authority, pageWorkspaceId } = pageWorkspace()
new MobileWebBrowserStreams({ ...posts, workspaceAuthority: authority }).start({
requestId: 'request-1',
subscriptionId: SUBSCRIPTION_ID,
payload: {
workspaceId: pageWorkspaceId,
pageId: 'raw-page',
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
},
client: host.client
})
return host.emit
}
},
{
name: 'speech',
// Push-driven from the shell's dictation runtime, so no host frame can be unusable.
@@ -135,7 +93,7 @@ const LEDGER_CASES: LedgerCase[] = [
}
]
// Ledgers that retire on an unusable host message; browser drops the frame instead, so it is absent.
// Ledgers that retire on an unusable host message; a push-driven ledger has none, so it is absent.
const RETIRING_LEDGER_CASES = LEDGER_CASES.filter(
(ledger): ledger is LedgerCase & { invalidCode: 'invalid_message' } => ledger.invalidCode !== null
)
@@ -6,7 +6,6 @@ import type {
} from '../../../src/shared/mobile-web/bridge-contract'
import type { RpcClient } from '../transport/rpc-client'
import { MobileWebAccountSubscriptions } from './mobile-web-account-subscriptions'
import { MobileWebBrowserStreams } from './mobile-web-browser-streams'
import {
isRetryableMobileWebBridgeError,
mobileWebBridgeErrorCode
@@ -16,9 +15,6 @@ import {
mobileWebBridgeRequestMessage
} from './mobile-web-bridge-roundtrip-fixture'
import { MobileWebWorkspaceSubscriptions } from './mobile-web-workspace-subscriptions'
import { MobileWebWorkspaceAuthority } from './mobile-web-workspace-authority'
const randomBytes = (length: number): Uint8Array => new Uint8Array(length).fill(4)
function stubClient(): RpcClient {
return { subscribe: vi.fn(() => () => {}) } as unknown as RpcClient
@@ -29,20 +25,9 @@ function ledgerStarters(): { name: string; start: (subscriptionId: string) => vo
const postEvent = async (): Promise<void> => {}
const isActive = (): boolean => true
const client = stubClient()
const workspaceAuthority = new MobileWebWorkspaceAuthority(randomBytes)
workspaceAuthority.synchronize([{ workspaceId: 'host-workspace', repoId: 'repo-1' }])
const pageWorkspaceId = workspaceAuthority.pageWorkspaceId('host-workspace')
const pageId = 'raw-page'
const postClosed = (): void => {}
const account = new MobileWebAccountSubscriptions({ isActive, postEvent, postClosed })
const workspace = new MobileWebWorkspaceSubscriptions({ isActive, postEvent, postClosed })
const browser = new MobileWebBrowserStreams({
isActive,
workspaceAuthority,
postEvent,
postClosed
})
return [
{
name: 'account',
@@ -51,25 +36,6 @@ function ledgerStarters(): { name: string; start: (subscriptionId: string) => vo
{
name: 'workspace',
start: (subscriptionId) => workspace.start({ requestId: 'r', subscriptionId, client })
},
{
name: 'browser',
start: (subscriptionId) =>
browser.start({
requestId: 'r',
subscriptionId,
payload: {
workspaceId: pageWorkspaceId,
pageId,
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
},
client
})
}
]
}
+6
View File
@@ -16,6 +16,9 @@ import { MOBILE_WEB_SESSION_TERMINAL_CREATION_METHODS } from './mobile-web-sessi
import { MOBILE_WEB_NATIVE_CHAT_FILE_METHODS } from './mobile-web-native-chat-files'
import { MOBILE_WEB_TERMINAL_ACTION_METHODS } from './mobile-web-terminal-actions'
import { MOBILE_WEB_NATIVE_CHAT_STREAM_METHOD } from './mobile-web-native-chat-stream'
import { MOBILE_WEB_BROWSER_INPUT_METHODS } from './mobile-web-browser-input'
import { MOBILE_WEB_BROWSER_NAVIGATION_METHODS } from './mobile-web-browser-navigation'
import { MOBILE_WEB_BROWSER_STREAM_METHODS } from './mobile-web-browser-stream'
import { MOBILE_WEB_NATIVE_CHAT_METHODS } from './mobile-web-native-chat'
import type { RpcAnyMethod } from '../core'
import { STATUS_METHODS } from './status'
@@ -142,6 +145,9 @@ export const ALL_RPC_METHODS: readonly RpcAnyMethod[] = [
MOBILE_WEB_SESSION_CAPABILITIES_METHOD,
...MOBILE_WEB_SESSION_QUICK_COMMAND_METHODS,
MOBILE_WEB_SESSION_BROWSER_CREATE_METHOD,
...MOBILE_WEB_BROWSER_INPUT_METHODS,
...MOBILE_WEB_BROWSER_NAVIGATION_METHODS,
...MOBILE_WEB_BROWSER_STREAM_METHODS,
MOBILE_WEB_NATIVE_CHAT_STREAM_METHOD,
...MOBILE_WEB_PACKAGE_METHODS
]
@@ -0,0 +1,41 @@
import { z } from 'zod'
import type { RpcContext } from '../core'
import { BROWSER_CORE_METHODS } from './browser-core'
import { BROWSER_EXTRA_METHODS } from './browser-extras'
/** The shell rewrites the page's workspace handle into `worktree`; `page` is the host browser page
* id the page already holds from `mobileWeb.session.createBrowser`. */
export const MobileWebBrowserTarget = z.object({
worktree: z.string().min(1).max(4096),
page: z.string().min(1).max(512)
})
export const MOBILE_WEB_BROWSER_APPLIED = { applied: true } as const
export const MobileWebBrowserCoordinate = z.number().finite().min(-100_000).max(100_000)
const commands = new Map(
[...BROWSER_CORE_METHODS, ...BROWSER_EXTRA_METHODS].map((method) => [method.name, method])
)
export async function dispatchMobileWebBrowserCommand(
name: string,
fields: Record<string, unknown>,
context: RpcContext
): Promise<unknown> {
const command = commands.get(name)
if (!command) {
throw new Error('method_not_found')
}
if (context.signal?.aborted) {
throw new Error('runtime_unavailable')
}
return command.handler(command.params!.parse(fields), context)
}
export function mobileWebBrowserTargetFields(params: z.infer<typeof MobileWebBrowserTarget>): {
worktree: string
page: string
} {
return { worktree: params.worktree, page: params.page }
}
@@ -0,0 +1,35 @@
import type { BrowserScreencastFrame } from '../../../../shared/browser-screencast-protocol'
import {
MobileWebBrowserEventSchema,
MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES,
type MobileWebBrowserFrameChunk
} from '../../../../shared/mobile-web/browser-operation-contract'
/** The generic host lane carries JSON only, so a binary screencast frame crosses it as a run of
* base64 chunks the page reassembles. Null when the frame cannot be described within the page
* contract's bounds. */
export function mobileWebBrowserFrameChunks(
frame: BrowserScreencastFrame
): MobileWebBrowserFrameChunk[] | null {
const chunkCount = Math.ceil(frame.image.byteLength / MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES)
const chunks: MobileWebBrowserFrameChunk[] = []
for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex += 1) {
const start = chunkIndex * MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES
const end = Math.min(frame.image.byteLength, start + MOBILE_WEB_BROWSER_FRAME_CHUNK_BYTES)
const parsed = MobileWebBrowserEventSchema.safeParse({
type: 'frameChunk',
frameSequence: frame.seq,
format: frame.format,
metadata: frame.metadata,
imageBytes: frame.image.byteLength,
chunkIndex,
chunkCount,
data: Buffer.from(frame.image.subarray(start, end)).toString('base64')
})
if (!parsed.success || parsed.data.type !== 'frameChunk') {
return null
}
chunks.push(parsed.data)
}
return chunks.length > 0 ? chunks : null
}
@@ -0,0 +1,41 @@
const CAPACITY = 40
const REFILL_PER_SECOND = 20
const IDLE_EVICTION_MS = 60_000
type Bucket = { tokens: number; updatedAt: number }
const buckets = new Map<string, Bucket>()
/** Pointer and keyboard traffic is per-gesture, and the shell forwards it blind, so the desktop is
* the only place the rate is bounded. One bucket per connection, shared by both. */
export function takeMobileWebBrowserInputToken(
connectionId: string | undefined,
now = Date.now()
): boolean {
const key = connectionId ?? 'local'
const bucket = buckets.get(key) ?? { tokens: CAPACITY, updatedAt: now }
if (!buckets.has(key)) {
evictIdleBuckets(now)
}
const refill = ((now - bucket.updatedAt) / 1000) * REFILL_PER_SECOND
bucket.tokens = Math.min(CAPACITY, bucket.tokens + Math.max(0, refill))
bucket.updatedAt = now
buckets.set(key, bucket)
if (bucket.tokens < 1) {
return false
}
bucket.tokens -= 1
return true
}
export function resetMobileWebBrowserInputRateLimit(): void {
buckets.clear()
}
function evictIdleBuckets(now: number): void {
for (const [key, bucket] of buckets) {
if (now - bucket.updatedAt >= IDLE_EVICTION_MS) {
buckets.delete(key)
}
}
}
@@ -0,0 +1,155 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { RpcContext } from '../core'
import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist'
import { MOBILE_WEB_BROWSER_INPUT_METHODS } from './mobile-web-browser-input'
import { resetMobileWebBrowserInputRateLimit } from './mobile-web-browser-input-rate-limit'
const methods = new Map(MOBILE_WEB_BROWSER_INPUT_METHODS.map((method) => [method.name, method]))
const pointer = methods.get('mobileWeb.browser.pointer')!
const keyboard = methods.get('mobileWeb.browser.keyboard')!
const dialog = methods.get('mobileWeb.browser.dialog')!
const TARGET = { worktree: 'id:workspace', page: 'page-1' }
function fixture() {
const runtime = {
browserMouseMove: vi.fn().mockResolvedValue({}),
browserMouseWheel: vi.fn().mockResolvedValue({}),
browserMouseClick: vi.fn().mockResolvedValue({}),
browserMouseDown: vi.fn().mockResolvedValue({}),
browserMouseUp: vi.fn().mockResolvedValue({}),
browserKeypress: vi.fn().mockResolvedValue({}),
browserKeyboardInsertText: vi.fn().mockResolvedValue({}),
browserDialogAccept: vi.fn().mockResolvedValue({}),
browserDialogDismiss: vi.fn().mockResolvedValue({})
}
const context = { runtime, connectionId: 'connection' } as unknown as RpcContext
return { context, runtime }
}
beforeEach(() => resetMobileWebBrowserInputRateLimit())
describe('host-owned browser input', () => {
it('turns one scroll into the host move and wheel pair', async () => {
const f = fixture()
expect(
await pointer.handler(
{ ...TARGET, action: 'scroll', x: 10, y: 20, dx: 0, dy: -40 },
f.context
)
).toEqual({ applied: true })
expect(f.runtime.browserMouseMove).toHaveBeenCalledWith({ ...TARGET, x: 10, y: 20 })
expect(f.runtime.browserMouseWheel).toHaveBeenCalledWith({ ...TARGET, dx: 0, dy: -40 })
})
it('clicks with the requested button, modifiers and radius', async () => {
const f = fixture()
await pointer.handler(
{ ...TARGET, action: 'click', x: 5, y: 6, button: 'right', modifiers: ['shift'], radius: 12 },
f.context
)
expect(f.runtime.browserMouseClick).toHaveBeenCalledWith({
...TARGET,
x: 5,
y: 6,
button: 'right',
modifiers: ['shift'],
radius: 12
})
expect(f.runtime.browserMouseDown).not.toHaveBeenCalled()
})
it('falls back to press and release when a plain click fails', async () => {
const f = fixture()
f.runtime.browserMouseClick.mockRejectedValueOnce(new Error('browser_error'))
await pointer.handler(
{ ...TARGET, action: 'click', x: 5, y: 6, button: 'left', modifiers: [] },
f.context
)
expect(f.runtime.browserMouseMove).toHaveBeenCalledWith({ ...TARGET, x: 5, y: 6 })
expect(f.runtime.browserMouseDown).toHaveBeenCalledWith({ ...TARGET, button: 'left' })
expect(f.runtime.browserMouseUp).toHaveBeenCalledWith({ ...TARGET, button: 'left' })
})
it('never synthesizes a press for a modified click that failed', async () => {
const f = fixture()
f.runtime.browserMouseClick.mockRejectedValueOnce(new Error('browser_error'))
await expect(
pointer.handler(
{ ...TARGET, action: 'click', x: 5, y: 6, button: 'left', modifiers: ['cmd'] },
f.context
)
).rejects.toThrow('browser_error')
expect(f.runtime.browserMouseDown).not.toHaveBeenCalled()
})
it('routes keyboard actions to the matching host command', async () => {
const f = fixture()
expect(
await keyboard.handler({ ...TARGET, action: 'insertText', text: 'hello' }, f.context)
).toEqual({ applied: true })
await keyboard.handler({ ...TARGET, action: 'keypress', key: 'Enter' }, f.context)
expect(f.runtime.browserKeyboardInsertText).toHaveBeenCalledWith({ ...TARGET, text: 'hello' })
expect(f.runtime.browserKeypress).toHaveBeenCalledWith({ ...TARGET, key: 'Enter' })
})
it('routes dialog actions to accept and dismiss', async () => {
const f = fixture()
await dialog.handler({ ...TARGET, action: 'accept' }, f.context)
await dialog.handler({ ...TARGET, action: 'dismiss' }, f.context)
expect(f.runtime.browserDialogAccept).toHaveBeenCalledWith(TARGET)
expect(f.runtime.browserDialogDismiss).toHaveBeenCalledWith(TARGET)
})
it('accepts only the four keys and two buttons the page may send', () => {
expect(keyboard.params!.safeParse({ ...TARGET, action: 'keypress', key: 'F12' }).success).toBe(
false
)
expect(keyboard.params!.safeParse({ ...TARGET, action: 'keypress', key: 'Tab' }).success).toBe(
true
)
expect(
pointer.params!.safeParse({
...TARGET,
action: 'click',
x: 1,
y: 1,
button: 'middle',
modifiers: []
}).success
).toBe(false)
})
it('rate-limits input per connection and lets a second connection through', async () => {
const f = fixture()
const other = { ...f.context, connectionId: 'other' } as RpcContext
const move = { ...TARGET, action: 'scroll', x: 1, y: 1, dx: 0, dy: 1 } as const
let rejected = 0
for (let attempt = 0; attempt < 60; attempt += 1) {
await Promise.resolve(pointer.handler(move, f.context)).catch(() => {
rejected += 1
})
}
expect(rejected).toBeGreaterThan(0)
await expect(pointer.handler(move, other)).resolves.toEqual({ applied: true })
})
it('exposes every input method to the page lane', () => {
for (const name of methods.keys()) {
expect(isMobileWebHostRpcMethod(name), name).toBe(true)
}
})
})
@@ -0,0 +1,139 @@
import { z } from 'zod'
import { defineMethod, type RpcContext } from '../core'
import {
dispatchMobileWebBrowserCommand,
mobileWebBrowserTargetFields,
MOBILE_WEB_BROWSER_APPLIED,
MobileWebBrowserCoordinate,
MobileWebBrowserTarget
} from './mobile-web-browser-command-dispatch'
import { takeMobileWebBrowserInputToken } from './mobile-web-browser-input-rate-limit'
const PointerParams = z.discriminatedUnion('action', [
MobileWebBrowserTarget.extend({
action: z.literal('scroll'),
x: MobileWebBrowserCoordinate,
y: MobileWebBrowserCoordinate,
dx: MobileWebBrowserCoordinate,
dy: MobileWebBrowserCoordinate
}),
MobileWebBrowserTarget.extend({
action: z.literal('click'),
x: MobileWebBrowserCoordinate,
y: MobileWebBrowserCoordinate,
button: z.enum(['left', 'right']),
modifiers: z.array(z.enum(['cmd', 'ctrl', 'alt', 'shift'])).max(4),
radius: z.number().finite().min(0).max(1000).optional()
})
])
const KeyboardParams = z.discriminatedUnion('action', [
MobileWebBrowserTarget.extend({
action: z.literal('insertText'),
text: z
.string()
.min(1)
.max(32 * 1024)
}),
MobileWebBrowserTarget.extend({
action: z.literal('keypress'),
key: z.enum(['Enter', 'Backspace', 'Tab', 'Escape'])
})
])
function requireInputToken(context: RpcContext): void {
if (!takeMobileWebBrowserInputToken(context.connectionId)) {
throw new Error('rate_limited')
}
}
export const MOBILE_WEB_BROWSER_INPUT_METHODS = [
defineMethod({
name: 'mobileWeb.browser.pointer',
params: PointerParams,
handler: async (params, context) => {
requireInputToken(context)
const target = mobileWebBrowserTargetFields(params)
if (params.action === 'scroll') {
await dispatchMobileWebBrowserCommand(
'browser.mouseMove',
{ ...target, x: params.x, y: params.y },
context
)
await dispatchMobileWebBrowserCommand(
'browser.mouseWheel',
{ ...target, dx: params.dx, dy: params.dy },
context
)
return MOBILE_WEB_BROWSER_APPLIED
}
try {
await dispatchMobileWebBrowserCommand(
'browser.mouseClick',
{
...target,
x: params.x,
y: params.y,
button: params.button,
modifiers: params.modifiers,
...(params.radius === undefined ? {} : { radius: params.radius })
},
context
)
} catch (error) {
// A modified click has no press/release equivalent, so only a plain click falls back.
if (params.modifiers.length > 0) {
throw error
}
await dispatchMobileWebBrowserCommand(
'browser.mouseMove',
{ ...target, x: params.x, y: params.y },
context
)
await dispatchMobileWebBrowserCommand(
'browser.mouseDown',
{ ...target, button: params.button },
context
)
await dispatchMobileWebBrowserCommand(
'browser.mouseUp',
{ ...target, button: params.button },
context
)
}
return MOBILE_WEB_BROWSER_APPLIED
}
}),
defineMethod({
name: 'mobileWeb.browser.keyboard',
params: KeyboardParams,
handler: async (params, context) => {
requireInputToken(context)
const target = mobileWebBrowserTargetFields(params)
await (params.action === 'insertText'
? dispatchMobileWebBrowserCommand(
'browser.keyboardInsertText',
{ ...target, text: params.text },
context
)
: dispatchMobileWebBrowserCommand(
'browser.keypress',
{ ...target, key: params.key },
context
))
return MOBILE_WEB_BROWSER_APPLIED
}
}),
defineMethod({
name: 'mobileWeb.browser.dialog',
params: MobileWebBrowserTarget.extend({ action: z.enum(['accept', 'dismiss']) }),
handler: async (params, context) => {
await dispatchMobileWebBrowserCommand(
params.action === 'accept' ? 'browser.dialogAccept' : 'browser.dialogDismiss',
mobileWebBrowserTargetFields(params),
context
)
return MOBILE_WEB_BROWSER_APPLIED
}
})
]
@@ -0,0 +1,66 @@
import { describe, expect, it, vi } from 'vitest'
import type { RpcContext } from '../core'
import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist'
import { MOBILE_WEB_BROWSER_NAVIGATION_METHODS } from './mobile-web-browser-navigation'
const methods = new Map(
MOBILE_WEB_BROWSER_NAVIGATION_METHODS.map((method) => [method.name, method])
)
const navigate = methods.get('mobileWeb.browser.navigate')!
const history = methods.get('mobileWeb.browser.history')!
const TARGET = { worktree: 'id:workspace', page: 'page-1' }
function fixture(landing: unknown) {
const runtime = {
browserGoto: vi.fn().mockResolvedValue(landing),
browserBack: vi.fn().mockResolvedValue({ url: 'https://secret.example/?token=abc' }),
browserForward: vi.fn().mockResolvedValue({ url: 'https://secret.example/' }),
browserReload: vi.fn().mockResolvedValue({ url: 'https://secret.example/' })
}
return { runtime, context: { runtime } as unknown as RpcContext }
}
describe('host-owned browser navigation', () => {
it('strips credentials from the URL the page is told it landed on', async () => {
const f = fixture({ url: 'https://app.example/callback?code=secret&view=1', title: 'App' })
expect(await navigate.handler({ ...TARGET, url: 'https://app.example/' }, f.context)).toEqual({
url: 'https://app.example/callback?view=1'
})
expect(f.runtime.browserGoto).toHaveBeenCalledWith({ ...TARGET, url: 'https://app.example/' })
})
it('answers about:blank when the host reports no usable URL', async () => {
const f = fixture({ title: 'App' })
expect(await navigate.handler({ ...TARGET, url: 'https://app.example/' }, f.context)).toEqual({
url: 'about:blank'
})
})
it('refuses a navigation URL that is not a credential-free http target', () => {
for (const url of ['file:///etc/passwd', 'javascript:alert(1)', 'https://a.example/?token=t']) {
expect(navigate.params!.safeParse({ ...TARGET, url }).success, url).toBe(false)
}
expect(navigate.params!.safeParse({ ...TARGET, url: 'https://a.example/x' }).success).toBe(true)
})
it('acknowledges history moves without echoing the host tab URL', async () => {
const f = fixture({ url: 'https://app.example/' })
for (const action of ['back', 'forward', 'reload'] as const) {
expect(await history.handler({ ...TARGET, action }, f.context)).toEqual({ applied: true })
}
expect(f.runtime.browserBack).toHaveBeenCalledWith(TARGET)
expect(f.runtime.browserForward).toHaveBeenCalledWith(TARGET)
expect(f.runtime.browserReload).toHaveBeenCalledWith(TARGET)
})
it('exposes every navigation method to the page lane', () => {
for (const name of methods.keys()) {
expect(isMobileWebHostRpcMethod(name), name).toBe(true)
}
})
})
@@ -0,0 +1,58 @@
import { z } from 'zod'
import { defineMethod } from '../core'
import {
isMobileWebPageBrowserNavigationUrl,
mobileWebPageBrowserUrl,
MOBILE_WEB_PAGE_BROWSER_URL_MAX_LENGTH
} from '../../../../shared/mobile-web/browser-url-privacy'
import {
dispatchMobileWebBrowserCommand,
mobileWebBrowserTargetFields,
MOBILE_WEB_BROWSER_APPLIED,
MobileWebBrowserTarget
} from './mobile-web-browser-command-dispatch'
const HISTORY_COMMANDS = {
back: 'browser.back',
forward: 'browser.forward',
reload: 'browser.reload'
} as const
export const MOBILE_WEB_BROWSER_NAVIGATION_METHODS = [
defineMethod({
name: 'mobileWeb.browser.navigate',
params: MobileWebBrowserTarget.extend({
url: z
.string()
.min(1)
.max(MOBILE_WEB_PAGE_BROWSER_URL_MAX_LENGTH)
.refine(isMobileWebPageBrowserNavigationUrl, 'Unsupported browser URL')
}),
handler: async (params, context) => {
const result = await dispatchMobileWebBrowserCommand(
'browser.goto',
{ ...mobileWebBrowserTargetFields(params), url: params.url },
context
)
// The landing URL can carry credentials the page must never see, so it is stripped here.
return {
url: mobileWebPageBrowserUrl(
typeof result === 'object' && result !== null && 'url' in result ? result.url : undefined
)
}
}
}),
defineMethod({
name: 'mobileWeb.browser.history',
params: MobileWebBrowserTarget.extend({ action: z.enum(['back', 'forward', 'reload']) }),
handler: async (params, context) => {
// The host result carries the raw tab URL; the page learns the new location from the stream.
await dispatchMobileWebBrowserCommand(
HISTORY_COMMANDS[params.action],
mobileWebBrowserTargetFields(params),
context
)
return MOBILE_WEB_BROWSER_APPLIED
}
})
]
@@ -1,10 +1,12 @@
import {
MobileWebBrowserEventSchema,
type MobileWebBrowserEvent
} from '../../../src/shared/mobile-web/browser-operation-contract'
import { mobileWebPageBrowserUrl } from '../../../src/shared/mobile-web/browser-url-privacy'
} from '../../../../shared/mobile-web/browser-operation-contract'
import { mobileWebPageBrowserUrl } from '../../../../shared/mobile-web/browser-url-privacy'
export function sanitizeMobileWebBrowserEvent(value: unknown): MobileWebBrowserEvent | null {
/** Screencast events carry raw tab URLs, host page ids and driver-supplied text. This is the only
* projection the page sees, so anything not named here never reaches it. */
export function mobileWebBrowserPageEvent(value: unknown): MobileWebBrowserEvent | null {
if (!isRecord(value)) {
return null
}
@@ -0,0 +1,179 @@
import { describe, expect, it, vi } from 'vitest'
import {
BrowserScreencastOpcode,
encodeBrowserScreencastFrame
} from '../../../../shared/browser-screencast-protocol'
import { isStreamingMethod, type RpcContext, type RpcMethod } from '../core'
import { isMobileWebHostRpcMethod } from './mobile-web-host-rpc-allowlist'
import { MOBILE_WEB_BROWSER_STREAM_METHODS } from './mobile-web-browser-stream'
const subscribe = MOBILE_WEB_BROWSER_STREAM_METHODS.find(isStreamingMethod)!
const unsubscribe = MOBILE_WEB_BROWSER_STREAM_METHODS.find(
(method) => !isStreamingMethod(method)
) as RpcMethod
const REQUEST = {
worktree: 'id:workspace',
page: 'page-1',
format: 'jpeg' as const,
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
}
type ScreencastOptions = {
emit: (event: unknown) => void
sendBinary: (bytes: Uint8Array) => boolean | void
signal?: AbortSignal
}
function fixture() {
const cleanups = new Map<string, () => void>()
let host!: ScreencastOptions
let release!: () => void
const started = new Promise<void>((resolve) => {
release = resolve
})
const runtime = {
browserScreencast: vi.fn(async (_params: unknown, options: ScreencastOptions) => {
host = options
release()
await new Promise<void>((resolve) =>
options.signal?.addEventListener('abort', () => resolve())
)
}),
registerSubscriptionCleanup: vi.fn((key: string, cleanup: () => void) =>
cleanups.set(key, cleanup)
),
cleanupSubscription: vi.fn((key: string) => cleanups.get(key)?.())
}
const events: unknown[] = []
const context = { runtime, connectionId: 'connection' } as unknown as RpcContext
const done = subscribe.handler(REQUEST, context, (event) => events.push(event))
return {
context,
done,
events,
runtime,
started,
getHost: () => host
}
}
function frameChunkOf(events: unknown[]) {
return events.filter(
(event): event is { type: string; data: string; chunkCount: number; frameSequence: number } =>
typeof event === 'object' &&
event !== null &&
(event as { type?: string }).type === 'frameChunk'
)
}
describe('host-owned browser screencast', () => {
it('announces the cancel id before any stream event', async () => {
const f = fixture()
await f.started
expect(f.events[0]).toMatchObject({ type: 'ready' })
expect(typeof (f.events[0] as { subscriptionId: string }).subscriptionId).toBe('string')
expect(f.events[0]).not.toHaveProperty('tab')
f.getHost().signal?.dispatchEvent(new Event('abort'))
await f.done
})
it('strips the tab URL and bounds the title before the page sees it', async () => {
const f = fixture()
await f.started
f.getHost().emit({
type: 'ready',
subscriptionId: 'browser-screencast:host',
browserPageId: 'private-page',
tab: {
url: 'https://app.example/?session_token=secret&keep=1',
title: 'x'.repeat(400),
canGoBack: true,
canGoForward: false
}
})
expect(f.events[1]).toEqual({
type: 'ready',
tab: {
url: 'https://app.example/?keep=1',
title: 'x'.repeat(240),
canGoBack: true,
canGoForward: false
}
})
expect(JSON.stringify(f.events)).not.toContain('private-page')
f.getHost().signal?.dispatchEvent(new Event('abort'))
await f.done
})
it('carries a binary frame across the JSON lane as base64 chunks', async () => {
const f = fixture()
await f.started
f.getHost().sendBinary(
encodeBrowserScreencastFrame({
opcode: BrowserScreencastOpcode.Frame,
seq: 7,
format: 'jpeg',
metadata: { imageWidth: 400, imageHeight: 300 },
image: new Uint8Array([1, 2, 3, 4])
})
)
const chunks = frameChunkOf(f.events)
expect(chunks).toHaveLength(1)
expect(chunks[0]).toMatchObject({ frameSequence: 7, chunkCount: 1, chunkIndex: 0 })
expect(Buffer.from(chunks[0]!.data, 'base64')).toEqual(Buffer.from([1, 2, 3, 4]))
f.getHost().signal?.dispatchEvent(new Event('abort'))
await f.done
})
it('ends the stream on a frame the page contract cannot describe', async () => {
const f = fixture()
await f.started
f.getHost().sendBinary(
encodeBrowserScreencastFrame({
opcode: BrowserScreencastOpcode.Frame,
seq: 1,
format: 'jpeg',
// Beyond the page contract's dimension bound.
metadata: { imageWidth: 99_999 },
image: new Uint8Array([9])
})
)
expect(f.events.at(-1)).toEqual({
type: 'error',
message: 'Browser frame cannot be displayed safely.'
})
await f.done
expect(f.events.filter((event) => (event as { type: string }).type === 'end')).toEqual([])
})
it('ends the page stream when the host cleanup for this connection runs', async () => {
const f = fixture()
await f.started
const key = f.runtime.registerSubscriptionCleanup.mock.calls[0]![0] as string
const subscriptionId = (f.events[0] as { subscriptionId: string }).subscriptionId
expect(key).toBe(`mobileWeb.browser:connection:${subscriptionId}`)
await unsubscribe.handler({ subscriptionId }, f.context)
await f.done
expect(f.events.at(-1)).toEqual({ type: 'end' })
})
it('exposes the stream and its cancel to the page lane', () => {
expect(isMobileWebHostRpcMethod('mobileWeb.browser.subscribe')).toBe(true)
expect(isMobileWebHostRpcMethod('mobileWeb.browser.unsubscribe')).toBe(true)
})
})
@@ -0,0 +1,115 @@
import { randomUUID } from 'node:crypto'
import { z } from 'zod'
import { decodeBrowserScreencastFrame } from '../../../../shared/browser-screencast-protocol'
import type { MobileWebBrowserEvent } from '../../../../shared/mobile-web/browser-operation-contract'
import { defineMethod, defineStreamingMethod, isStreamingMethod } from '../core'
import { BROWSER_SCREENCAST_METHODS } from './browser-screencast'
import { MobileWebBrowserTarget } from './mobile-web-browser-command-dispatch'
import { mobileWebBrowserFrameChunks } from './mobile-web-browser-frame-chunks'
import { mobileWebBrowserPageEvent } from './mobile-web-browser-page-event'
const source = BROWSER_SCREENCAST_METHODS.find((method) => method.name === 'browser.screencast')
if (!source || !isStreamingMethod(source)) {
throw new Error('Missing browser screencast stream')
}
const stream = source
function subscriptionKey(connectionId: string | undefined, subscriptionId: string): string {
return `mobileWeb.browser:${connectionId ?? 'local'}:${subscriptionId}`
}
export const MOBILE_WEB_BROWSER_STREAM_METHODS = [
defineStreamingMethod({
name: 'mobileWeb.browser.subscribe',
params: MobileWebBrowserTarget.extend({
format: z.enum(['jpeg', 'png']),
quality: z.number().int().min(1).max(100),
maxWidth: z.number().int().min(1).max(2400),
maxHeight: z.number().int().min(1).max(2160),
viewportWidth: z.number().int().min(1).max(10_000).optional(),
viewportHeight: z.number().int().min(1).max(10_000).optional(),
deviceScaleFactor: z.number().finite().min(0.1).max(10).optional(),
mobile: z.boolean().optional(),
everyNthFrame: z.number().int().min(1).max(60),
minFrameIntervalMs: z.number().int().min(16).max(10_000)
}),
handler: async (params, context, emit) => {
const subscriptionId = randomUUID()
const key = subscriptionKey(context.connectionId, subscriptionId)
const inner = new AbortController()
let closed = false
const close = (event?: MobileWebBrowserEvent): void => {
if (closed) {
return
}
closed = true
if (event) {
emit(event)
}
inner.abort()
}
const end = (): void => close({ type: 'end' })
context.runtime.registerSubscriptionCleanup(key, end, context.connectionId)
context.signal?.addEventListener('abort', end, { once: true })
if (context.signal?.aborted) {
end()
return
}
// The shell learns the cancel id from this frame, so it precedes anything the stream sends.
emit({ type: 'ready', subscriptionId })
const deliverFrame = (bytes: Uint8Array): boolean => {
if (closed) {
return true
}
const frame = decodeBrowserScreencastFrame(bytes)
if (!frame) {
return true
}
const chunks = mobileWebBrowserFrameChunks(frame)
// An error retires the stream on this lane, which is the honest end for a producer whose
// frames the page contract cannot describe at all.
if (!chunks) {
close({ type: 'error', message: 'Browser frame cannot be displayed safely.' })
return true
}
for (const chunk of chunks) {
emit(chunk)
}
return true
}
try {
await stream.handler(
stream.params!.parse(params),
{ ...context, signal: inner.signal, sendBinary: deliverFrame },
(event) => {
if (closed) {
return
}
const projected = mobileWebBrowserPageEvent(event)
if (!projected) {
return
}
if (projected.type === 'end' || projected.type === 'error') {
close(projected)
return
}
emit(projected)
}
)
} finally {
context.signal?.removeEventListener('abort', end)
context.runtime.cleanupSubscription(key)
}
}
}),
defineMethod({
name: 'mobileWeb.browser.unsubscribe',
params: z.object({ subscriptionId: z.string().uuid() }),
handler: (params, context) => {
context.runtime.cleanupSubscription(
subscriptionKey(context.connectionId, params.subscriptionId)
)
return { unsubscribed: true }
}
})
]
@@ -56,6 +56,12 @@ export const MOBILE_WEB_HOST_RPC_METHODS = new Set([
'mobileWeb.session.activate',
'mobileWeb.session.close',
'mobileWeb.session.createBrowser',
'mobileWeb.browser.subscribe',
'mobileWeb.browser.navigate',
'mobileWeb.browser.history',
'mobileWeb.browser.pointer',
'mobileWeb.browser.keyboard',
'mobileWeb.browser.dialog',
'mobileWeb.session.quickCommands',
'mobileWeb.session.quickCommandMutate',
'mobileWeb.session.createQuickCommand',
@@ -53,6 +53,7 @@ it('admits every allowlisted method and cancel through authenticated mobile disp
}
expect(MOBILE_WEB_HOST_RPC_CANCEL_METHODS).toEqual(
new Set([
'mobileWeb.browser.unsubscribe',
'mobileWeb.files.unwatch',
'mobileWeb.nativeChat.unsubscribe',
'mobileWeb.session.unsubscribe'
@@ -25,6 +25,7 @@ import type {
} from '../../shared/mobile-web/browser-operation-contract'
import type { MobileWebBrowserRequestClient } from './mobile-web-browser-request-client'
import { mobileWebBrowserNavigationClientBindings } from './mobile-web-browser-navigation-client-bindings'
import { subscribeMobileWebHostBrowser } from './mobile-web-host-browser-subscription'
import { MobileWebAccountRequestClient } from './mobile-web-account-request-client'
import { MobileWebAgentHistoryRequestClient } from './mobile-web-agent-history-request-client'
import { mobileWebFileClientBindings } from './mobile-web-file-client-bindings'
@@ -264,7 +265,7 @@ export class MobileWebBridgeClient {
onEvent: (event: MobileWebBrowserEvent) => void,
onError: (error: MobileWebBridgeClientError) => void
): MobileWebBridgeSubscription {
return this.subscriptions.subscribeBrowser(payload, onEvent, onError)
return subscribeMobileWebHostBrowser(this.subscriptions, payload, onEvent, onError)
}
receive(message: MobileWebBridgeShellMessage): void {
@@ -19,7 +19,6 @@ import { deliverMobileWebSubscriptionEvent } from './mobile-web-bridge-subscript
import {
accountSubscriptionSetup,
terminalSubscriptionSetup,
browserSubscriptionSetup,
speechSubscriptionSetup,
workspaceSubscriptionSetup,
type MobileWebBridgeSubscriptionSetup
@@ -94,10 +93,6 @@ export class MobileWebBridgeSubscriptionClient {
return this.subscribeWith(hostSubscriptionSetup(...args))
}
subscribeBrowser(...args: Parameters<typeof browserSubscriptionSetup>) {
return this.subscribeWith(browserSubscriptionSetup(...args))
}
subscribeSpeech(...args: Parameters<typeof speechSubscriptionSetup>) {
return this.subscribeWith(speechSubscriptionSetup(...args))
}
@@ -17,12 +17,6 @@ import {
type MobileWebWorkspaceChange
} from '../../shared/mobile-web/bridge-operation-contract'
import type { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import {
MobileWebBrowserEventSchema,
MobileWebBrowserStreamPayloadSchema,
type MobileWebBrowserEvent,
type MobileWebBrowserStreamPayload
} from '../../shared/mobile-web/browser-operation-contract'
import {
MobileWebSpeechEventSchema,
MobileWebSpeechSubscribePayloadSchema,
@@ -67,21 +61,6 @@ export function speechSubscriptionSetup(
}
}
export function browserSubscriptionSetup(
payload: MobileWebBrowserStreamPayload,
onEvent: (event: MobileWebBrowserEvent) => void,
onError: (error: MobileWebBridgeClientError) => void
): MobileWebBridgeSubscriptionSetup {
return {
capability: 'browser',
payload,
payloadSchema: MobileWebBrowserStreamPayloadSchema,
eventSchema: MobileWebBrowserEventSchema,
onEvent: (value) => onEvent(value as MobileWebBrowserEvent),
onError
}
}
export function workspaceSubscriptionSetup(
onEvent: (event: MobileWebWorkspaceChange) => void,
onError: (error: MobileWebBridgeClientError) => void
@@ -0,0 +1,137 @@
import { describe, expect, it, vi } from 'vitest'
import { MobileWebBrowserRequestClient } from './mobile-web-browser-request-client'
import type { MobileWebOneShotRequestClient } from './mobile-web-one-shot-request-client'
const TARGET = { workspaceId: 'workspace-1', pageId: 'browser-1' } as const
function fixture(result: unknown = { applied: true }) {
const request = vi.fn(async (..._args: unknown[]) => result)
const requests = { supports: () => true, request } as unknown as MobileWebOneShotRequestClient
return { request, client: new MobileWebBrowserRequestClient(requests) }
}
function payloads(
request: ReturnType<typeof fixture>['request']
): { method: string; workspaceId?: string; params: Record<string, unknown> }[] {
return request.mock.calls.map(
(call) => call[2] as { method: string; workspaceId?: string; params: Record<string, unknown> }
)
}
describe('page-owned browser commands', () => {
it('names the workspace in the envelope and the page in the params', async () => {
const f = fixture({ url: 'https://app.example/' })
await expect(f.client.navigate({ ...TARGET, url: 'https://app.example/' })).resolves.toEqual({
url: 'https://app.example/'
})
expect(payloads(f.request)).toEqual([
{
method: 'mobileWeb.browser.navigate',
workspaceId: 'workspace-1',
params: { page: 'browser-1', url: 'https://app.example/' }
}
])
})
it('sends one host request per pointer, keyboard and dialog action', async () => {
const f = fixture()
await expect(
f.client.pointer({ ...TARGET, action: 'scroll', x: 1, y: 2, dx: 0, dy: -30 })
).resolves.toBeNull()
await f.client.pointer({
...TARGET,
action: 'click',
x: 3,
y: 4,
button: 'left',
modifiers: ['cmd'],
radius: 8
})
await f.client.keyboard({ ...TARGET, action: 'insertText', text: 'hi' })
await f.client.keyboard({ ...TARGET, action: 'keypress', key: 'Escape' })
await f.client.dialog({ ...TARGET, action: 'accept' })
expect(payloads(f.request)).toEqual([
{
method: 'mobileWeb.browser.pointer',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'scroll', x: 1, y: 2, dx: 0, dy: -30 }
},
{
method: 'mobileWeb.browser.pointer',
workspaceId: 'workspace-1',
params: {
page: 'browser-1',
action: 'click',
x: 3,
y: 4,
button: 'left',
modifiers: ['cmd'],
radius: 8
}
},
{
method: 'mobileWeb.browser.keyboard',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'insertText', text: 'hi' }
},
{
method: 'mobileWeb.browser.keyboard',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'keypress', key: 'Escape' }
},
{
method: 'mobileWeb.browser.dialog',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action: 'accept' }
}
])
})
it('carries back, forward and reload as one history method', async () => {
const f = fixture()
await f.client.back(TARGET)
await f.client.forward(TARGET)
await f.client.reload(TARGET)
expect(payloads(f.request)).toEqual(
['back', 'forward', 'reload'].map((action) => ({
method: 'mobileWeb.browser.history',
workspaceId: 'workspace-1',
params: { page: 'browser-1', action }
}))
)
})
it('rejects a command the host did not acknowledge as applied', async () => {
const f = fixture({ applied: false })
await expect(f.client.reload(TARGET)).rejects.toMatchObject({ code: 'invalid_message' })
})
it('rejects a navigation result with no URL and tolerates an unknown field', async () => {
const f = fixture({ title: 'App' })
await expect(f.client.navigate({ ...TARGET, url: 'https://a.example/' })).rejects.toMatchObject(
{
code: 'invalid_message'
}
)
const forwardCompatible = fixture({ url: 'https://a.example/', title: 'App' })
await expect(
forwardCompatible.client.navigate({ ...TARGET, url: 'https://a.example/' })
).resolves.toEqual({ url: 'https://a.example/' })
})
it('never forwards the opaque workspace handle inside the host params', async () => {
const f = fixture()
await f.client.dialog({ ...TARGET, action: 'dismiss' })
expect(JSON.stringify(payloads(f.request)[0]!.params)).not.toContain('workspace-1')
})
})
@@ -1,32 +1,39 @@
import type { z } from 'zod'
import {
MobileWebBrowserCommandResultSchema,
MobileWebBrowserDialogPayloadSchema,
MobileWebBrowserKeyboardPayloadSchema,
MobileWebBrowserNavigatePayloadSchema,
MobileWebBrowserAckSchema,
MobileWebBrowserNavigateResultSchema,
MobileWebBrowserPointerPayloadSchema,
MobileWebBrowserTargetPayloadSchema,
type MobileWebBrowserDialogPayload,
type MobileWebBrowserKeyboardPayload,
type MobileWebBrowserNavigatePayload,
type MobileWebBrowserPointerPayload,
type MobileWebBrowserTargetPayload
} from '../../shared/mobile-web/browser-operation-contract'
import { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import type { MobileWebBridgeRequestOptions } from './mobile-web-bridge-request-state'
import { requestMobileWebHost } from './mobile-web-host-request-client'
import type { MobileWebOneShotRequestClient } from './mobile-web-one-shot-request-client'
/** The workspace handle rides in the host request envelope, and the browser page id is the only
* target field the desktop wrapper reads. */
function hostParams<TPayload extends MobileWebBrowserTargetPayload>({
workspaceId: _workspaceId,
pageId,
...fields
}: TPayload): Record<string, unknown> {
return { page: pageId, ...fields }
}
export class MobileWebBrowserRequestClient {
constructor(private readonly requests: MobileWebOneShotRequestClient) {}
navigate(
async navigate(
payload: MobileWebBrowserNavigatePayload,
options?: MobileWebBridgeRequestOptions
): Promise<{ url: string }> {
return this.requests.request(
'browser',
'navigate',
return this.send(
'mobileWeb.browser.navigate',
payload,
MobileWebBrowserNavigatePayloadSchema,
hostParams(payload),
MobileWebBrowserNavigateResultSchema,
options
)
@@ -36,77 +43,85 @@ export class MobileWebBrowserRequestClient {
payload: MobileWebBrowserPointerPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
'pointer',
payload,
MobileWebBrowserPointerPayloadSchema,
MobileWebBrowserCommandResultSchema,
options
)
return this.command('mobileWeb.browser.pointer', payload, hostParams(payload), options)
}
keyboard(
payload: MobileWebBrowserKeyboardPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
'keyboard',
payload,
MobileWebBrowserKeyboardPayloadSchema,
MobileWebBrowserCommandResultSchema,
options
)
return this.command('mobileWeb.browser.keyboard', payload, hostParams(payload), options)
}
dialog(
payload: MobileWebBrowserDialogPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
'dialog',
payload,
MobileWebBrowserDialogPayloadSchema,
MobileWebBrowserCommandResultSchema,
options
)
return this.command('mobileWeb.browser.dialog', payload, hostParams(payload), options)
}
back(
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.command('back', payload, options)
return this.history('back', payload, options)
}
forward(
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.command('forward', payload, options)
return this.history('forward', payload, options)
}
reload(
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.command('reload', payload, options)
return this.history('reload', payload, options)
}
private command(
operation: 'back' | 'forward' | 'reload',
private history(
action: 'back' | 'forward' | 'reload',
payload: MobileWebBrowserTargetPayload,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
return this.requests.request(
'browser',
operation,
return this.command(
'mobileWeb.browser.history',
payload,
MobileWebBrowserTargetPayloadSchema,
MobileWebBrowserCommandResultSchema,
{ ...hostParams(payload), action },
options
)
}
private async command(
method: string,
payload: MobileWebBrowserTargetPayload,
params: Record<string, unknown>,
options?: MobileWebBridgeRequestOptions
): Promise<null> {
await this.send(method, payload, params, MobileWebBrowserAckSchema, options)
return null
}
private async send<TResult>(
method: string,
payload: MobileWebBrowserTargetPayload,
params: Record<string, unknown>,
schema: z.ZodType<TResult>,
options?: MobileWebBridgeRequestOptions
): Promise<TResult> {
const result = await requestMobileWebHost(
this.requests,
method,
payload.workspaceId,
params,
options
)
const parsed = schema.safeParse(result)
if (!parsed.success) {
throw new MobileWebBridgeClientError('invalid_message', false)
}
return parsed.data
}
}
@@ -0,0 +1,100 @@
import { describe, expect, it, vi } from 'vitest'
import type { MobileWebBrowserEvent } from '../../shared/mobile-web/browser-operation-contract'
import type { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import type { MobileWebBridgeSubscriptionClient } from './mobile-web-bridge-subscription-client'
import { subscribeMobileWebHostBrowser } from './mobile-web-host-browser-subscription'
const PAYLOAD = {
workspaceId: 'workspace-1',
pageId: 'browser-1',
format: 'jpeg' as const,
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
}
function fixture() {
const unsubscribe = vi.fn()
let emit: (event: unknown) => void = () => {}
const subscribeHost = vi.fn((_payload: unknown, onEvent: (event: unknown) => void) => {
emit = onEvent
return { ready: Promise.resolve(), unsubscribe }
})
const subscriptions = { subscribeHost } as unknown as MobileWebBridgeSubscriptionClient
const events: MobileWebBrowserEvent[] = []
const errors: MobileWebBridgeClientError[] = []
const subscription = subscribeMobileWebHostBrowser(
subscriptions,
PAYLOAD,
(event) => events.push(event),
(error) => errors.push(error)
)
return { emit: (event: unknown) => emit(event), errors, events, subscribeHost, subscription }
}
describe('page-owned browser stream', () => {
it('opens the host stream with the page id and the screencast request', () => {
const f = fixture()
expect(f.subscribeHost.mock.calls[0]![0]).toEqual({
method: 'mobileWeb.browser.subscribe',
workspaceId: 'workspace-1',
params: {
page: 'browser-1',
format: 'jpeg',
quality: 72,
maxWidth: 800,
maxHeight: 600,
everyNthFrame: 1,
minFrameIntervalMs: 100
}
})
})
it('swallows the lane handshake and delivers the browser ready that carries tab state', () => {
const f = fixture()
const tab = { url: 'https://a.example/', title: 'A', canGoBack: false, canGoForward: false }
f.emit({ type: 'ready', subscriptionId: 'lane-id' })
f.emit({ type: 'ready', tab })
expect(f.events).toEqual([{ type: 'ready', tab }])
expect(f.errors).toEqual([])
})
it('reports an event the browser contract cannot describe', () => {
const f = fixture()
f.emit({ type: 'frameChunk', chunkIndex: 4, chunkCount: 1 })
expect(f.events).toEqual([])
expect(f.errors.map((error) => error.code)).toEqual(['invalid_message'])
})
it('stops delivering once the page unsubscribes', () => {
const f = fixture()
f.subscription.unsubscribe()
f.emit({ type: 'dialogClosed' })
expect(f.events).toEqual([])
})
it('refuses a screencast request the contract does not admit', async () => {
const errors: MobileWebBridgeClientError[] = []
const subscribeHost = vi.fn()
const subscription = subscribeMobileWebHostBrowser(
{ subscribeHost } as unknown as MobileWebBridgeSubscriptionClient,
{ ...PAYLOAD, quality: 0 },
() => {},
(error) => errors.push(error)
)
await expect(subscription.ready).rejects.toMatchObject({ code: 'invalid_request' })
await Promise.resolve()
expect(subscribeHost).not.toHaveBeenCalled()
expect(errors.map((error) => error.code)).toEqual(['invalid_request'])
})
})
@@ -0,0 +1,70 @@
import {
MobileWebBrowserEventSchema,
MobileWebBrowserStreamPayloadSchema,
type MobileWebBrowserEvent,
type MobileWebBrowserStreamPayload
} from '../../shared/mobile-web/browser-operation-contract'
import { MobileWebBridgeClientError } from './mobile-web-bridge-client-error'
import type { MobileWebBridgeSubscription } from './mobile-web-bridge-subscription'
import type { MobileWebBridgeSubscriptionClient } from './mobile-web-bridge-subscription-client'
export type MobileWebBrowserSubscriptionArgs = [
payload: MobileWebBrowserStreamPayload,
onEvent: (event: MobileWebBrowserEvent) => void,
onError: (error: MobileWebBridgeClientError) => void
]
export function subscribeMobileWebHostBrowser(
subscriptions: MobileWebBridgeSubscriptionClient,
...[payload, onEvent, onError]: MobileWebBrowserSubscriptionArgs
): MobileWebBridgeSubscription {
if (!MobileWebBrowserStreamPayloadSchema.safeParse(payload).success) {
const error = new MobileWebBridgeClientError('invalid_request', false)
queueMicrotask(() => onError(error))
return { ready: Promise.reject(error), unsubscribe() {} }
}
const { workspaceId, pageId, ...request } = payload
let cancelled = false
const current = subscriptions.subscribeHost(
{
method: 'mobileWeb.browser.subscribe',
workspaceId,
params: { page: pageId, ...request }
},
(event) => {
if (cancelled || isSubscriptionHandshake(event)) {
return
}
const parsed = MobileWebBrowserEventSchema.safeParse(event)
if (!parsed.success) {
onError(new MobileWebBridgeClientError('invalid_message', false))
return
}
onEvent(parsed.data)
},
(error) => {
if (!cancelled) {
onError(error)
}
}
)
return {
ready: current.ready,
unsubscribe() {
cancelled = true
current.unsubscribe()
}
}
}
/** The lane opens with a bare `ready` carrying the cancel id; the browser's own `ready` carries
* tab state, which is what the pane waits for. */
function isSubscriptionHandshake(event: unknown): boolean {
return (
typeof event === 'object' &&
event !== null &&
'type' in event &&
event.type === 'ready' &&
!('tab' in event)
)
}
@@ -134,16 +134,6 @@ export const MOBILE_WEB_BRIDGE_OPERATIONS = {
createProviderIssue: 'mutation',
updateIssueSource: 'mutation'
},
browser: {
subscribe: 'subscription',
navigate: 'mutation',
back: 'mutation',
forward: 'mutation',
reload: 'mutation',
dialog: 'mutation',
pointer: 'mutation',
keyboard: 'mutation'
},
account: {
snapshot: 'read',
select: 'mutation',
@@ -92,13 +92,14 @@ export const MobileWebBrowserDialogPayloadSchema = MobileWebBrowserTargetSchema.
action: z.enum(['accept', 'dismiss'])
}).strict()
export const MobileWebBrowserNavigateResultSchema = z
.object({
url: z.string().min(1).max(MOBILE_WEB_BROWSER_URL_MAX_LENGTH)
})
.strict()
// Desktop-produced results stay open: a newer host may add a field a cached page has never seen.
export const MobileWebBrowserNavigateResultSchema = z.object({
url: z.string().min(1).max(MOBILE_WEB_BROWSER_URL_MAX_LENGTH)
})
export const MobileWebBrowserCommandResultSchema = z.null()
/** Every browser command the page issues answers with this acknowledgement; the host result
* carries the raw tab URL, which the desktop wrapper never forwards. */
export const MobileWebBrowserAckSchema = z.object({ applied: z.literal(true) })
const MobileWebBrowserTabStateSchema = z
.object({