perf(browser): credit screencast frames to the phone (#39189)

This commit is contained in:
Kelvin Amoaba
2026-11-07 04:57:47 +00:00
committed by GitHub
parent b26ad4bd58
commit 1813a01fa6
17 changed files with 471 additions and 38 deletions
@@ -70,21 +70,25 @@ async function renderPane(): Promise<{
renderer: ReactTestRenderer
stream: Subscription | undefined
subscriptions: Subscription[]
sendRequest: ReturnType<typeof vi.fn>
}> {
pageCounter += 1
const subscriptions: Subscription[] = []
const client = {
subscribe: (
_method: string,
_params: unknown,
listener: (payload: unknown) => void,
options?: { onBinaryFrame?: (frame: BrowserScreencastFrame) => void }
) => {
const sendRequest = vi.fn().mockResolvedValue({})
const client: RpcClient = {
sendRequest,
subscribe: (_method, _params, listener, options) => {
subscriptions.push({ listener, onBinaryFrame: options?.onBinaryFrame })
return () => {}
},
request: vi.fn()
} as unknown as RpcClient
updateTerminalSubscriptionViewport: vi.fn(),
getState: () => 'connected',
getReconnectAttempt: () => 0,
getLastConnectedAt: () => null,
onStateChange: () => () => {},
notifyForeground: vi.fn(),
close: vi.fn()
}
const tab: MobileBrowserTab = {
type: 'browser',
@@ -129,18 +133,19 @@ async function renderPane(): Promise<{
act(() => {
viewport.props.onLayout({ nativeEvent: { layout: { width: 360, height: 640 } } })
})
return { renderer: mounted, stream: subscriptions[0], subscriptions }
return { renderer: mounted, stream: subscriptions[0], subscriptions, sendRequest }
}
async function renderStreamingPane(): Promise<{
renderer: ReactTestRenderer
stream: Subscription
sendRequest: ReturnType<typeof vi.fn>
}> {
const { renderer, stream } = await renderPane()
const { renderer, stream, sendRequest } = await renderPane()
if (!stream) {
throw new Error('browser.screencast subscription not created')
}
return { renderer, stream }
return { renderer, stream, sendRequest }
}
function errorMessages(renderer: ReactTestRenderer): string[] {
@@ -196,3 +201,26 @@ describe('MobileBrowserPane against a shell with no binary screencast lane', ()
}
})
})
describe('MobileBrowserPane frame acks', () => {
// Old hosts strip the opt-in and would refuse the unknown ack method.
it('acks received frames only after ready echoes frameAck', async () => {
const legacy = await renderStreamingPane()
act(() => {
legacy.stream.listener({ type: 'ready', subscriptionId: 'sub-legacy', tab: {} })
legacy.stream.onBinaryFrame?.(makeFrame())
})
expect(legacy.sendRequest).not.toHaveBeenCalled()
const credited = await renderStreamingPane()
act(() => {
credited.stream.listener({ type: 'ready', subscriptionId: 'sub-1', frameAck: true, tab: {} })
credited.stream.onBinaryFrame?.(makeFrame())
})
expect(credited.sendRequest).toHaveBeenCalledWith('browser.screencast.ack', {
worktree: `id:wt-${pageCounter}`,
subscriptionId: 'sub-1',
seq: 1
})
})
})
@@ -205,17 +205,25 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
startupTimer = null
}
}
const worktree = `id:${worktreeId}`
// Set once `ready` echoes frameAck; hosts that strip the opt-in never get an ack.
let ackSubscriptionId: string | null = null
const unsubscribe = client.subscribe(
'browser.screencast',
{
worktree: `id:${worktreeId}`,
worktree,
page: tab.browserPageId,
...streamRequest
...streamRequest,
frameAck: 1
},
(payload) => {
if (streamGenerationRef.current !== generation) {
return
}
const ready = readScreencastReady(payload)
if (ready) {
ackSubscriptionId = ready.frameAck === true ? ready.subscriptionId : null
}
handleBrowserScreencastEvent({
busyRef,
clearStartupTimer,
@@ -234,6 +242,16 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
return
}
clearStartupTimer()
if (ackSubscriptionId) {
// Cumulative, so a frame the shell dropped is covered by the next one's ack.
void client
.sendRequest('browser.screencast.ack', {
worktree,
subscriptionId: ackSubscriptionId,
seq: frame.seq
})
.catch(() => {})
}
if (cacheKey) {
framePacer.push(frame, cacheKey)
}
@@ -269,3 +287,22 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
sendBrowserRequest
}
}
function readScreencastReady(
payload: unknown
): { subscriptionId: string; frameAck: unknown } | null {
if (
typeof payload !== 'object' ||
payload === null ||
!('type' in payload) ||
payload.type !== 'ready' ||
!('subscriptionId' in payload) ||
typeof payload.subscriptionId !== 'string'
) {
return null
}
return {
subscriptionId: payload.subscriptionId,
frameAck: 'frameAck' in payload ? payload.frameAck : undefined
}
}
@@ -14,7 +14,10 @@ const BROWSER_RPC_METHODS = [
// Known Log-suffix mismatches: remove entries when fixed; never add entries to hide new failures.
const UNROUTABLE_RUNTIME_COMMANDS = new Set(['browserConsoleLog', 'browserNetworkLog'])
const RPC_METHODS_WITHOUT_BROWSER_COMMAND = new Set(['browser.screencast.unsubscribe'])
const RPC_METHODS_WITHOUT_BROWSER_COMMAND = new Set([
'browser.screencast.unsubscribe',
'browser.screencast.ack'
])
async function runtimeCommandFor(
method: (typeof BROWSER_RPC_METHODS)[number]
@@ -5,6 +5,7 @@ import type {
BrowserScreencastViewport
} from '../../../browser/cdp/screencast/browser-screencast-stream-types'
import type { ScreencastSubscriberDeliveryState } from './browser-screencast-ghost-subscriber-eviction'
import type { BrowserScreencastFrameCredit } from './browser-screencast-frame-credit'
import type { RpcBinarySender } from '../../rpc/rpc-binary-sender'
import type { BrowserCommandTargetParams } from '../commands/browser-command-contracts'
@@ -19,6 +20,8 @@ export type BrowserScreencastParams = {
mobile?: boolean
everyNthFrame?: number
minFrameIntervalMs?: number
// The viewer acks frames with browser.screencast.ack; `ready` echoes `frameAck: true`.
frameAck?: 1
} & BrowserCommandTargetParams
export type BrowserScreencastStartResult = {
@@ -42,6 +45,8 @@ export type ActiveBrowserScreencastSubscriber = {
// Device identity survives reconnects; connection ids cannot replace stale subscriptions.
pairedDeviceId?: string
delivery: ScreencastSubscriberDeliveryState
// Present when the viewer acks frames; `release` unregisters it from acks.
frameCredit: { credit: BrowserScreencastFrameCredit; release: () => void } | null
}
export type ActiveBrowserScreencastPage = {
@@ -0,0 +1,137 @@
// Frames a phone may have in flight before it acks one. Two cover decode and one round trip on a fast link.
const BASE_WINDOW_FRAMES = 2
// A backstop only: the queue-delay test bounds buffering. 16 keeps 10 fps up to a 1.6 s round trip.
const MAX_WINDOW_FRAMES = 16
// Beyond the base window, a frame goes out only while the link has queued less than this.
const QUEUE_DELAY_ALLOWANCE_MS = 100
// The shell drops oversize frames unannounced; probe with one frame, backing off, when acks stop.
const STALL_PROBE_MS = 2_000
const STALL_PROBE_MAX_MS = 8_000
export type BrowserScreencastFrameCredit = {
canSend: () => boolean
recordSent: (seq: number) => void
/** Cumulative: credits every sent frame up to and including `seq`. */
acknowledge: (seq: number) => void
dispose: () => void
}
type SentFrame = { seq: number; sentAt: number }
/** Bounds a viewer's unacked frames by queueing delay rather than a fixed count, so a long but
* idle link keeps its frame rate while a slow one stops filling buffers between host and phone. */
export function createBrowserScreencastFrameCredit(options: {
/** Credit opened up: send the newest held frame, if any. */
onCreditAvailable: () => void
}): BrowserScreencastFrameCredit {
const inFlight: SentFrame[] = []
// Lowest send-to-ack time seen; never raised, so queueing delay cannot inflate the window.
let minRttMs: number | null = null
let probeDelayMs = STALL_PROBE_MS
let probeTimer: ReturnType<typeof setTimeout> | null = null
let probeReady = false
const clearProbe = (): void => {
if (probeTimer) {
clearTimeout(probeTimer)
probeTimer = null
}
}
const armProbe = (): void => {
if (probeTimer) {
return
}
probeTimer = setTimeout(() => {
probeTimer = null
probeReady = true
probeDelayMs = Math.min(probeDelayMs * 2, STALL_PROBE_MAX_MS)
options.onCreditAvailable()
}, probeDelayMs)
}
const hasCredit = (): boolean => {
if (probeReady || inFlight.length < BASE_WINDOW_FRAMES) {
return true
}
if (inFlight.length >= MAX_WINDOW_FRAMES || minRttMs === null) {
return false
}
return Date.now() - inFlight[0]!.sentAt < minRttMs + QUEUE_DELAY_ALLOWANCE_MS
}
return {
canSend() {
if (hasCredit()) {
return true
}
armProbe()
return false
},
recordSent(seq) {
probeReady = false
inFlight.push({ seq, sentAt: Date.now() })
// A probe past the cap means older frames are presumed lost.
if (inFlight.length > MAX_WINDOW_FRAMES) {
inFlight.shift()
}
},
acknowledge(seq) {
// inFlight is in seq order, so the acked frames are a prefix.
let count = 0
while (count < inFlight.length && inFlight[count]!.seq <= seq) {
count += 1
}
if (count === 0) {
return
}
const newest = inFlight[count - 1]!
if (newest.seq === seq) {
const rtt = Date.now() - newest.sentAt
minRttMs = minRttMs === null ? rtt : Math.min(minRttMs, rtt)
}
inFlight.splice(0, count)
clearProbe()
probeDelayMs = STALL_PROBE_MS
options.onCreditAvailable()
},
dispose: clearProbe
}
}
type CreditedSubscription = {
pairedDeviceId: string | undefined
credit: BrowserScreencastFrameCredit
}
// Keyed by subscription id: the relay keeps it intact, and it is unguessable per viewer.
const creditedSubscriptions = new Map<string, CreditedSubscription>()
export function registerBrowserScreencastFrameCredit(
subscriptionId: string,
pairedDeviceId: string | undefined,
credit: BrowserScreencastFrameCredit
): () => void {
const entry = { pairedDeviceId, credit }
creditedSubscriptions.set(subscriptionId, entry)
return () => {
if (creditedSubscriptions.get(subscriptionId) === entry) {
creditedSubscriptions.delete(subscriptionId)
}
credit.dispose()
}
}
/** False when the subscription ended or belongs to another device; late acks are normal. */
export function acknowledgeBrowserScreencastFrame(
subscriptionId: string,
pairedDeviceId: string | undefined,
seq: number
): boolean {
const entry = creditedSubscriptions.get(subscriptionId)
if (!entry || entry.pairedDeviceId !== pairedDeviceId) {
return false
}
entry.credit.acknowledge(seq)
return true
}
@@ -1,6 +1,10 @@
import { readBrowserScreencastFrameSeq } from '../../../../shared/browser-screencast-protocol'
import type { ActiveBrowserScreencastSubscriber } from './browser-screencast-contracts'
import { recordScreencastSubscriberSend } from './browser-screencast-ghost-subscriber-eviction'
import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission'
import {
sendRemoteBrowserScreencastFrame,
type ScreencastFrameSendOutcome
} from './remote-browser-screencast-frame-admission'
// Match the frame pacer's backpressure retry interval.
export const SCREENCAST_PENDING_FRAME_RETRY_MS = 50
@@ -10,7 +14,12 @@ export function deliverScreencastSubscriberFrame(
subscriber: ActiveBrowserScreencastSubscriber,
bytes: Uint8Array<ArrayBufferLike>
): void {
const outcome = sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes)
// Held for credit, not refused: the viewer's ack or the stall probe sends the newest held frame.
if (subscriber.frameCredit && !subscriber.frameCredit.credit.canSend()) {
subscriber.pendingFrame = bytes
return
}
const outcome = sendSubscriberFrame(subscriber, bytes)
subscriber.pendingFrame = outcome === 'handled' ? null : bytes
// Backlog proves an active socket, not a ghost.
if (outcome !== 'backlogged') {
@@ -21,6 +30,14 @@ export function deliverScreencastSubscriberFrame(
}
}
/** Credit opened up: send the held frame now instead of on the next produced one. */
export function flushCreditedScreencastFrame(subscriber: ActiveBrowserScreencastSubscriber): void {
if (subscriber.pendingFrameRetry) {
return
}
retryPendingFrame(subscriber)
}
export function cancelScreencastSubscriberFrameRetry(
subscriber: ActiveBrowserScreencastSubscriber
): void {
@@ -29,6 +46,36 @@ export function cancelScreencastSubscriberFrameRetry(
subscriber.pendingFrameRetry = null
}
subscriber.pendingFrame = null
subscriber.frameCredit?.release()
subscriber.frameCredit = null
}
function sendSubscriberFrame(
subscriber: ActiveBrowserScreencastSubscriber,
bytes: Uint8Array<ArrayBufferLike>
): ScreencastFrameSendOutcome {
const outcome = sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes)
if (outcome === 'handled' && subscriber.frameCredit) {
const seq = readBrowserScreencastFrameSeq(bytes)
if (seq !== null) {
subscriber.frameCredit.credit.recordSent(seq)
}
}
return outcome
}
function retryPendingFrame(subscriber: ActiveBrowserScreencastSubscriber): void {
const bytes = subscriber.pendingFrame
if (!bytes || (subscriber.frameCredit && !subscriber.frameCredit.credit.canSend())) {
return
}
// Count ghost refusals only on produced frames, never retries.
if (sendSubscriberFrame(subscriber, bytes) === 'handled') {
subscriber.pendingFrame = null
subscriber.delivery = recordScreencastSubscriberSend(subscriber.delivery, true)
} else {
schedulePendingFrameRetry(subscriber)
}
}
function schedulePendingFrameRetry(subscriber: ActiveBrowserScreencastSubscriber): void {
@@ -37,16 +84,6 @@ function schedulePendingFrameRetry(subscriber: ActiveBrowserScreencastSubscriber
}
subscriber.pendingFrameRetry = setTimeout(() => {
subscriber.pendingFrameRetry = null
const bytes = subscriber.pendingFrame
if (!bytes) {
return
}
// Count ghost refusals only on produced frames, never retries.
if (sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes) === 'handled') {
subscriber.pendingFrame = null
subscriber.delivery = recordScreencastSubscriberSend(subscriber.delivery, true)
} else {
schedulePendingFrameRetry(subscriber)
}
retryPendingFrame(subscriber)
}, SCREENCAST_PENDING_FRAME_RETRY_MS)
}
@@ -6,8 +6,13 @@ import { rendererPublicationThrottle } from '../../../window/renderer-publicatio
import { mintBrowserScreencastSubscriptionId } from './browser-screencast-subscription-id'
import {
cancelScreencastSubscriberFrameRetry,
deliverScreencastSubscriberFrame
deliverScreencastSubscriberFrame,
flushCreditedScreencastFrame
} from './browser-screencast-subscriber-frame-delivery'
import {
createBrowserScreencastFrameCredit,
registerBrowserScreencastFrameCredit
} from './browser-screencast-frame-credit'
import {
INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY,
screencastSubscriberIsGhost
@@ -151,7 +156,7 @@ export class RuntimeBrowserScreencasts {
})
.catch(() => {})
}
active.subscribers.set(subscriptionId, {
const subscriber: ActiveBrowserScreencastSubscriber = {
sendBinary: stream.sendBinary,
emit: stream.emit,
done: subscriberDone,
@@ -161,8 +166,19 @@ export class RuntimeBrowserScreencasts {
pendingFrame: null,
pendingFrameRetry: null,
pairedDeviceId: stream.pairedDeviceId,
delivery: INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY
})
delivery: INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY,
frameCredit: null
}
if (params.frameAck === 1) {
const credit = createBrowserScreencastFrameCredit({
onCreditAvailable: () => flushCreditedScreencastFrame(subscriber)
})
subscriber.frameCredit = {
credit,
release: registerBrowserScreencastFrameCredit(subscriptionId, stream.pairedDeviceId, credit)
}
}
active.subscribers.set(subscriptionId, subscriber)
// A sizeless viewport owner would clear emulation for every viewer.
if (hasScreencastViewportSize(viewport)) {
active.viewportOwnerSubscriptionId = subscriptionId
@@ -172,6 +188,7 @@ export class RuntimeBrowserScreencasts {
session = await active.started
} catch (error) {
active.subscribers.delete(subscriptionId)
cancelScreencastSubscriberFrameRetry(subscriber)
resolveSubscriberDone()
throw error
}
@@ -219,7 +236,8 @@ export class RuntimeBrowserScreencasts {
subscriptionId,
browserPageId,
format: active.format,
tab: this.catalog.describeBrowserTab(browserPageId, target.worktreeId)
tab: this.catalog.describeBrowserTab(browserPageId, target.worktreeId),
...(subscriber.frameCredit ? { frameAck: true as const } : {})
}
}
}
@@ -0,0 +1,133 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
BrowserScreencastOpcode,
decodeBrowserScreencastFrame,
encodeBrowserScreencastFrame
} from '../../../../shared/browser-screencast-protocol'
import { acknowledgeBrowserScreencastFrame } from './browser-screencast-frame-credit'
import {
createSinglePageBrowserCommandsHost,
createStreamedGuest
} from '../client-pages/single-page-browser-commands-host-test-double'
const { webContentsFromId, startBrowserScreencast } = vi.hoisted(() => ({
webContentsFromId: vi.fn(),
startBrowserScreencast: vi.fn()
}))
vi.mock('electron', () => ({
ipcMain: { on: vi.fn(), removeListener: vi.fn() },
webContents: { fromId: webContentsFromId }
}))
vi.mock('../../../browser/cdp/screencast/browser-screencast-stream', () => ({
startBrowserScreencast
}))
const PHONE = 'phone-1'
function frame(seq: number): Uint8Array {
return encodeBrowserScreencastFrame({
opcode: BrowserScreencastOpcode.Frame,
seq,
format: 'jpeg',
metadata: {},
image: new Uint8Array([seq])
})
}
async function subscribe(frameAck: boolean) {
const { RuntimeBrowserCommands } = await import('../commands/browser-commands')
startBrowserScreencast.mockResolvedValue({
stop: vi.fn(),
done: new Promise<void>(() => {}),
updateViewport: vi.fn(async () => {})
})
const sent: number[] = []
const sentAt: number[] = []
const commands = new RuntimeBrowserCommands(createSinglePageBrowserCommandsHost())
const subscription = await commands.browserScreencast(
{ worktree: 'id:wt-1', page: 'page-1', format: 'jpeg', ...(frameAck ? { frameAck: 1 } : {}) },
{
pairedDeviceId: PHONE,
sendBinary: (bytes) => {
sent.push(decodeBrowserScreencastFrame(bytes)!.seq)
sentAt.push(Date.now())
return true
}
}
)
const onFrame: (bytes: Uint8Array) => unknown = startBrowserScreencast.mock.calls[0][1].onFrame
const ack = (seq: number, device = PHONE) =>
acknowledgeBrowserScreencastFrame(subscription.subscriptionId, device, seq)
return { subscription, onFrame, sent, sentAt, ack }
}
describe('screencast frame credit', () => {
beforeEach(() => {
vi.useFakeTimers()
webContentsFromId.mockReset()
webContentsFromId.mockReturnValue(createStreamedGuest())
startBrowserScreencast.mockReset()
})
afterEach(() => {
vi.useRealTimers()
})
it('sends every frame and echoes nothing when the viewer does not ask for credit', async () => {
const { subscription, onFrame, sent } = await subscribe(false)
for (let seq = 0; seq < 5; seq++) {
onFrame(frame(seq))
}
expect(subscription.ready.frameAck).toBeUndefined()
expect(sent).toEqual([0, 1, 2, 3, 4])
subscription.session.stop()
})
it('holds the newest frame while two are unacked and sends it on a cumulative ack', async () => {
const { subscription, onFrame, sent, ack } = await subscribe(true)
expect(subscription.ready.frameAck).toBe(true)
for (let seq = 0; seq < 4; seq++) {
onFrame(frame(seq))
}
expect(sent).toEqual([0, 1])
expect(ack(1, 'other-phone')).toBe(false)
expect(sent).toEqual([0, 1])
expect(ack(1)).toBe(true)
expect(sent).toEqual([0, 1, 3])
subscription.session.stop()
})
// #25263: a fixed two-frame window delivered about a third of 10 fps on an idle 600 ms link.
it('keeps 10 fps on an uncongested 600 ms round trip', async () => {
const { subscription, onFrame, sent, sentAt, ack } = await subscribe(true)
let acked = 0
for (let tick = 0; tick < 100; tick++) {
onFrame(frame(tick))
for (let step = 0; step < 10; step++) {
await vi.advanceTimersByTimeAsync(10)
// Each frame's ack lands 600 ms after it went out.
while (acked < sent.length && sentAt[acked]! + 600 <= Date.now()) {
ack(sent[acked]!)
acked += 1
}
}
}
// Only frames 2–4 are skipped, superseded while the first ack is still in flight.
expect(sent).toEqual([0, 1, ...Array.from({ length: 95 }, (_, index) => index + 5)])
subscription.session.stop()
})
// #25263: a 2 s expiry let ten unacked frames through in 10 s. Probes back off to one per 8 s.
it('sends one probe frame per backed-off stall when no ack ever comes', async () => {
const { subscription, onFrame, sent } = await subscribe(true)
for (let tick = 0; tick < 300; tick++) {
onFrame(frame(tick))
await vi.advanceTimersByTimeAsync(100)
}
// The base window, then probes 2 s, 4 s, 8 s and 8 s after each hold (t = 2.2, 6.2, 14.2, 22.2 s).
expect(sent).toEqual([0, 1, 21, 61, 141, 221])
subscription.session.stop()
})
})
@@ -24,7 +24,8 @@ export function localHostedBrowserPageRequest(
// Server workspace selectors cannot resolve here; page ids are stable.
const { worktree: _serverSelector, ...params } = parsed.data
const page =
request.method === 'browser.screencast.unsubscribe'
request.method === 'browser.screencast.unsubscribe' ||
request.method === 'browser.screencast.ack'
? browserScreencastSubscriptionPageId(params.subscriptionId ?? '')
: params.page || null
if (page === null || !hostsBrowserPage(environmentId, page)) {
@@ -4,7 +4,11 @@ import { BrowserError } from '../../../../browser/browser-error'
import '../../../../../shared/runtime-types'
import { BROWSER_UNAVAILABLE_ERROR_CODE } from '../../../../../shared/runtime-session-contracts'
import { runtimeBrowserCommandsFactoryIsAvailable } from '../../../browser/installation/runtime-browser-commands-factory'
import { ScreencastUnsubscribe } from '../../../../../shared/rpc-contract/browser-screencast-params'
import {
ScreencastAck,
ScreencastUnsubscribe
} from '../../../../../shared/rpc-contract/browser-screencast-params'
import { acknowledgeBrowserScreencastFrame } from '../../../browser/screencast/browser-screencast-frame-credit'
export const BROWSER_SCREENCAST_METHODS = [
defineStreamingMethod({
@@ -40,5 +44,14 @@ export const BROWSER_SCREENCAST_METHODS = [
runtime.cleanupSubscription(params.subscriptionId)
return { unsubscribed: true }
}
}),
defineMethod({
name: 'browser.screencast.ack',
permission: 'workspace',
params: ScreencastAck,
// Late acks for an ended stream are normal; report them instead of failing the request.
handler: async (params, { pairedDeviceId }) => ({
credited: acknowledgeBrowserScreencastFrame(params.subscriptionId, pairedDeviceId, params.seq)
})
})
]
@@ -13,7 +13,8 @@ export function getRuntimeFeatureInteractionId(
if (method === 'browser.profileClearDefaultCookies') {
return hasBooleanResult(result, 'cleared') ? 'cookie-import' : null
}
if (method === 'browser.screencast.unsubscribe') {
// Stream control (unsubscribe, frame acks) is not a browser-use interaction.
if (method.startsWith('browser.screencast.')) {
return null
}
if (method.startsWith('browser.') && isBrowserPaneUiRuntimeRpcParams(rawParams)) {
@@ -34,6 +34,7 @@ export const EXECUTION_HOST_METHODS = [
'browser.mouseWheel',
'browser.reload',
'browser.screencast',
'browser.screencast.ack',
'browser.screencast.unsubscribe',
'browser.tabCreate',
'browser.viewport',
@@ -113,6 +113,14 @@ export function encodeBrowserScreencastFrame(frame: BrowserScreencastFrame): Uin
return out
}
/** The seq of an encoded frame, read from its header without decoding the metadata. */
export function readBrowserScreencastFrameSeq(bytes: Uint8Array): number | null {
if (bytes.byteLength < HEADER_BYTES) {
return null
}
return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(4, true)
}
export function decodeBrowserScreencastFrame(bytes: Uint8Array): BrowserScreencastFrame | null {
if (bytes.byteLength < HEADER_BYTES) {
return null
+3 -1
View File
@@ -59,7 +59,9 @@ export const Screencast = BrowserTarget.extend({
deviceScaleFactor: OptionalFiniteNumber,
mobile: OptionalBoolean,
everyNthFrame: OptionalFiniteNumber,
minFrameIntervalMs: OptionalFiniteNumber
minFrameIntervalMs: OptionalFiniteNumber,
// Ask for frame credit; the host echoes `frameAck: true` in `ready`. Old hosts strip it.
frameAck: z.literal(1).optional()
})
export const FullScreenshot = BrowserTarget.extend({
@@ -3,3 +3,9 @@ import { z } from 'zod'
export const ScreencastUnsubscribe = z.object({
subscriptionId: z.string().min(1, 'Missing required --subscription-id')
})
export const ScreencastAck = z.object({
subscriptionId: z.string().min(1, 'Missing required --subscription-id'),
// Cumulative: the newest frame seq the viewer has received.
seq: z.number().int().nonnegative()
})
+2 -1
View File
@@ -156,7 +156,7 @@ import {
Viewport,
Wait
} from './browser-params'
import { ScreencastUnsubscribe } from './browser-screencast-params'
import { ScreencastAck, ScreencastUnsubscribe } from './browser-screencast-params'
import { BrowserOpenUrlParams, BrowserTabCreateParams } from './browser-tab-create-params'
import { ClientEventsSubscribeParams, ClientEventsUnsubscribeParams } from './client-events-params'
import {
@@ -942,6 +942,7 @@ export const RPC_PARAMS_BY_METHOD = {
'browser.profileList': null,
'browser.reload': ExpectUrlBrowserTarget,
'browser.screencast': Screencast,
'browser.screencast.ack': ScreencastAck,
'browser.screencast.unsubscribe': ScreencastUnsubscribe,
'browser.screenshot': Screenshot,
'browser.scroll': Scroll,
+2
View File
@@ -77,6 +77,8 @@ export type BrowserScreencastReadyResult = {
browserPageId: string
format: 'jpeg' | 'png'
tab: BrowserTabInfo
// Echo of `frameAck: 1`: the host holds frames until the viewer acks them.
frameAck?: true
}
export type BrowserScreencastEndResult = { type: 'end'; subscriptionId: string }