diff --git a/mobile/src/browser/mobile-browser-frameless-stream.test.tsx b/mobile/src/browser/mobile-browser-frameless-stream.test.tsx index 3807b1ed406..44ed835ea0e 100644 --- a/mobile/src/browser/mobile-browser-frameless-stream.test.tsx +++ b/mobile/src/browser/mobile-browser-frameless-stream.test.tsx @@ -70,21 +70,25 @@ async function renderPane(): Promise<{ renderer: ReactTestRenderer stream: Subscription | undefined subscriptions: Subscription[] + sendRequest: ReturnType }> { pageCounter += 1 const subscriptions: Subscription[] = [] - const client = { - subscribe: ( - _method: string, - _params: unknown, - listener: (payload: unknown) => void, - options?: { onBinaryFrame?: (frame: BrowserScreencastFrame) => void } - ) => { + const sendRequest = vi.fn().mockResolvedValue({}) + const client: RpcClient = { + sendRequest, + subscribe: (_method, _params, listener, options) => { subscriptions.push({ listener, onBinaryFrame: options?.onBinaryFrame }) return () => {} }, - request: vi.fn() - } as unknown as RpcClient + updateTerminalSubscriptionViewport: vi.fn(), + getState: () => 'connected', + getReconnectAttempt: () => 0, + getLastConnectedAt: () => null, + onStateChange: () => () => {}, + notifyForeground: vi.fn(), + close: vi.fn() + } const tab: MobileBrowserTab = { type: 'browser', @@ -129,18 +133,19 @@ async function renderPane(): Promise<{ act(() => { viewport.props.onLayout({ nativeEvent: { layout: { width: 360, height: 640 } } }) }) - return { renderer: mounted, stream: subscriptions[0], subscriptions } + return { renderer: mounted, stream: subscriptions[0], subscriptions, sendRequest } } async function renderStreamingPane(): Promise<{ renderer: ReactTestRenderer stream: Subscription + sendRequest: ReturnType }> { - const { renderer, stream } = await renderPane() + const { renderer, stream, sendRequest } = await renderPane() if (!stream) { throw new Error('browser.screencast subscription not created') } - return { renderer, stream } + return { renderer, stream, sendRequest } } function errorMessages(renderer: ReactTestRenderer): string[] { @@ -196,3 +201,26 @@ describe('MobileBrowserPane against a shell with no binary screencast lane', () } }) }) + +describe('MobileBrowserPane frame acks', () => { + // Old hosts strip the opt-in and would refuse the unknown ack method. + it('acks received frames only after ready echoes frameAck', async () => { + const legacy = await renderStreamingPane() + act(() => { + legacy.stream.listener({ type: 'ready', subscriptionId: 'sub-legacy', tab: {} }) + legacy.stream.onBinaryFrame?.(makeFrame()) + }) + expect(legacy.sendRequest).not.toHaveBeenCalled() + + const credited = await renderStreamingPane() + act(() => { + credited.stream.listener({ type: 'ready', subscriptionId: 'sub-1', frameAck: true, tab: {} }) + credited.stream.onBinaryFrame?.(makeFrame()) + }) + expect(credited.sendRequest).toHaveBeenCalledWith('browser.screencast.ack', { + worktree: `id:wt-${pageCounter}`, + subscriptionId: 'sub-1', + seq: 1 + }) + }) +}) diff --git a/mobile/src/browser/use-mobile-browser-stream.ts b/mobile/src/browser/use-mobile-browser-stream.ts index e23d0379639..b131db46121 100644 --- a/mobile/src/browser/use-mobile-browser-stream.ts +++ b/mobile/src/browser/use-mobile-browser-stream.ts @@ -205,17 +205,25 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) { startupTimer = null } } + const worktree = `id:${worktreeId}` + // Set once `ready` echoes frameAck; hosts that strip the opt-in never get an ack. + let ackSubscriptionId: string | null = null const unsubscribe = client.subscribe( 'browser.screencast', { - worktree: `id:${worktreeId}`, + worktree, page: tab.browserPageId, - ...streamRequest + ...streamRequest, + frameAck: 1 }, (payload) => { if (streamGenerationRef.current !== generation) { return } + const ready = readScreencastReady(payload) + if (ready) { + ackSubscriptionId = ready.frameAck === true ? ready.subscriptionId : null + } handleBrowserScreencastEvent({ busyRef, clearStartupTimer, @@ -234,6 +242,16 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) { return } clearStartupTimer() + if (ackSubscriptionId) { + // Cumulative, so a frame the shell dropped is covered by the next one's ack. + void client + .sendRequest('browser.screencast.ack', { + worktree, + subscriptionId: ackSubscriptionId, + seq: frame.seq + }) + .catch(() => {}) + } if (cacheKey) { framePacer.push(frame, cacheKey) } @@ -269,3 +287,22 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) { sendBrowserRequest } } + +function readScreencastReady( + payload: unknown +): { subscriptionId: string; frameAck: unknown } | null { + if ( + typeof payload !== 'object' || + payload === null || + !('type' in payload) || + payload.type !== 'ready' || + !('subscriptionId' in payload) || + typeof payload.subscriptionId !== 'string' + ) { + return null + } + return { + subscriptionId: payload.subscriptionId, + frameAck: 'frameAck' in payload ? payload.frameAck : undefined + } +} diff --git a/src/main/orcad/electron-sidecar-method-routing.test.ts b/src/main/orcad/electron-sidecar-method-routing.test.ts index d2ea62414a8..84056c5e299 100644 --- a/src/main/orcad/electron-sidecar-method-routing.test.ts +++ b/src/main/orcad/electron-sidecar-method-routing.test.ts @@ -14,7 +14,10 @@ const BROWSER_RPC_METHODS = [ // Known Log-suffix mismatches: remove entries when fixed; never add entries to hide new failures. const UNROUTABLE_RUNTIME_COMMANDS = new Set(['browserConsoleLog', 'browserNetworkLog']) -const RPC_METHODS_WITHOUT_BROWSER_COMMAND = new Set(['browser.screencast.unsubscribe']) +const RPC_METHODS_WITHOUT_BROWSER_COMMAND = new Set([ + 'browser.screencast.unsubscribe', + 'browser.screencast.ack' +]) async function runtimeCommandFor( method: (typeof BROWSER_RPC_METHODS)[number] diff --git a/src/main/runtime/browser/screencast/browser-screencast-contracts.ts b/src/main/runtime/browser/screencast/browser-screencast-contracts.ts index cbc4cb5735d..447c69a5457 100644 --- a/src/main/runtime/browser/screencast/browser-screencast-contracts.ts +++ b/src/main/runtime/browser/screencast/browser-screencast-contracts.ts @@ -5,6 +5,7 @@ import type { BrowserScreencastViewport } from '../../../browser/cdp/screencast/browser-screencast-stream-types' import type { ScreencastSubscriberDeliveryState } from './browser-screencast-ghost-subscriber-eviction' +import type { BrowserScreencastFrameCredit } from './browser-screencast-frame-credit' import type { RpcBinarySender } from '../../rpc/rpc-binary-sender' import type { BrowserCommandTargetParams } from '../commands/browser-command-contracts' @@ -19,6 +20,8 @@ export type BrowserScreencastParams = { mobile?: boolean everyNthFrame?: number minFrameIntervalMs?: number + // The viewer acks frames with browser.screencast.ack; `ready` echoes `frameAck: true`. + frameAck?: 1 } & BrowserCommandTargetParams export type BrowserScreencastStartResult = { @@ -42,6 +45,8 @@ export type ActiveBrowserScreencastSubscriber = { // Device identity survives reconnects; connection ids cannot replace stale subscriptions. pairedDeviceId?: string delivery: ScreencastSubscriberDeliveryState + // Present when the viewer acks frames; `release` unregisters it from acks. + frameCredit: { credit: BrowserScreencastFrameCredit; release: () => void } | null } export type ActiveBrowserScreencastPage = { diff --git a/src/main/runtime/browser/screencast/browser-screencast-frame-credit.ts b/src/main/runtime/browser/screencast/browser-screencast-frame-credit.ts new file mode 100644 index 00000000000..fc07716fc0c --- /dev/null +++ b/src/main/runtime/browser/screencast/browser-screencast-frame-credit.ts @@ -0,0 +1,137 @@ +// Frames a phone may have in flight before it acks one. Two cover decode and one round trip on a fast link. +const BASE_WINDOW_FRAMES = 2 +// A backstop only: the queue-delay test bounds buffering. 16 keeps 10 fps up to a 1.6 s round trip. +const MAX_WINDOW_FRAMES = 16 +// Beyond the base window, a frame goes out only while the link has queued less than this. +const QUEUE_DELAY_ALLOWANCE_MS = 100 +// The shell drops oversize frames unannounced; probe with one frame, backing off, when acks stop. +const STALL_PROBE_MS = 2_000 +const STALL_PROBE_MAX_MS = 8_000 + +export type BrowserScreencastFrameCredit = { + canSend: () => boolean + recordSent: (seq: number) => void + /** Cumulative: credits every sent frame up to and including `seq`. */ + acknowledge: (seq: number) => void + dispose: () => void +} + +type SentFrame = { seq: number; sentAt: number } + +/** Bounds a viewer's unacked frames by queueing delay rather than a fixed count, so a long but + * idle link keeps its frame rate while a slow one stops filling buffers between host and phone. */ +export function createBrowserScreencastFrameCredit(options: { + /** Credit opened up: send the newest held frame, if any. */ + onCreditAvailable: () => void +}): BrowserScreencastFrameCredit { + const inFlight: SentFrame[] = [] + // Lowest send-to-ack time seen; never raised, so queueing delay cannot inflate the window. + let minRttMs: number | null = null + let probeDelayMs = STALL_PROBE_MS + let probeTimer: ReturnType | null = null + let probeReady = false + + const clearProbe = (): void => { + if (probeTimer) { + clearTimeout(probeTimer) + probeTimer = null + } + } + + const armProbe = (): void => { + if (probeTimer) { + return + } + probeTimer = setTimeout(() => { + probeTimer = null + probeReady = true + probeDelayMs = Math.min(probeDelayMs * 2, STALL_PROBE_MAX_MS) + options.onCreditAvailable() + }, probeDelayMs) + } + + const hasCredit = (): boolean => { + if (probeReady || inFlight.length < BASE_WINDOW_FRAMES) { + return true + } + if (inFlight.length >= MAX_WINDOW_FRAMES || minRttMs === null) { + return false + } + return Date.now() - inFlight[0]!.sentAt < minRttMs + QUEUE_DELAY_ALLOWANCE_MS + } + + return { + canSend() { + if (hasCredit()) { + return true + } + armProbe() + return false + }, + recordSent(seq) { + probeReady = false + inFlight.push({ seq, sentAt: Date.now() }) + // A probe past the cap means older frames are presumed lost. + if (inFlight.length > MAX_WINDOW_FRAMES) { + inFlight.shift() + } + }, + acknowledge(seq) { + // inFlight is in seq order, so the acked frames are a prefix. + let count = 0 + while (count < inFlight.length && inFlight[count]!.seq <= seq) { + count += 1 + } + if (count === 0) { + return + } + const newest = inFlight[count - 1]! + if (newest.seq === seq) { + const rtt = Date.now() - newest.sentAt + minRttMs = minRttMs === null ? rtt : Math.min(minRttMs, rtt) + } + inFlight.splice(0, count) + clearProbe() + probeDelayMs = STALL_PROBE_MS + options.onCreditAvailable() + }, + dispose: clearProbe + } +} + +type CreditedSubscription = { + pairedDeviceId: string | undefined + credit: BrowserScreencastFrameCredit +} + +// Keyed by subscription id: the relay keeps it intact, and it is unguessable per viewer. +const creditedSubscriptions = new Map() + +export function registerBrowserScreencastFrameCredit( + subscriptionId: string, + pairedDeviceId: string | undefined, + credit: BrowserScreencastFrameCredit +): () => void { + const entry = { pairedDeviceId, credit } + creditedSubscriptions.set(subscriptionId, entry) + return () => { + if (creditedSubscriptions.get(subscriptionId) === entry) { + creditedSubscriptions.delete(subscriptionId) + } + credit.dispose() + } +} + +/** False when the subscription ended or belongs to another device; late acks are normal. */ +export function acknowledgeBrowserScreencastFrame( + subscriptionId: string, + pairedDeviceId: string | undefined, + seq: number +): boolean { + const entry = creditedSubscriptions.get(subscriptionId) + if (!entry || entry.pairedDeviceId !== pairedDeviceId) { + return false + } + entry.credit.acknowledge(seq) + return true +} diff --git a/src/main/runtime/browser/screencast/browser-screencast-subscriber-frame-delivery.ts b/src/main/runtime/browser/screencast/browser-screencast-subscriber-frame-delivery.ts index 6e3fa04f998..0e8e03f603f 100644 --- a/src/main/runtime/browser/screencast/browser-screencast-subscriber-frame-delivery.ts +++ b/src/main/runtime/browser/screencast/browser-screencast-subscriber-frame-delivery.ts @@ -1,6 +1,10 @@ +import { readBrowserScreencastFrameSeq } from '../../../../shared/browser-screencast-protocol' import type { ActiveBrowserScreencastSubscriber } from './browser-screencast-contracts' import { recordScreencastSubscriberSend } from './browser-screencast-ghost-subscriber-eviction' -import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission' +import { + sendRemoteBrowserScreencastFrame, + type ScreencastFrameSendOutcome +} from './remote-browser-screencast-frame-admission' // Match the frame pacer's backpressure retry interval. export const SCREENCAST_PENDING_FRAME_RETRY_MS = 50 @@ -10,7 +14,12 @@ export function deliverScreencastSubscriberFrame( subscriber: ActiveBrowserScreencastSubscriber, bytes: Uint8Array ): void { - const outcome = sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes) + // Held for credit, not refused: the viewer's ack or the stall probe sends the newest held frame. + if (subscriber.frameCredit && !subscriber.frameCredit.credit.canSend()) { + subscriber.pendingFrame = bytes + return + } + const outcome = sendSubscriberFrame(subscriber, bytes) subscriber.pendingFrame = outcome === 'handled' ? null : bytes // Backlog proves an active socket, not a ghost. if (outcome !== 'backlogged') { @@ -21,6 +30,14 @@ export function deliverScreencastSubscriberFrame( } } +/** Credit opened up: send the held frame now instead of on the next produced one. */ +export function flushCreditedScreencastFrame(subscriber: ActiveBrowserScreencastSubscriber): void { + if (subscriber.pendingFrameRetry) { + return + } + retryPendingFrame(subscriber) +} + export function cancelScreencastSubscriberFrameRetry( subscriber: ActiveBrowserScreencastSubscriber ): void { @@ -29,6 +46,36 @@ export function cancelScreencastSubscriberFrameRetry( subscriber.pendingFrameRetry = null } subscriber.pendingFrame = null + subscriber.frameCredit?.release() + subscriber.frameCredit = null +} + +function sendSubscriberFrame( + subscriber: ActiveBrowserScreencastSubscriber, + bytes: Uint8Array +): ScreencastFrameSendOutcome { + const outcome = sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes) + if (outcome === 'handled' && subscriber.frameCredit) { + const seq = readBrowserScreencastFrameSeq(bytes) + if (seq !== null) { + subscriber.frameCredit.credit.recordSent(seq) + } + } + return outcome +} + +function retryPendingFrame(subscriber: ActiveBrowserScreencastSubscriber): void { + const bytes = subscriber.pendingFrame + if (!bytes || (subscriber.frameCredit && !subscriber.frameCredit.credit.canSend())) { + return + } + // Count ghost refusals only on produced frames, never retries. + if (sendSubscriberFrame(subscriber, bytes) === 'handled') { + subscriber.pendingFrame = null + subscriber.delivery = recordScreencastSubscriberSend(subscriber.delivery, true) + } else { + schedulePendingFrameRetry(subscriber) + } } function schedulePendingFrameRetry(subscriber: ActiveBrowserScreencastSubscriber): void { @@ -37,16 +84,6 @@ function schedulePendingFrameRetry(subscriber: ActiveBrowserScreencastSubscriber } subscriber.pendingFrameRetry = setTimeout(() => { subscriber.pendingFrameRetry = null - const bytes = subscriber.pendingFrame - if (!bytes) { - return - } - // Count ghost refusals only on produced frames, never retries. - if (sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes) === 'handled') { - subscriber.pendingFrame = null - subscriber.delivery = recordScreencastSubscriberSend(subscriber.delivery, true) - } else { - schedulePendingFrameRetry(subscriber) - } + retryPendingFrame(subscriber) }, SCREENCAST_PENDING_FRAME_RETRY_MS) } diff --git a/src/main/runtime/browser/screencast/browser-screencasts.ts b/src/main/runtime/browser/screencast/browser-screencasts.ts index 10a4b832248..22099101de4 100644 --- a/src/main/runtime/browser/screencast/browser-screencasts.ts +++ b/src/main/runtime/browser/screencast/browser-screencasts.ts @@ -6,8 +6,13 @@ import { rendererPublicationThrottle } from '../../../window/renderer-publicatio import { mintBrowserScreencastSubscriptionId } from './browser-screencast-subscription-id' import { cancelScreencastSubscriberFrameRetry, - deliverScreencastSubscriberFrame + deliverScreencastSubscriberFrame, + flushCreditedScreencastFrame } from './browser-screencast-subscriber-frame-delivery' +import { + createBrowserScreencastFrameCredit, + registerBrowserScreencastFrameCredit +} from './browser-screencast-frame-credit' import { INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY, screencastSubscriberIsGhost @@ -151,7 +156,7 @@ export class RuntimeBrowserScreencasts { }) .catch(() => {}) } - active.subscribers.set(subscriptionId, { + const subscriber: ActiveBrowserScreencastSubscriber = { sendBinary: stream.sendBinary, emit: stream.emit, done: subscriberDone, @@ -161,8 +166,19 @@ export class RuntimeBrowserScreencasts { pendingFrame: null, pendingFrameRetry: null, pairedDeviceId: stream.pairedDeviceId, - delivery: INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY - }) + delivery: INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY, + frameCredit: null + } + if (params.frameAck === 1) { + const credit = createBrowserScreencastFrameCredit({ + onCreditAvailable: () => flushCreditedScreencastFrame(subscriber) + }) + subscriber.frameCredit = { + credit, + release: registerBrowserScreencastFrameCredit(subscriptionId, stream.pairedDeviceId, credit) + } + } + active.subscribers.set(subscriptionId, subscriber) // A sizeless viewport owner would clear emulation for every viewer. if (hasScreencastViewportSize(viewport)) { active.viewportOwnerSubscriptionId = subscriptionId @@ -172,6 +188,7 @@ export class RuntimeBrowserScreencasts { session = await active.started } catch (error) { active.subscribers.delete(subscriptionId) + cancelScreencastSubscriberFrameRetry(subscriber) resolveSubscriberDone() throw error } @@ -219,7 +236,8 @@ export class RuntimeBrowserScreencasts { subscriptionId, browserPageId, format: active.format, - tab: this.catalog.describeBrowserTab(browserPageId, target.worktreeId) + tab: this.catalog.describeBrowserTab(browserPageId, target.worktreeId), + ...(subscriber.frameCredit ? { frameAck: true as const } : {}) } } } diff --git a/src/main/runtime/browser/screencast/orca-runtime-browser-screencast-frame-credit.test.ts b/src/main/runtime/browser/screencast/orca-runtime-browser-screencast-frame-credit.test.ts new file mode 100644 index 00000000000..38eba2bc2bb --- /dev/null +++ b/src/main/runtime/browser/screencast/orca-runtime-browser-screencast-frame-credit.test.ts @@ -0,0 +1,133 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + BrowserScreencastOpcode, + decodeBrowserScreencastFrame, + encodeBrowserScreencastFrame +} from '../../../../shared/browser-screencast-protocol' +import { acknowledgeBrowserScreencastFrame } from './browser-screencast-frame-credit' +import { + createSinglePageBrowserCommandsHost, + createStreamedGuest +} from '../client-pages/single-page-browser-commands-host-test-double' + +const { webContentsFromId, startBrowserScreencast } = vi.hoisted(() => ({ + webContentsFromId: vi.fn(), + startBrowserScreencast: vi.fn() +})) + +vi.mock('electron', () => ({ + ipcMain: { on: vi.fn(), removeListener: vi.fn() }, + webContents: { fromId: webContentsFromId } +})) +vi.mock('../../../browser/cdp/screencast/browser-screencast-stream', () => ({ + startBrowserScreencast +})) + +const PHONE = 'phone-1' + +function frame(seq: number): Uint8Array { + return encodeBrowserScreencastFrame({ + opcode: BrowserScreencastOpcode.Frame, + seq, + format: 'jpeg', + metadata: {}, + image: new Uint8Array([seq]) + }) +} + +async function subscribe(frameAck: boolean) { + const { RuntimeBrowserCommands } = await import('../commands/browser-commands') + startBrowserScreencast.mockResolvedValue({ + stop: vi.fn(), + done: new Promise(() => {}), + updateViewport: vi.fn(async () => {}) + }) + const sent: number[] = [] + const sentAt: number[] = [] + const commands = new RuntimeBrowserCommands(createSinglePageBrowserCommandsHost()) + const subscription = await commands.browserScreencast( + { worktree: 'id:wt-1', page: 'page-1', format: 'jpeg', ...(frameAck ? { frameAck: 1 } : {}) }, + { + pairedDeviceId: PHONE, + sendBinary: (bytes) => { + sent.push(decodeBrowserScreencastFrame(bytes)!.seq) + sentAt.push(Date.now()) + return true + } + } + ) + const onFrame: (bytes: Uint8Array) => unknown = startBrowserScreencast.mock.calls[0][1].onFrame + const ack = (seq: number, device = PHONE) => + acknowledgeBrowserScreencastFrame(subscription.subscriptionId, device, seq) + return { subscription, onFrame, sent, sentAt, ack } +} + +describe('screencast frame credit', () => { + beforeEach(() => { + vi.useFakeTimers() + webContentsFromId.mockReset() + webContentsFromId.mockReturnValue(createStreamedGuest()) + startBrowserScreencast.mockReset() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('sends every frame and echoes nothing when the viewer does not ask for credit', async () => { + const { subscription, onFrame, sent } = await subscribe(false) + for (let seq = 0; seq < 5; seq++) { + onFrame(frame(seq)) + } + expect(subscription.ready.frameAck).toBeUndefined() + expect(sent).toEqual([0, 1, 2, 3, 4]) + subscription.session.stop() + }) + + it('holds the newest frame while two are unacked and sends it on a cumulative ack', async () => { + const { subscription, onFrame, sent, ack } = await subscribe(true) + expect(subscription.ready.frameAck).toBe(true) + for (let seq = 0; seq < 4; seq++) { + onFrame(frame(seq)) + } + expect(sent).toEqual([0, 1]) + + expect(ack(1, 'other-phone')).toBe(false) + expect(sent).toEqual([0, 1]) + + expect(ack(1)).toBe(true) + expect(sent).toEqual([0, 1, 3]) + subscription.session.stop() + }) + + // #25263: a fixed two-frame window delivered about a third of 10 fps on an idle 600 ms link. + it('keeps 10 fps on an uncongested 600 ms round trip', async () => { + const { subscription, onFrame, sent, sentAt, ack } = await subscribe(true) + let acked = 0 + for (let tick = 0; tick < 100; tick++) { + onFrame(frame(tick)) + for (let step = 0; step < 10; step++) { + await vi.advanceTimersByTimeAsync(10) + // Each frame's ack lands 600 ms after it went out. + while (acked < sent.length && sentAt[acked]! + 600 <= Date.now()) { + ack(sent[acked]!) + acked += 1 + } + } + } + // Only frames 2–4 are skipped, superseded while the first ack is still in flight. + expect(sent).toEqual([0, 1, ...Array.from({ length: 95 }, (_, index) => index + 5)]) + subscription.session.stop() + }) + + // #25263: a 2 s expiry let ten unacked frames through in 10 s. Probes back off to one per 8 s. + it('sends one probe frame per backed-off stall when no ack ever comes', async () => { + const { subscription, onFrame, sent } = await subscribe(true) + for (let tick = 0; tick < 300; tick++) { + onFrame(frame(tick)) + await vi.advanceTimersByTimeAsync(100) + } + // The base window, then probes 2 s, 4 s, 8 s and 8 s after each hold (t = 2.2, 6.2, 14.2, 22.2 s). + expect(sent).toEqual([0, 1, 21, 61, 141, 221]) + subscription.session.stop() + }) +}) diff --git a/src/main/runtime/mobile-desktop-relay/hosted-browser-page-route.ts b/src/main/runtime/mobile-desktop-relay/hosted-browser-page-route.ts index b09b11e14e8..1b2bc64c900 100644 --- a/src/main/runtime/mobile-desktop-relay/hosted-browser-page-route.ts +++ b/src/main/runtime/mobile-desktop-relay/hosted-browser-page-route.ts @@ -24,7 +24,8 @@ export function localHostedBrowserPageRequest( // Server workspace selectors cannot resolve here; page ids are stable. const { worktree: _serverSelector, ...params } = parsed.data const page = - request.method === 'browser.screencast.unsubscribe' + request.method === 'browser.screencast.unsubscribe' || + request.method === 'browser.screencast.ack' ? browserScreencastSubscriptionPageId(params.subscriptionId ?? '') : params.page || null if (page === null || !hostsBrowserPage(environmentId, page)) { diff --git a/src/main/runtime/rpc/methods/browser/browser-screencast.ts b/src/main/runtime/rpc/methods/browser/browser-screencast.ts index fb4542f8d9f..d55f3f9aebe 100644 --- a/src/main/runtime/rpc/methods/browser/browser-screencast.ts +++ b/src/main/runtime/rpc/methods/browser/browser-screencast.ts @@ -4,7 +4,11 @@ import { BrowserError } from '../../../../browser/browser-error' import '../../../../../shared/runtime-types' import { BROWSER_UNAVAILABLE_ERROR_CODE } from '../../../../../shared/runtime-session-contracts' import { runtimeBrowserCommandsFactoryIsAvailable } from '../../../browser/installation/runtime-browser-commands-factory' -import { ScreencastUnsubscribe } from '../../../../../shared/rpc-contract/browser-screencast-params' +import { + ScreencastAck, + ScreencastUnsubscribe +} from '../../../../../shared/rpc-contract/browser-screencast-params' +import { acknowledgeBrowserScreencastFrame } from '../../../browser/screencast/browser-screencast-frame-credit' export const BROWSER_SCREENCAST_METHODS = [ defineStreamingMethod({ @@ -40,5 +44,14 @@ export const BROWSER_SCREENCAST_METHODS = [ runtime.cleanupSubscription(params.subscriptionId) return { unsubscribed: true } } + }), + defineMethod({ + name: 'browser.screencast.ack', + permission: 'workspace', + params: ScreencastAck, + // Late acks for an ended stream are normal; report them instead of failing the request. + handler: async (params, { pairedDeviceId }) => ({ + credited: acknowledgeBrowserScreencastFrame(params.subscriptionId, pairedDeviceId, params.seq) + }) }) ] diff --git a/src/main/runtime/rpc/runtime-feature-interaction.ts b/src/main/runtime/rpc/runtime-feature-interaction.ts index 23f28001fcd..4aae960ec05 100644 --- a/src/main/runtime/rpc/runtime-feature-interaction.ts +++ b/src/main/runtime/rpc/runtime-feature-interaction.ts @@ -13,7 +13,8 @@ export function getRuntimeFeatureInteractionId( if (method === 'browser.profileClearDefaultCookies') { return hasBooleanResult(result, 'cleared') ? 'cookie-import' : null } - if (method === 'browser.screencast.unsubscribe') { + // Stream control (unsubscribe, frame acks) is not a browser-use interaction. + if (method.startsWith('browser.screencast.')) { return null } if (method.startsWith('browser.') && isBrowserPaneUiRuntimeRpcParams(rawParams)) { diff --git a/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts b/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts index 854efdbe03b..2a3a346e6cd 100644 --- a/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts +++ b/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts @@ -34,6 +34,7 @@ export const EXECUTION_HOST_METHODS = [ 'browser.mouseWheel', 'browser.reload', 'browser.screencast', + 'browser.screencast.ack', 'browser.screencast.unsubscribe', 'browser.tabCreate', 'browser.viewport', diff --git a/src/shared/browser-screencast-protocol.ts b/src/shared/browser-screencast-protocol.ts index 842fe86cb3d..1262c8fb3a3 100644 --- a/src/shared/browser-screencast-protocol.ts +++ b/src/shared/browser-screencast-protocol.ts @@ -113,6 +113,14 @@ export function encodeBrowserScreencastFrame(frame: BrowserScreencastFrame): Uin return out } +/** The seq of an encoded frame, read from its header without decoding the metadata. */ +export function readBrowserScreencastFrameSeq(bytes: Uint8Array): number | null { + if (bytes.byteLength < HEADER_BYTES) { + return null + } + return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(4, true) +} + export function decodeBrowserScreencastFrame(bytes: Uint8Array): BrowserScreencastFrame | null { if (bytes.byteLength < HEADER_BYTES) { return null diff --git a/src/shared/rpc-contract/browser-params.ts b/src/shared/rpc-contract/browser-params.ts index b606c9ebd44..236e64b134c 100644 --- a/src/shared/rpc-contract/browser-params.ts +++ b/src/shared/rpc-contract/browser-params.ts @@ -59,7 +59,9 @@ export const Screencast = BrowserTarget.extend({ deviceScaleFactor: OptionalFiniteNumber, mobile: OptionalBoolean, everyNthFrame: OptionalFiniteNumber, - minFrameIntervalMs: OptionalFiniteNumber + minFrameIntervalMs: OptionalFiniteNumber, + // Ask for frame credit; the host echoes `frameAck: true` in `ready`. Old hosts strip it. + frameAck: z.literal(1).optional() }) export const FullScreenshot = BrowserTarget.extend({ diff --git a/src/shared/rpc-contract/browser-screencast-params.ts b/src/shared/rpc-contract/browser-screencast-params.ts index 016b7e0f83b..57ede6a5a74 100644 --- a/src/shared/rpc-contract/browser-screencast-params.ts +++ b/src/shared/rpc-contract/browser-screencast-params.ts @@ -3,3 +3,9 @@ import { z } from 'zod' export const ScreencastUnsubscribe = z.object({ subscriptionId: z.string().min(1, 'Missing required --subscription-id') }) + +export const ScreencastAck = z.object({ + subscriptionId: z.string().min(1, 'Missing required --subscription-id'), + // Cumulative: the newest frame seq the viewer has received. + seq: z.number().int().nonnegative() +}) diff --git a/src/shared/rpc-contract/rpc-params-catalog.generated.ts b/src/shared/rpc-contract/rpc-params-catalog.generated.ts index c2510f5b812..7a8997ca741 100644 --- a/src/shared/rpc-contract/rpc-params-catalog.generated.ts +++ b/src/shared/rpc-contract/rpc-params-catalog.generated.ts @@ -156,7 +156,7 @@ import { Viewport, Wait } from './browser-params' -import { ScreencastUnsubscribe } from './browser-screencast-params' +import { ScreencastAck, ScreencastUnsubscribe } from './browser-screencast-params' import { BrowserOpenUrlParams, BrowserTabCreateParams } from './browser-tab-create-params' import { ClientEventsSubscribeParams, ClientEventsUnsubscribeParams } from './client-events-params' import { @@ -942,6 +942,7 @@ export const RPC_PARAMS_BY_METHOD = { 'browser.profileList': null, 'browser.reload': ExpectUrlBrowserTarget, 'browser.screencast': Screencast, + 'browser.screencast.ack': ScreencastAck, 'browser.screencast.unsubscribe': ScreencastUnsubscribe, 'browser.screenshot': Screenshot, 'browser.scroll': Scroll, diff --git a/src/shared/runtime-browser-contracts.ts b/src/shared/runtime-browser-contracts.ts index a89b68f8981..1e1fefc767f 100644 --- a/src/shared/runtime-browser-contracts.ts +++ b/src/shared/runtime-browser-contracts.ts @@ -77,6 +77,8 @@ export type BrowserScreencastReadyResult = { browserPageId: string format: 'jpeg' | 'png' tab: BrowserTabInfo + // Echo of `frameAck: 1`: the host holds frames until the viewer acks them. + frameAck?: true } export type BrowserScreencastEndResult = { type: 'end'; subscriptionId: string }