mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 11:24:37 +00:00
perf(browser): credit screencast frames to the phone (#39189)
This commit is contained in:
@@ -70,21 +70,25 @@ async function renderPane(): Promise<{
|
||||
renderer: ReactTestRenderer
|
||||
stream: Subscription | undefined
|
||||
subscriptions: Subscription[]
|
||||
sendRequest: ReturnType<typeof vi.fn>
|
||||
}> {
|
||||
pageCounter += 1
|
||||
const subscriptions: Subscription[] = []
|
||||
const client = {
|
||||
subscribe: (
|
||||
_method: string,
|
||||
_params: unknown,
|
||||
listener: (payload: unknown) => void,
|
||||
options?: { onBinaryFrame?: (frame: BrowserScreencastFrame) => void }
|
||||
) => {
|
||||
const sendRequest = vi.fn().mockResolvedValue({})
|
||||
const client: RpcClient = {
|
||||
sendRequest,
|
||||
subscribe: (_method, _params, listener, options) => {
|
||||
subscriptions.push({ listener, onBinaryFrame: options?.onBinaryFrame })
|
||||
return () => {}
|
||||
},
|
||||
request: vi.fn()
|
||||
} as unknown as RpcClient
|
||||
updateTerminalSubscriptionViewport: vi.fn(),
|
||||
getState: () => 'connected',
|
||||
getReconnectAttempt: () => 0,
|
||||
getLastConnectedAt: () => null,
|
||||
onStateChange: () => () => {},
|
||||
notifyForeground: vi.fn(),
|
||||
close: vi.fn()
|
||||
}
|
||||
|
||||
const tab: MobileBrowserTab = {
|
||||
type: 'browser',
|
||||
@@ -129,18 +133,19 @@ async function renderPane(): Promise<{
|
||||
act(() => {
|
||||
viewport.props.onLayout({ nativeEvent: { layout: { width: 360, height: 640 } } })
|
||||
})
|
||||
return { renderer: mounted, stream: subscriptions[0], subscriptions }
|
||||
return { renderer: mounted, stream: subscriptions[0], subscriptions, sendRequest }
|
||||
}
|
||||
|
||||
async function renderStreamingPane(): Promise<{
|
||||
renderer: ReactTestRenderer
|
||||
stream: Subscription
|
||||
sendRequest: ReturnType<typeof vi.fn>
|
||||
}> {
|
||||
const { renderer, stream } = await renderPane()
|
||||
const { renderer, stream, sendRequest } = await renderPane()
|
||||
if (!stream) {
|
||||
throw new Error('browser.screencast subscription not created')
|
||||
}
|
||||
return { renderer, stream }
|
||||
return { renderer, stream, sendRequest }
|
||||
}
|
||||
|
||||
function errorMessages(renderer: ReactTestRenderer): string[] {
|
||||
@@ -196,3 +201,26 @@ describe('MobileBrowserPane against a shell with no binary screencast lane', ()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('MobileBrowserPane frame acks', () => {
|
||||
// Old hosts strip the opt-in and would refuse the unknown ack method.
|
||||
it('acks received frames only after ready echoes frameAck', async () => {
|
||||
const legacy = await renderStreamingPane()
|
||||
act(() => {
|
||||
legacy.stream.listener({ type: 'ready', subscriptionId: 'sub-legacy', tab: {} })
|
||||
legacy.stream.onBinaryFrame?.(makeFrame())
|
||||
})
|
||||
expect(legacy.sendRequest).not.toHaveBeenCalled()
|
||||
|
||||
const credited = await renderStreamingPane()
|
||||
act(() => {
|
||||
credited.stream.listener({ type: 'ready', subscriptionId: 'sub-1', frameAck: true, tab: {} })
|
||||
credited.stream.onBinaryFrame?.(makeFrame())
|
||||
})
|
||||
expect(credited.sendRequest).toHaveBeenCalledWith('browser.screencast.ack', {
|
||||
worktree: `id:wt-${pageCounter}`,
|
||||
subscriptionId: 'sub-1',
|
||||
seq: 1
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -205,17 +205,25 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
|
||||
startupTimer = null
|
||||
}
|
||||
}
|
||||
const worktree = `id:${worktreeId}`
|
||||
// Set once `ready` echoes frameAck; hosts that strip the opt-in never get an ack.
|
||||
let ackSubscriptionId: string | null = null
|
||||
const unsubscribe = client.subscribe(
|
||||
'browser.screencast',
|
||||
{
|
||||
worktree: `id:${worktreeId}`,
|
||||
worktree,
|
||||
page: tab.browserPageId,
|
||||
...streamRequest
|
||||
...streamRequest,
|
||||
frameAck: 1
|
||||
},
|
||||
(payload) => {
|
||||
if (streamGenerationRef.current !== generation) {
|
||||
return
|
||||
}
|
||||
const ready = readScreencastReady(payload)
|
||||
if (ready) {
|
||||
ackSubscriptionId = ready.frameAck === true ? ready.subscriptionId : null
|
||||
}
|
||||
handleBrowserScreencastEvent({
|
||||
busyRef,
|
||||
clearStartupTimer,
|
||||
@@ -234,6 +242,16 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
|
||||
return
|
||||
}
|
||||
clearStartupTimer()
|
||||
if (ackSubscriptionId) {
|
||||
// Cumulative, so a frame the shell dropped is covered by the next one's ack.
|
||||
void client
|
||||
.sendRequest('browser.screencast.ack', {
|
||||
worktree,
|
||||
subscriptionId: ackSubscriptionId,
|
||||
seq: frame.seq
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
if (cacheKey) {
|
||||
framePacer.push(frame, cacheKey)
|
||||
}
|
||||
@@ -269,3 +287,22 @@ export function useMobileBrowserStream(args: MobileBrowserStreamArgs) {
|
||||
sendBrowserRequest
|
||||
}
|
||||
}
|
||||
|
||||
function readScreencastReady(
|
||||
payload: unknown
|
||||
): { subscriptionId: string; frameAck: unknown } | null {
|
||||
if (
|
||||
typeof payload !== 'object' ||
|
||||
payload === null ||
|
||||
!('type' in payload) ||
|
||||
payload.type !== 'ready' ||
|
||||
!('subscriptionId' in payload) ||
|
||||
typeof payload.subscriptionId !== 'string'
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
subscriptionId: payload.subscriptionId,
|
||||
frameAck: 'frameAck' in payload ? payload.frameAck : undefined
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,10 @@ const BROWSER_RPC_METHODS = [
|
||||
// Known Log-suffix mismatches: remove entries when fixed; never add entries to hide new failures.
|
||||
const UNROUTABLE_RUNTIME_COMMANDS = new Set(['browserConsoleLog', 'browserNetworkLog'])
|
||||
|
||||
const RPC_METHODS_WITHOUT_BROWSER_COMMAND = new Set(['browser.screencast.unsubscribe'])
|
||||
const RPC_METHODS_WITHOUT_BROWSER_COMMAND = new Set([
|
||||
'browser.screencast.unsubscribe',
|
||||
'browser.screencast.ack'
|
||||
])
|
||||
|
||||
async function runtimeCommandFor(
|
||||
method: (typeof BROWSER_RPC_METHODS)[number]
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
BrowserScreencastViewport
|
||||
} from '../../../browser/cdp/screencast/browser-screencast-stream-types'
|
||||
import type { ScreencastSubscriberDeliveryState } from './browser-screencast-ghost-subscriber-eviction'
|
||||
import type { BrowserScreencastFrameCredit } from './browser-screencast-frame-credit'
|
||||
import type { RpcBinarySender } from '../../rpc/rpc-binary-sender'
|
||||
import type { BrowserCommandTargetParams } from '../commands/browser-command-contracts'
|
||||
|
||||
@@ -19,6 +20,8 @@ export type BrowserScreencastParams = {
|
||||
mobile?: boolean
|
||||
everyNthFrame?: number
|
||||
minFrameIntervalMs?: number
|
||||
// The viewer acks frames with browser.screencast.ack; `ready` echoes `frameAck: true`.
|
||||
frameAck?: 1
|
||||
} & BrowserCommandTargetParams
|
||||
|
||||
export type BrowserScreencastStartResult = {
|
||||
@@ -42,6 +45,8 @@ export type ActiveBrowserScreencastSubscriber = {
|
||||
// Device identity survives reconnects; connection ids cannot replace stale subscriptions.
|
||||
pairedDeviceId?: string
|
||||
delivery: ScreencastSubscriberDeliveryState
|
||||
// Present when the viewer acks frames; `release` unregisters it from acks.
|
||||
frameCredit: { credit: BrowserScreencastFrameCredit; release: () => void } | null
|
||||
}
|
||||
|
||||
export type ActiveBrowserScreencastPage = {
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
// Frames a phone may have in flight before it acks one. Two cover decode and one round trip on a fast link.
|
||||
const BASE_WINDOW_FRAMES = 2
|
||||
// A backstop only: the queue-delay test bounds buffering. 16 keeps 10 fps up to a 1.6 s round trip.
|
||||
const MAX_WINDOW_FRAMES = 16
|
||||
// Beyond the base window, a frame goes out only while the link has queued less than this.
|
||||
const QUEUE_DELAY_ALLOWANCE_MS = 100
|
||||
// The shell drops oversize frames unannounced; probe with one frame, backing off, when acks stop.
|
||||
const STALL_PROBE_MS = 2_000
|
||||
const STALL_PROBE_MAX_MS = 8_000
|
||||
|
||||
export type BrowserScreencastFrameCredit = {
|
||||
canSend: () => boolean
|
||||
recordSent: (seq: number) => void
|
||||
/** Cumulative: credits every sent frame up to and including `seq`. */
|
||||
acknowledge: (seq: number) => void
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
type SentFrame = { seq: number; sentAt: number }
|
||||
|
||||
/** Bounds a viewer's unacked frames by queueing delay rather than a fixed count, so a long but
|
||||
* idle link keeps its frame rate while a slow one stops filling buffers between host and phone. */
|
||||
export function createBrowserScreencastFrameCredit(options: {
|
||||
/** Credit opened up: send the newest held frame, if any. */
|
||||
onCreditAvailable: () => void
|
||||
}): BrowserScreencastFrameCredit {
|
||||
const inFlight: SentFrame[] = []
|
||||
// Lowest send-to-ack time seen; never raised, so queueing delay cannot inflate the window.
|
||||
let minRttMs: number | null = null
|
||||
let probeDelayMs = STALL_PROBE_MS
|
||||
let probeTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let probeReady = false
|
||||
|
||||
const clearProbe = (): void => {
|
||||
if (probeTimer) {
|
||||
clearTimeout(probeTimer)
|
||||
probeTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
const armProbe = (): void => {
|
||||
if (probeTimer) {
|
||||
return
|
||||
}
|
||||
probeTimer = setTimeout(() => {
|
||||
probeTimer = null
|
||||
probeReady = true
|
||||
probeDelayMs = Math.min(probeDelayMs * 2, STALL_PROBE_MAX_MS)
|
||||
options.onCreditAvailable()
|
||||
}, probeDelayMs)
|
||||
}
|
||||
|
||||
const hasCredit = (): boolean => {
|
||||
if (probeReady || inFlight.length < BASE_WINDOW_FRAMES) {
|
||||
return true
|
||||
}
|
||||
if (inFlight.length >= MAX_WINDOW_FRAMES || minRttMs === null) {
|
||||
return false
|
||||
}
|
||||
return Date.now() - inFlight[0]!.sentAt < minRttMs + QUEUE_DELAY_ALLOWANCE_MS
|
||||
}
|
||||
|
||||
return {
|
||||
canSend() {
|
||||
if (hasCredit()) {
|
||||
return true
|
||||
}
|
||||
armProbe()
|
||||
return false
|
||||
},
|
||||
recordSent(seq) {
|
||||
probeReady = false
|
||||
inFlight.push({ seq, sentAt: Date.now() })
|
||||
// A probe past the cap means older frames are presumed lost.
|
||||
if (inFlight.length > MAX_WINDOW_FRAMES) {
|
||||
inFlight.shift()
|
||||
}
|
||||
},
|
||||
acknowledge(seq) {
|
||||
// inFlight is in seq order, so the acked frames are a prefix.
|
||||
let count = 0
|
||||
while (count < inFlight.length && inFlight[count]!.seq <= seq) {
|
||||
count += 1
|
||||
}
|
||||
if (count === 0) {
|
||||
return
|
||||
}
|
||||
const newest = inFlight[count - 1]!
|
||||
if (newest.seq === seq) {
|
||||
const rtt = Date.now() - newest.sentAt
|
||||
minRttMs = minRttMs === null ? rtt : Math.min(minRttMs, rtt)
|
||||
}
|
||||
inFlight.splice(0, count)
|
||||
clearProbe()
|
||||
probeDelayMs = STALL_PROBE_MS
|
||||
options.onCreditAvailable()
|
||||
},
|
||||
dispose: clearProbe
|
||||
}
|
||||
}
|
||||
|
||||
type CreditedSubscription = {
|
||||
pairedDeviceId: string | undefined
|
||||
credit: BrowserScreencastFrameCredit
|
||||
}
|
||||
|
||||
// Keyed by subscription id: the relay keeps it intact, and it is unguessable per viewer.
|
||||
const creditedSubscriptions = new Map<string, CreditedSubscription>()
|
||||
|
||||
export function registerBrowserScreencastFrameCredit(
|
||||
subscriptionId: string,
|
||||
pairedDeviceId: string | undefined,
|
||||
credit: BrowserScreencastFrameCredit
|
||||
): () => void {
|
||||
const entry = { pairedDeviceId, credit }
|
||||
creditedSubscriptions.set(subscriptionId, entry)
|
||||
return () => {
|
||||
if (creditedSubscriptions.get(subscriptionId) === entry) {
|
||||
creditedSubscriptions.delete(subscriptionId)
|
||||
}
|
||||
credit.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
/** False when the subscription ended or belongs to another device; late acks are normal. */
|
||||
export function acknowledgeBrowserScreencastFrame(
|
||||
subscriptionId: string,
|
||||
pairedDeviceId: string | undefined,
|
||||
seq: number
|
||||
): boolean {
|
||||
const entry = creditedSubscriptions.get(subscriptionId)
|
||||
if (!entry || entry.pairedDeviceId !== pairedDeviceId) {
|
||||
return false
|
||||
}
|
||||
entry.credit.acknowledge(seq)
|
||||
return true
|
||||
}
|
||||
+50
-13
@@ -1,6 +1,10 @@
|
||||
import { readBrowserScreencastFrameSeq } from '../../../../shared/browser-screencast-protocol'
|
||||
import type { ActiveBrowserScreencastSubscriber } from './browser-screencast-contracts'
|
||||
import { recordScreencastSubscriberSend } from './browser-screencast-ghost-subscriber-eviction'
|
||||
import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission'
|
||||
import {
|
||||
sendRemoteBrowserScreencastFrame,
|
||||
type ScreencastFrameSendOutcome
|
||||
} from './remote-browser-screencast-frame-admission'
|
||||
|
||||
// Match the frame pacer's backpressure retry interval.
|
||||
export const SCREENCAST_PENDING_FRAME_RETRY_MS = 50
|
||||
@@ -10,7 +14,12 @@ export function deliverScreencastSubscriberFrame(
|
||||
subscriber: ActiveBrowserScreencastSubscriber,
|
||||
bytes: Uint8Array<ArrayBufferLike>
|
||||
): void {
|
||||
const outcome = sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes)
|
||||
// Held for credit, not refused: the viewer's ack or the stall probe sends the newest held frame.
|
||||
if (subscriber.frameCredit && !subscriber.frameCredit.credit.canSend()) {
|
||||
subscriber.pendingFrame = bytes
|
||||
return
|
||||
}
|
||||
const outcome = sendSubscriberFrame(subscriber, bytes)
|
||||
subscriber.pendingFrame = outcome === 'handled' ? null : bytes
|
||||
// Backlog proves an active socket, not a ghost.
|
||||
if (outcome !== 'backlogged') {
|
||||
@@ -21,6 +30,14 @@ export function deliverScreencastSubscriberFrame(
|
||||
}
|
||||
}
|
||||
|
||||
/** Credit opened up: send the held frame now instead of on the next produced one. */
|
||||
export function flushCreditedScreencastFrame(subscriber: ActiveBrowserScreencastSubscriber): void {
|
||||
if (subscriber.pendingFrameRetry) {
|
||||
return
|
||||
}
|
||||
retryPendingFrame(subscriber)
|
||||
}
|
||||
|
||||
export function cancelScreencastSubscriberFrameRetry(
|
||||
subscriber: ActiveBrowserScreencastSubscriber
|
||||
): void {
|
||||
@@ -29,6 +46,36 @@ export function cancelScreencastSubscriberFrameRetry(
|
||||
subscriber.pendingFrameRetry = null
|
||||
}
|
||||
subscriber.pendingFrame = null
|
||||
subscriber.frameCredit?.release()
|
||||
subscriber.frameCredit = null
|
||||
}
|
||||
|
||||
function sendSubscriberFrame(
|
||||
subscriber: ActiveBrowserScreencastSubscriber,
|
||||
bytes: Uint8Array<ArrayBufferLike>
|
||||
): ScreencastFrameSendOutcome {
|
||||
const outcome = sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes)
|
||||
if (outcome === 'handled' && subscriber.frameCredit) {
|
||||
const seq = readBrowserScreencastFrameSeq(bytes)
|
||||
if (seq !== null) {
|
||||
subscriber.frameCredit.credit.recordSent(seq)
|
||||
}
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
function retryPendingFrame(subscriber: ActiveBrowserScreencastSubscriber): void {
|
||||
const bytes = subscriber.pendingFrame
|
||||
if (!bytes || (subscriber.frameCredit && !subscriber.frameCredit.credit.canSend())) {
|
||||
return
|
||||
}
|
||||
// Count ghost refusals only on produced frames, never retries.
|
||||
if (sendSubscriberFrame(subscriber, bytes) === 'handled') {
|
||||
subscriber.pendingFrame = null
|
||||
subscriber.delivery = recordScreencastSubscriberSend(subscriber.delivery, true)
|
||||
} else {
|
||||
schedulePendingFrameRetry(subscriber)
|
||||
}
|
||||
}
|
||||
|
||||
function schedulePendingFrameRetry(subscriber: ActiveBrowserScreencastSubscriber): void {
|
||||
@@ -37,16 +84,6 @@ function schedulePendingFrameRetry(subscriber: ActiveBrowserScreencastSubscriber
|
||||
}
|
||||
subscriber.pendingFrameRetry = setTimeout(() => {
|
||||
subscriber.pendingFrameRetry = null
|
||||
const bytes = subscriber.pendingFrame
|
||||
if (!bytes) {
|
||||
return
|
||||
}
|
||||
// Count ghost refusals only on produced frames, never retries.
|
||||
if (sendRemoteBrowserScreencastFrame(subscriber.sendBinary, bytes) === 'handled') {
|
||||
subscriber.pendingFrame = null
|
||||
subscriber.delivery = recordScreencastSubscriberSend(subscriber.delivery, true)
|
||||
} else {
|
||||
schedulePendingFrameRetry(subscriber)
|
||||
}
|
||||
retryPendingFrame(subscriber)
|
||||
}, SCREENCAST_PENDING_FRAME_RETRY_MS)
|
||||
}
|
||||
|
||||
@@ -6,8 +6,13 @@ import { rendererPublicationThrottle } from '../../../window/renderer-publicatio
|
||||
import { mintBrowserScreencastSubscriptionId } from './browser-screencast-subscription-id'
|
||||
import {
|
||||
cancelScreencastSubscriberFrameRetry,
|
||||
deliverScreencastSubscriberFrame
|
||||
deliverScreencastSubscriberFrame,
|
||||
flushCreditedScreencastFrame
|
||||
} from './browser-screencast-subscriber-frame-delivery'
|
||||
import {
|
||||
createBrowserScreencastFrameCredit,
|
||||
registerBrowserScreencastFrameCredit
|
||||
} from './browser-screencast-frame-credit'
|
||||
import {
|
||||
INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY,
|
||||
screencastSubscriberIsGhost
|
||||
@@ -151,7 +156,7 @@ export class RuntimeBrowserScreencasts {
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
active.subscribers.set(subscriptionId, {
|
||||
const subscriber: ActiveBrowserScreencastSubscriber = {
|
||||
sendBinary: stream.sendBinary,
|
||||
emit: stream.emit,
|
||||
done: subscriberDone,
|
||||
@@ -161,8 +166,19 @@ export class RuntimeBrowserScreencasts {
|
||||
pendingFrame: null,
|
||||
pendingFrameRetry: null,
|
||||
pairedDeviceId: stream.pairedDeviceId,
|
||||
delivery: INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY
|
||||
})
|
||||
delivery: INITIAL_SCREENCAST_SUBSCRIBER_DELIVERY,
|
||||
frameCredit: null
|
||||
}
|
||||
if (params.frameAck === 1) {
|
||||
const credit = createBrowserScreencastFrameCredit({
|
||||
onCreditAvailable: () => flushCreditedScreencastFrame(subscriber)
|
||||
})
|
||||
subscriber.frameCredit = {
|
||||
credit,
|
||||
release: registerBrowserScreencastFrameCredit(subscriptionId, stream.pairedDeviceId, credit)
|
||||
}
|
||||
}
|
||||
active.subscribers.set(subscriptionId, subscriber)
|
||||
// A sizeless viewport owner would clear emulation for every viewer.
|
||||
if (hasScreencastViewportSize(viewport)) {
|
||||
active.viewportOwnerSubscriptionId = subscriptionId
|
||||
@@ -172,6 +188,7 @@ export class RuntimeBrowserScreencasts {
|
||||
session = await active.started
|
||||
} catch (error) {
|
||||
active.subscribers.delete(subscriptionId)
|
||||
cancelScreencastSubscriberFrameRetry(subscriber)
|
||||
resolveSubscriberDone()
|
||||
throw error
|
||||
}
|
||||
@@ -219,7 +236,8 @@ export class RuntimeBrowserScreencasts {
|
||||
subscriptionId,
|
||||
browserPageId,
|
||||
format: active.format,
|
||||
tab: this.catalog.describeBrowserTab(browserPageId, target.worktreeId)
|
||||
tab: this.catalog.describeBrowserTab(browserPageId, target.worktreeId),
|
||||
...(subscriber.frameCredit ? { frameAck: true as const } : {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
BrowserScreencastOpcode,
|
||||
decodeBrowserScreencastFrame,
|
||||
encodeBrowserScreencastFrame
|
||||
} from '../../../../shared/browser-screencast-protocol'
|
||||
import { acknowledgeBrowserScreencastFrame } from './browser-screencast-frame-credit'
|
||||
import {
|
||||
createSinglePageBrowserCommandsHost,
|
||||
createStreamedGuest
|
||||
} from '../client-pages/single-page-browser-commands-host-test-double'
|
||||
|
||||
const { webContentsFromId, startBrowserScreencast } = vi.hoisted(() => ({
|
||||
webContentsFromId: vi.fn(),
|
||||
startBrowserScreencast: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
ipcMain: { on: vi.fn(), removeListener: vi.fn() },
|
||||
webContents: { fromId: webContentsFromId }
|
||||
}))
|
||||
vi.mock('../../../browser/cdp/screencast/browser-screencast-stream', () => ({
|
||||
startBrowserScreencast
|
||||
}))
|
||||
|
||||
const PHONE = 'phone-1'
|
||||
|
||||
function frame(seq: number): Uint8Array {
|
||||
return encodeBrowserScreencastFrame({
|
||||
opcode: BrowserScreencastOpcode.Frame,
|
||||
seq,
|
||||
format: 'jpeg',
|
||||
metadata: {},
|
||||
image: new Uint8Array([seq])
|
||||
})
|
||||
}
|
||||
|
||||
async function subscribe(frameAck: boolean) {
|
||||
const { RuntimeBrowserCommands } = await import('../commands/browser-commands')
|
||||
startBrowserScreencast.mockResolvedValue({
|
||||
stop: vi.fn(),
|
||||
done: new Promise<void>(() => {}),
|
||||
updateViewport: vi.fn(async () => {})
|
||||
})
|
||||
const sent: number[] = []
|
||||
const sentAt: number[] = []
|
||||
const commands = new RuntimeBrowserCommands(createSinglePageBrowserCommandsHost())
|
||||
const subscription = await commands.browserScreencast(
|
||||
{ worktree: 'id:wt-1', page: 'page-1', format: 'jpeg', ...(frameAck ? { frameAck: 1 } : {}) },
|
||||
{
|
||||
pairedDeviceId: PHONE,
|
||||
sendBinary: (bytes) => {
|
||||
sent.push(decodeBrowserScreencastFrame(bytes)!.seq)
|
||||
sentAt.push(Date.now())
|
||||
return true
|
||||
}
|
||||
}
|
||||
)
|
||||
const onFrame: (bytes: Uint8Array) => unknown = startBrowserScreencast.mock.calls[0][1].onFrame
|
||||
const ack = (seq: number, device = PHONE) =>
|
||||
acknowledgeBrowserScreencastFrame(subscription.subscriptionId, device, seq)
|
||||
return { subscription, onFrame, sent, sentAt, ack }
|
||||
}
|
||||
|
||||
describe('screencast frame credit', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
webContentsFromId.mockReset()
|
||||
webContentsFromId.mockReturnValue(createStreamedGuest())
|
||||
startBrowserScreencast.mockReset()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('sends every frame and echoes nothing when the viewer does not ask for credit', async () => {
|
||||
const { subscription, onFrame, sent } = await subscribe(false)
|
||||
for (let seq = 0; seq < 5; seq++) {
|
||||
onFrame(frame(seq))
|
||||
}
|
||||
expect(subscription.ready.frameAck).toBeUndefined()
|
||||
expect(sent).toEqual([0, 1, 2, 3, 4])
|
||||
subscription.session.stop()
|
||||
})
|
||||
|
||||
it('holds the newest frame while two are unacked and sends it on a cumulative ack', async () => {
|
||||
const { subscription, onFrame, sent, ack } = await subscribe(true)
|
||||
expect(subscription.ready.frameAck).toBe(true)
|
||||
for (let seq = 0; seq < 4; seq++) {
|
||||
onFrame(frame(seq))
|
||||
}
|
||||
expect(sent).toEqual([0, 1])
|
||||
|
||||
expect(ack(1, 'other-phone')).toBe(false)
|
||||
expect(sent).toEqual([0, 1])
|
||||
|
||||
expect(ack(1)).toBe(true)
|
||||
expect(sent).toEqual([0, 1, 3])
|
||||
subscription.session.stop()
|
||||
})
|
||||
|
||||
// #25263: a fixed two-frame window delivered about a third of 10 fps on an idle 600 ms link.
|
||||
it('keeps 10 fps on an uncongested 600 ms round trip', async () => {
|
||||
const { subscription, onFrame, sent, sentAt, ack } = await subscribe(true)
|
||||
let acked = 0
|
||||
for (let tick = 0; tick < 100; tick++) {
|
||||
onFrame(frame(tick))
|
||||
for (let step = 0; step < 10; step++) {
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
// Each frame's ack lands 600 ms after it went out.
|
||||
while (acked < sent.length && sentAt[acked]! + 600 <= Date.now()) {
|
||||
ack(sent[acked]!)
|
||||
acked += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only frames 2–4 are skipped, superseded while the first ack is still in flight.
|
||||
expect(sent).toEqual([0, 1, ...Array.from({ length: 95 }, (_, index) => index + 5)])
|
||||
subscription.session.stop()
|
||||
})
|
||||
|
||||
// #25263: a 2 s expiry let ten unacked frames through in 10 s. Probes back off to one per 8 s.
|
||||
it('sends one probe frame per backed-off stall when no ack ever comes', async () => {
|
||||
const { subscription, onFrame, sent } = await subscribe(true)
|
||||
for (let tick = 0; tick < 300; tick++) {
|
||||
onFrame(frame(tick))
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
}
|
||||
// The base window, then probes 2 s, 4 s, 8 s and 8 s after each hold (t = 2.2, 6.2, 14.2, 22.2 s).
|
||||
expect(sent).toEqual([0, 1, 21, 61, 141, 221])
|
||||
subscription.session.stop()
|
||||
})
|
||||
})
|
||||
@@ -24,7 +24,8 @@ export function localHostedBrowserPageRequest(
|
||||
// Server workspace selectors cannot resolve here; page ids are stable.
|
||||
const { worktree: _serverSelector, ...params } = parsed.data
|
||||
const page =
|
||||
request.method === 'browser.screencast.unsubscribe'
|
||||
request.method === 'browser.screencast.unsubscribe' ||
|
||||
request.method === 'browser.screencast.ack'
|
||||
? browserScreencastSubscriptionPageId(params.subscriptionId ?? '')
|
||||
: params.page || null
|
||||
if (page === null || !hostsBrowserPage(environmentId, page)) {
|
||||
|
||||
@@ -4,7 +4,11 @@ import { BrowserError } from '../../../../browser/browser-error'
|
||||
import '../../../../../shared/runtime-types'
|
||||
import { BROWSER_UNAVAILABLE_ERROR_CODE } from '../../../../../shared/runtime-session-contracts'
|
||||
import { runtimeBrowserCommandsFactoryIsAvailable } from '../../../browser/installation/runtime-browser-commands-factory'
|
||||
import { ScreencastUnsubscribe } from '../../../../../shared/rpc-contract/browser-screencast-params'
|
||||
import {
|
||||
ScreencastAck,
|
||||
ScreencastUnsubscribe
|
||||
} from '../../../../../shared/rpc-contract/browser-screencast-params'
|
||||
import { acknowledgeBrowserScreencastFrame } from '../../../browser/screencast/browser-screencast-frame-credit'
|
||||
|
||||
export const BROWSER_SCREENCAST_METHODS = [
|
||||
defineStreamingMethod({
|
||||
@@ -40,5 +44,14 @@ export const BROWSER_SCREENCAST_METHODS = [
|
||||
runtime.cleanupSubscription(params.subscriptionId)
|
||||
return { unsubscribed: true }
|
||||
}
|
||||
}),
|
||||
defineMethod({
|
||||
name: 'browser.screencast.ack',
|
||||
permission: 'workspace',
|
||||
params: ScreencastAck,
|
||||
// Late acks for an ended stream are normal; report them instead of failing the request.
|
||||
handler: async (params, { pairedDeviceId }) => ({
|
||||
credited: acknowledgeBrowserScreencastFrame(params.subscriptionId, pairedDeviceId, params.seq)
|
||||
})
|
||||
})
|
||||
]
|
||||
|
||||
@@ -13,7 +13,8 @@ export function getRuntimeFeatureInteractionId(
|
||||
if (method === 'browser.profileClearDefaultCookies') {
|
||||
return hasBooleanResult(result, 'cleared') ? 'cookie-import' : null
|
||||
}
|
||||
if (method === 'browser.screencast.unsubscribe') {
|
||||
// Stream control (unsubscribe, frame acks) is not a browser-use interaction.
|
||||
if (method.startsWith('browser.screencast.')) {
|
||||
return null
|
||||
}
|
||||
if (method.startsWith('browser.') && isBrowserPaneUiRuntimeRpcParams(rawParams)) {
|
||||
|
||||
@@ -34,6 +34,7 @@ export const EXECUTION_HOST_METHODS = [
|
||||
'browser.mouseWheel',
|
||||
'browser.reload',
|
||||
'browser.screencast',
|
||||
'browser.screencast.ack',
|
||||
'browser.screencast.unsubscribe',
|
||||
'browser.tabCreate',
|
||||
'browser.viewport',
|
||||
|
||||
@@ -113,6 +113,14 @@ export function encodeBrowserScreencastFrame(frame: BrowserScreencastFrame): Uin
|
||||
return out
|
||||
}
|
||||
|
||||
/** The seq of an encoded frame, read from its header without decoding the metadata. */
|
||||
export function readBrowserScreencastFrameSeq(bytes: Uint8Array): number | null {
|
||||
if (bytes.byteLength < HEADER_BYTES) {
|
||||
return null
|
||||
}
|
||||
return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(4, true)
|
||||
}
|
||||
|
||||
export function decodeBrowserScreencastFrame(bytes: Uint8Array): BrowserScreencastFrame | null {
|
||||
if (bytes.byteLength < HEADER_BYTES) {
|
||||
return null
|
||||
|
||||
@@ -59,7 +59,9 @@ export const Screencast = BrowserTarget.extend({
|
||||
deviceScaleFactor: OptionalFiniteNumber,
|
||||
mobile: OptionalBoolean,
|
||||
everyNthFrame: OptionalFiniteNumber,
|
||||
minFrameIntervalMs: OptionalFiniteNumber
|
||||
minFrameIntervalMs: OptionalFiniteNumber,
|
||||
// Ask for frame credit; the host echoes `frameAck: true` in `ready`. Old hosts strip it.
|
||||
frameAck: z.literal(1).optional()
|
||||
})
|
||||
|
||||
export const FullScreenshot = BrowserTarget.extend({
|
||||
|
||||
@@ -3,3 +3,9 @@ import { z } from 'zod'
|
||||
export const ScreencastUnsubscribe = z.object({
|
||||
subscriptionId: z.string().min(1, 'Missing required --subscription-id')
|
||||
})
|
||||
|
||||
export const ScreencastAck = z.object({
|
||||
subscriptionId: z.string().min(1, 'Missing required --subscription-id'),
|
||||
// Cumulative: the newest frame seq the viewer has received.
|
||||
seq: z.number().int().nonnegative()
|
||||
})
|
||||
|
||||
+2
-1
@@ -156,7 +156,7 @@ import {
|
||||
Viewport,
|
||||
Wait
|
||||
} from './browser-params'
|
||||
import { ScreencastUnsubscribe } from './browser-screencast-params'
|
||||
import { ScreencastAck, ScreencastUnsubscribe } from './browser-screencast-params'
|
||||
import { BrowserOpenUrlParams, BrowserTabCreateParams } from './browser-tab-create-params'
|
||||
import { ClientEventsSubscribeParams, ClientEventsUnsubscribeParams } from './client-events-params'
|
||||
import {
|
||||
@@ -942,6 +942,7 @@ export const RPC_PARAMS_BY_METHOD = {
|
||||
'browser.profileList': null,
|
||||
'browser.reload': ExpectUrlBrowserTarget,
|
||||
'browser.screencast': Screencast,
|
||||
'browser.screencast.ack': ScreencastAck,
|
||||
'browser.screencast.unsubscribe': ScreencastUnsubscribe,
|
||||
'browser.screenshot': Screenshot,
|
||||
'browser.scroll': Scroll,
|
||||
|
||||
@@ -77,6 +77,8 @@ export type BrowserScreencastReadyResult = {
|
||||
browserPageId: string
|
||||
format: 'jpeg' | 'png'
|
||||
tab: BrowserTabInfo
|
||||
// Echo of `frameAck: 1`: the host holds frames until the viewer acks them.
|
||||
frameAck?: true
|
||||
}
|
||||
|
||||
export type BrowserScreencastEndResult = { type: 'end'; subscriptionId: string }
|
||||
|
||||
Reference in New Issue
Block a user