fix(browser): stop an over-limit screencast frame from killing the paired runtime socket

A screencast frame larger than the 8 MiB outbound binary limit made the host's
E2EE channel close the whole runtime socket with 1013, and the producer read the
resulting `false` as backpressure and retried the identical frame forever. That
is a permanent condition treated as transient, so the connection could never
recover on its own.

Admit frames at the paired-runtime boundary: an over-limit frame is reported as
handled so the stream advances to the next frame instead of retrying a doomed
one, and it never reaches the transport. Genuine backpressure (`false` from the
transport, including the pre-`ready` gate) still round-trips as before.

Also apply the caller's maxWidth/maxHeight to the navigation snapshot capture.
That path fed `capturePage` device pixels straight into the encoder, so it could
emit a frame several times larger than the live path is allowed to send - which
is what makes an over-limit frame reachable on an ordinary page load.

Refs STA-2970
This commit is contained in:
Jinwoo-H
2026-08-05 04:14:06 -04:00
parent 249645832f
commit 265b0aeeb9
6 changed files with 303 additions and 3 deletions
@@ -0,0 +1,94 @@
import { Buffer } from 'node:buffer'
import { EventEmitter } from 'node:events'
import { describe, expect, it, vi } from 'vitest'
import { startBrowserScreencast } from './browser-screencast-stream'
function createMockWebContents(capturePage: () => Promise<unknown>) {
let attached = false
const dbg = new EventEmitter() as EventEmitter & {
isAttached: ReturnType<typeof vi.fn>
attach: ReturnType<typeof vi.fn>
detach: ReturnType<typeof vi.fn>
sendCommand: ReturnType<typeof vi.fn>
}
dbg.isAttached = vi.fn(() => attached)
dbg.attach = vi.fn(() => {
attached = true
})
dbg.detach = vi.fn(() => {
attached = false
})
dbg.sendCommand = vi.fn(async () => ({}))
return { isDestroyed: vi.fn(() => false), debugger: dbg, capturePage: vi.fn(capturePage) }
}
function createCapturedImage(width: number, height: number) {
const resized = {
getSize: vi.fn(() => ({ width, height })),
resize: vi.fn(),
toJPEG: vi.fn(() => Buffer.from('scaled-frame')),
toPNG: vi.fn(() => Buffer.from('scaled-frame'))
}
const image = {
getSize: vi.fn(() => ({ width, height })),
resize: vi.fn(() => resized),
toJPEG: vi.fn(() => Buffer.from('captured-frame')),
toPNG: vi.fn(() => Buffer.from('captured-frame'))
}
return { image, resized }
}
describe('browser screencast snapshot scaling', () => {
it('scales a hi-DPI capture down to the requested frame bounds', async () => {
const { image, resized } = createCapturedImage(4000, 3000)
const webContents = createMockWebContents(async () => image)
const onFrame = vi.fn()
const session = await startBrowserScreencast(webContents as never, {
format: 'jpeg',
quality: 70,
maxWidth: 1440,
maxHeight: 1200,
viewportWidth: 2000,
viewportHeight: 1500,
deviceScaleFactor: 2,
everyNthFrame: 2,
minFrameIntervalMs: 0,
onFrame
})
await vi.waitFor(() => expect(onFrame).toHaveBeenCalledTimes(1))
expect(image.resize).toHaveBeenCalledWith({ width: 1440, height: 1080 })
expect(resized.toJPEG).toHaveBeenCalledWith(70)
expect(image.toJPEG).not.toHaveBeenCalled()
session.stop()
await session.done
})
it('leaves a capture already within the frame bounds unscaled', async () => {
const { image } = createCapturedImage(1200, 800)
const webContents = createMockWebContents(async () => image)
const onFrame = vi.fn()
const session = await startBrowserScreencast(webContents as never, {
format: 'jpeg',
quality: 70,
maxWidth: 1440,
maxHeight: 1200,
viewportWidth: 1200,
viewportHeight: 800,
everyNthFrame: 2,
minFrameIntervalMs: 0,
onFrame
})
await vi.waitFor(() => expect(onFrame).toHaveBeenCalledTimes(1))
expect(image.resize).not.toHaveBeenCalled()
expect(image.toJPEG).toHaveBeenCalledWith(70)
session.stop()
await session.done
})
})
+17 -2
View File
@@ -1,6 +1,6 @@
/* eslint-disable max-lines -- Why: screencast setup, CDP lifecycle, metadata normalization, and stream teardown stay together so frame behavior cannot drift across files. */
import { Buffer } from 'node:buffer'
import type { WebContents } from 'electron'
import type { NativeImage, WebContents } from 'electron'
import {
BrowserScreencastOpcode,
encodeBrowserScreencastFrame,
@@ -86,6 +86,20 @@ function scaleToFit(
}
}
// Why: capturePage returns device pixels, so a hi-DPI viewport yields a bitmap several times
// larger than the live path is allowed to send. Apply the caller's cap here too.
function scaleSnapshotToFit(image: NativeImage, options: BrowserScreencastOptions): NativeImage {
const size = image.getSize()
if (!size.width || !size.height) {
return image
}
const fitted = scaleToFit(size.width, size.height, options.maxWidth, options.maxHeight)
if (fitted.width === size.width && fitted.height === size.height) {
return image
}
return image.resize(fitted)
}
function isNearSize(
actual: { width: number; height: number },
expected: { width: number; height: number }
@@ -518,8 +532,9 @@ export async function startBrowserScreencast(
width: viewportWidth,
height: viewportHeight
})
const capture = scaleSnapshotToFit(nativeImage, options)
const buffer =
options.format === 'png' ? nativeImage.toPNG() : nativeImage.toJPEG(options.quality)
options.format === 'png' ? capture.toPNG() : capture.toJPEG(options.quality)
if (buffer.byteLength > 0) {
image = new Uint8Array(buffer)
}
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { AgentBrowserBridge } from '../browser/agent-browser-bridge'
import { REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES } from '../../shared/remote-runtime-memory-limits'
import type { RuntimeBrowserCommandHost } from './orca-runtime-browser'
const {
@@ -579,6 +580,32 @@ describe('RuntimeBrowserCommands browser screencast', () => {
await second.session.done
expect(secondStop).toHaveBeenCalledTimes(1)
}, 10_000)
it('admits screencast frames through the paired-runtime size guard', async () => {
const { RuntimeBrowserCommands } = await import('./orca-runtime-browser')
webContentsFromIdMock.mockReturnValue({ isDestroyed: () => false })
const done = deferred<void>()
startBrowserScreencastMock.mockResolvedValue({
stop: vi.fn(() => done.resolve()),
done: done.promise
})
const sendBinary = vi.fn(() => true)
const commands = new RuntimeBrowserCommands(createHost())
const started = await commands.browserScreencast(
{ worktree: 'id:wt-1', page: 'page-1', format: 'jpeg' },
{ sendBinary }
)
const { onFrame } = startBrowserScreencastMock.mock.calls[0][1]
expect(onFrame(new Uint8Array(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1))).toBe(true)
expect(sendBinary).not.toHaveBeenCalled()
expect(onFrame(new Uint8Array(64))).toBe(true)
expect(sendBinary).toHaveBeenCalledTimes(1)
started.session.stop()
await started.session.done
})
})
describe('RuntimeBrowserCommands headless offscreen routing', () => {
+2 -1
View File
@@ -68,6 +68,7 @@ import {
selectBrowserProfile
} from '../browser/browser-cookie-import'
import { waitForTabRegistration, waitForWorktreeTabRegistration } from '../ipc/browser'
import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission'
export type BrowserCommandTargetParams = {
worktree?: string
@@ -507,7 +508,7 @@ export class RuntimeBrowserCommands {
mobile: params.mobile === true,
everyNthFrame: clampInteger(params.everyNthFrame, 1, 10, 2),
minFrameIntervalMs: clampInteger(params.minFrameIntervalMs, 0, 1000, 0),
onFrame: stream.sendBinary,
onFrame: (bytes) => sendRemoteBrowserScreencastFrame(stream.sendBinary, bytes),
onEvent: stream.emit,
onError: (message) => stream.emit?.({ type: 'error', message })
})
@@ -0,0 +1,149 @@
import { Buffer } from 'node:buffer'
import { EventEmitter } from 'node:events'
import { describe, expect, it, vi } from 'vitest'
import { deriveSharedKey, encrypt, generateKeyPair } from '../../shared/e2ee-crypto'
import { REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES } from '../../shared/remote-runtime-memory-limits'
import { startBrowserScreencast } from '../browser/browser-screencast-stream'
import { E2EEChannel } from './rpc/e2ee-channel'
import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission'
function createMockWebContents() {
let attached = false
const dbg = new EventEmitter() as EventEmitter & {
isAttached: ReturnType<typeof vi.fn>
attach: ReturnType<typeof vi.fn>
detach: ReturnType<typeof vi.fn>
sendCommand: ReturnType<typeof vi.fn>
}
dbg.isAttached = vi.fn(() => attached)
dbg.attach = vi.fn(() => {
attached = true
})
dbg.detach = vi.fn(() => {
attached = false
})
dbg.sendCommand = vi.fn(async () => ({}))
return { isDestroyed: vi.fn(() => false), debugger: dbg }
}
// Why: exercises the real E2EE channel so the oracle fails if an over-limit frame ever reaches
// the transport that answers it with close code 1013.
function createRuntimeBinarySender() {
const serverKeys = generateKeyPair()
const clientKeys = generateKeyPair()
const ws = {
OPEN: 1 as const,
readyState: 1,
send: vi.fn(),
close: vi.fn()
}
const onTransportError = vi.fn((code: number, reason: string) => ws.close(code, reason))
const channel = new E2EEChannel(ws as never, {
serverSecretKey: serverKeys.secretKey,
resolveAuthenticatedDevice: (token) =>
token === 'valid-token'
? { deviceId: 'device-1', deviceToken: token, scope: 'runtime' }
: null,
onReady: vi.fn(),
onError: onTransportError
})
const sharedKey = deriveSharedKey(clientKeys.secretKey, serverKeys.publicKey)
channel.handleRawMessage(
JSON.stringify({
type: 'e2ee_hello',
publicKeyB64: Buffer.from(clientKeys.publicKey).toString('base64')
})
)
channel.handleRawMessage(
encrypt(JSON.stringify({ type: 'e2ee_auth', deviceToken: 'valid-token' }), sharedKey)
)
let sendBinary: ((bytes: Uint8Array<ArrayBufferLike>) => boolean | void) | undefined
channel.onMessage((_plaintext, _sendText, sendBinaryReply) => {
sendBinary = sendBinaryReply
})
channel.handleRawMessage(encrypt('start-screencast', sharedKey))
if (!sendBinary) {
throw new Error('Runtime binary sender was not established')
}
return { channel, ws, onTransportError, sendBinary }
}
describe('sendRemoteBrowserScreencastFrame', () => {
it('reports an over-limit frame as handled so the producer does not retry it', () => {
const sendBinary = vi.fn(() => true)
const oversized = new Uint8Array(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1)
expect(sendRemoteBrowserScreencastFrame(sendBinary, oversized)).toBe(true)
expect(sendBinary).not.toHaveBeenCalled()
})
it('still forwards backpressure for a frame the transport can accept', () => {
const withinLimit = new Uint8Array(1024)
expect(
sendRemoteBrowserScreencastFrame(
vi.fn(() => false),
withinLimit
)
).toBe(false)
expect(
sendRemoteBrowserScreencastFrame(
vi.fn(() => true),
withinLimit
)
).toBe(true)
expect(sendRemoteBrowserScreencastFrame(vi.fn(), withinLimit)).toBe(true)
})
it('drops an oversized encoded frame without closing or stalling the paired stream', async () => {
vi.useFakeTimers()
const webContents = createMockWebContents()
const transport = createRuntimeBinarySender()
const session = await startBrowserScreencast(webContents as never, {
format: 'jpeg',
quality: 70,
maxWidth: 3840,
maxHeight: 2160,
everyNthFrame: 1,
minFrameIntervalMs: 0,
onFrame: (bytes) => sendRemoteBrowserScreencastFrame(transport.sendBinary, bytes)
})
try {
webContents.debugger.emit('message', {}, 'Page.screencastFrame', {
data: Buffer.alloc(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1).toString('base64'),
sessionId: 42,
metadata: {}
})
await Promise.resolve()
expect(transport.onTransportError).not.toHaveBeenCalled()
expect(transport.ws.close).not.toHaveBeenCalled()
expect(webContents.debugger.sendCommand).toHaveBeenCalledWith('Page.screencastFrameAck', {
sessionId: 42
})
const sendsAfterDrop = transport.ws.send.mock.calls.length
await vi.advanceTimersByTimeAsync(500)
expect(transport.ws.send).toHaveBeenCalledTimes(sendsAfterDrop)
webContents.debugger.emit('message', {}, 'Page.screencastFrame', {
data: Buffer.from('next-frame').toString('base64'),
sessionId: 43,
metadata: {}
})
await Promise.resolve()
expect(transport.onTransportError).not.toHaveBeenCalled()
expect(webContents.debugger.sendCommand).toHaveBeenCalledWith('Page.screencastFrameAck', {
sessionId: 43
})
expect(transport.ws.send.mock.calls.length).toBeGreaterThan(sendsAfterDrop)
} finally {
session.stop()
await session.done
transport.channel.destroy()
vi.useRealTimers()
}
})
})
@@ -0,0 +1,14 @@
import { isRemoteRuntimeBinaryFrameWithinLimit } from '../../shared/remote-runtime-memory-limits'
// Why: the E2EE channel closes the socket (1013) on an over-limit binary frame, and the
// screencast producer reads `false` as backpressure and retries the identical frame. Reporting
// an over-limit frame as handled drops it so the stream advances instead of retrying forever.
export function sendRemoteBrowserScreencastFrame(
sendBinary: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void,
bytes: Uint8Array<ArrayBufferLike>
): boolean {
if (!isRemoteRuntimeBinaryFrameWithinLimit(bytes)) {
return true
}
return sendBinary(bytes) !== false
}