From fa8a93577e828a44331c39ef112bcdd1a564422e Mon Sep 17 00:00:00 2001 From: Jake Writer Date: Tue, 28 Jul 2026 11:58:09 -0600 Subject: [PATCH] fix(juggler): record video headful and under a virtual display (#93) After the screencastFrameAck/timestamp fix, recording worked headless but still produced nothing usable anywhere else: `headless="virtual"` and plain headful both emitted a valid .webm containing 24 pure-white frames -- Playwright's filler for a screencast that never delivered a frame. nsScreencastService only has a working source when the browser is headless (HeadlessWindowCapturer). Outside headless, CreateWindowCapturer falls through to libwebrtc's X11 window capturer, which fails three different ways: * no XComposite -> startVideoRecording() succeeds and then never delivers a frame. This is Camoufox's own Xvfb configuration, which passes `-extension COMPOSITE`; * XComposite enabled -> the browser segfaults during capture (reproduced on the shipped 152.0.4-beta.28 as well, so it is not specific to this branch); * Wayland -> nsWindow::GetNativeData(NS_NATIVE_WINDOW_WEBRTC_DEVICE_ID) is documented as unhandled and returns null, so the service throws NS_ERROR_FAILURE ("Failed to get native window id") and no capture starts. Capture from the compositor instead when not headless, via WindowGlobalParent.drawSnapshot() -- the same call Page.screenshot already uses, which is why screenshots have always worked in every mode. It renders page content directly and does not care about the windowing system. The tick is ack-driven, mirroring nsScreencastService's kMaxFramesInFlight = 1, so a slow consumer throttles capture rather than queueing JPEGs. Headless keeps the native C++ capturer, which is cheaper and already correct. Measured on the packaged Linux build, 3s recording of an animated page, frames decoded to PNG and inspected rather than trusting file existence: before after headless 100 frames, real unchanged, real headless="virtual" 24 frames, all white 100 frames, real headful (Xvfb, X11) 24 frames, all white 99 frames, real headful (Wayland env) no capture at all 99 frames, real tests/async/test_video.py passes 5/5 both headless and headful. Enabling Composite no longer crashes either, since X11 window capture is now unused. Co-Authored-By: Claude Opus 5 (1M context) --- additions/juggler/TargetRegistry.js | 138 ++++++++++++++++++++++++++-- pythonlib/camoufox/virtdisplay.py | 16 ++-- 2 files changed, 139 insertions(+), 15 deletions(-) diff --git a/additions/juggler/TargetRegistry.js b/additions/juggler/TargetRegistry.js index f65bb68..746d475 100644 --- a/additions/juggler/TargetRegistry.js +++ b/additions/juggler/TargetRegistry.js @@ -7,6 +7,9 @@ const {Preferences} = ChromeUtils.importESModule("resource://gre/modules/Prefere const {ContextualIdentityService} = ChromeUtils.importESModule("resource://gre/modules/ContextualIdentityService.sys.mjs"); const {NetUtil} = ChromeUtils.importESModule('resource://gre/modules/NetUtil.sys.mjs'); const {AppConstants} = ChromeUtils.importESModule("resource://gre/modules/AppConstants.sys.mjs"); +// This module's scope has no timer globals (unlike the content-side juggler +// scripts), so the screencast tick has to import them explicitly. +const {setTimeout, clearTimeout} = ChromeUtils.importESModule("resource://gre/modules/Timer.sys.mjs"); const Cr = Components.results; @@ -15,6 +18,12 @@ const helper = new Helper(); const IDENTITY_NAME = 'JUGGLER '; const HUNDRED_YEARS = 60 * 60 * 24 * 365 * 100; +// Capture rate for the compositor-backed screencast. Playwright muxes at 25fps +// and repeats frames to fill gaps, so this is an upper bound on capture cost +// rather than the video's frame rate; the ack-driven backpressure in +// _startSnapshotScreencast lowers it further whenever encoding cannot keep up. +const SNAPSHOT_SCREENCAST_FPS = 25; + const ALL_PERMISSIONS = [ 'geo', 'desktop-notification', @@ -865,6 +874,27 @@ export class PageTarget { if (width < 10 || width > 10000 || height < 10 || height > 10000) throw new Error("Invalid size"); + // nsScreencastService only has a working capture source when the browser is + // headless (HeadlessWindowCapturer). Outside headless it falls through to + // libwebrtc's X11 window capturer, which does not work here in either + // configuration: + // + // * without the XComposite extension, startVideoRecording() succeeds and + // then never delivers a single frame -- the recording silently comes + // out as Playwright's blank filler; + // * with XComposite enabled, the browser segfaults during capture; + // * on Wayland it cannot start at all, because + // nsWindow::GetNativeData(NS_NATIVE_WINDOW_WEBRTC_DEVICE_ID) is + // documented as unhandled there and returns null, so the service throws + // NS_ERROR_FAILURE ("Failed to get native window id"). + // + // Capture from the compositor instead. drawSnapshot() is what + // Page.screenshot already uses, it renders the page content directly, and + // it is independent of the windowing system -- so headful, Xvfb and Wayland + // all record identically. + if (!Services.appinfo.headless) + return this._startSnapshotScreencast({ width, height, quality }); + // Firefox 152 renamed `ownerGlobal` to `documentGlobal` on nodes. const docShell = (this._gBrowser.documentGlobal || this._gBrowser.ownerGlobal).docShell; // Exclude address bar and navigation control from the video. @@ -896,24 +926,118 @@ export class PageTarget { return { screencastId }; } + // Compositor-backed screencast, used whenever the native capturer has no + // usable source (see startScreencast). Frames come from the same + // drawSnapshot() call Page.screenshot uses, so this works headful, under Xvfb + // and on Wayland alike. + _startSnapshotScreencast({ width, height, quality }) { + const screencastId = Services.uuid.generateUUID().toString().replace(/[{}-]/g, ''); + const jpegQuality = Math.min(Math.max(quality ?? 90, 0), 100) / 100; + const state = { + stopped: false, + // Mirrors nsScreencastService's kMaxFramesInFlight = 1: hold the next + // capture until the client acks the previous frame, so a slow consumer + // throttles the capture instead of queueing unbounded JPEGs. + inFlight: false, + }; + this._screencastRecordingInfo = { screencastId, snapshotState: state }; + + const captureFrame = async () => { + const browsingContext = this.linkedBrowser()?.browsingContext; + const windowGlobal = browsingContext?.currentWindowGlobal; + if (!windowGlobal) + return; + + const viewport = this._viewportSize || this._browserContext.defaultViewportSize; + const rect = viewport + ? new DOMRect(0, 0, viewport.width, viewport.height) + : this.linkedBrowser().getBoundingClientRect(); + if (!rect.width || !rect.height) + return; + + // Fit inside the requested frame without distorting; ffmpeg pads the rest. + const scale = Math.min(width / rect.width, height / rect.height); + const frameWidth = Math.max(1, Math.round(rect.width * scale)); + const frameHeight = Math.max(1, Math.round(rect.height * scale)); + + // drawSnapshot rejects with NS_ERROR_LOSS_OF_SIGNIFICANT_DATA while a + // navigation is in flight. Drop that frame rather than ending the video. + let snapshot; + try { + snapshot = await windowGlobal.drawSnapshot( + new DOMRect(0, 0, rect.width, rect.height), scale, 'rgb(255,255,255)'); + } catch (e) { + return; + } + if (state.stopped) { + snapshot.close(); + return; + } + + const doc = this._window.document; + const canvas = doc.createElementNS('http://www.w3.org/1999/xhtml', 'canvas'); + canvas.width = frameWidth; + canvas.height = frameHeight; + canvas.getContext('2d').drawImage(snapshot, 0, 0); + snapshot.close(); + + const dataURL = canvas.toDataURL('image/jpeg', jpegQuality); + state.inFlight = true; + this.emit(PageTarget.Events.ScreencastFrame, { + data: dataURL.substring(dataURL.indexOf(',') + 1), + deviceWidth: frameWidth, + deviceHeight: frameHeight, + timestamp: Date.now() / 1000, + }); + }; + + const intervalMs = 1000 / SNAPSHOT_SCREENCAST_FPS; + const tick = async () => { + if (state.stopped) + return; + if (!state.inFlight) { + try { + await captureFrame(); + } catch (e) { + dump(`juggler: snapshot screencast frame failed: ${e}\n`); + } + } + if (!state.stopped) + state.timer = setTimeout(tick, intervalMs); + }; + state.timer = setTimeout(tick, 0); + + return { screencastId }; + } + screencastFrameAck({ screencastId }) { - if (!this._screencastRecordingInfo) + const info = this._screencastRecordingInfo; + if (!info) return; // A client that omits the id is acking whatever is currently recording -- // there can only be one screencast per page target. Only reject an id that // is present and refers to some other (stale) session. - const activeId = this._screencastRecordingInfo.screencastId; - if (screencastId !== undefined && screencastId !== activeId) + if (screencastId !== undefined && screencastId !== info.screencastId) return; - screencastService.screencastFrameAck(activeId); + if (info.snapshotState) { + info.snapshotState.inFlight = false; + return; + } + screencastService.screencastFrameAck(info.screencastId); } stopScreencast() { - if (!this._screencastRecordingInfo) + const info = this._screencastRecordingInfo; + if (!info) throw new Error('No screencast in progress'); - const { screencastId } = this._screencastRecordingInfo; this._screencastRecordingInfo = undefined; - screencastService.stopVideoRecording(screencastId); + if (info.snapshotState) { + info.snapshotState.stopped = true; + if (info.snapshotState.timer) + clearTimeout(info.snapshotState.timer); + return; + } + screencastService.stopVideoRecording(info.screencastId); } ensureContextMenuClosed() { diff --git a/pythonlib/camoufox/virtdisplay.py b/pythonlib/camoufox/virtdisplay.py index 897451c..650911a 100644 --- a/pythonlib/camoufox/virtdisplay.py +++ b/pythonlib/camoufox/virtdisplay.py @@ -32,19 +32,19 @@ SCREEN_ENV_VAR = "CAMOUFOX_VIRTUAL_DISPLAY_SIZE" # The Composite extension, disabled by default (Xvfb's `-extension COMPOSITE`). # # This was briefly enabled by default on the theory that #93 (no video under -# headless="virtual") was caused by disabling it. Measurement says otherwise: +# headless="virtual") was caused by disabling it. It was not: #93 was a juggler +# bug, fixed by capturing the screencast from the compositor instead of from +# libwebrtc's X11 window capturer. Both states were measured before that fix: # # composite off, record_video_dir -> a valid .webm of 24 pure-white frames # composite ON, record_video_dir -> browser dies with SIGSEGV, no video # composite ON, no recording -> fine # -# So compositing does not fix #93, and turning it on converts a blank recording -# into a crash whenever someone records under a virtual display. Reproduced on -# both this build and the shipped 152.0.4-beta.28, so the segfault is in the -# screencast capture path, not something this branch introduced. -# -# Left as an opt-in for hosts with real GL where it may behave differently: -# set CAMOUFOX_VIRTUAL_DISPLAY_COMPOSITE=1 to enable it. +# The segfault was inside the X11 capturer, which the browser no longer uses, so +# enabling Composite is no longer dangerous -- but it is also no longer good for +# anything, since recording never touches X11 window capture now. Leave it off +# (Camoufox's long-standing default) and keep the escape hatch: +# CAMOUFOX_VIRTUAL_DISPLAY_COMPOSITE=1 enables it. COMPOSITE_ENV_VAR = "CAMOUFOX_VIRTUAL_DISPLAY_COMPOSITE"