mirror of
https://github.com/daijro/camoufox.git
synced 2026-11-19 00:07:28 +00:00
Merge remote-tracking branch 'upstream/main' into pr/windows-identity
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -880,6 +880,36 @@ def server():
|
||||
launch_server()
|
||||
|
||||
|
||||
@cli.command(name="gpu-xorg-config")
|
||||
@click.argument("gpu", required=False)
|
||||
def gpu_xorg_config(gpu: Optional[str] = None) -> None:
|
||||
"""
|
||||
Print the Xorg config headless="virtual-gpu" needs for GPU
|
||||
"""
|
||||
from .exceptions import GpuDisplayError
|
||||
from .gpudisplay import select_gpu, xorg_config
|
||||
|
||||
try:
|
||||
click.echo(xorg_config(select_gpu(gpu)), nl=False)
|
||||
except GpuDisplayError as e:
|
||||
raise click.ClickException(str(e)) from e
|
||||
|
||||
|
||||
@cli.command(name="gpu-setup")
|
||||
@click.argument("gpu", required=False)
|
||||
def gpu_setup(gpu: Optional[str] = None) -> None:
|
||||
"""
|
||||
Print a root script that sets this machine up for headless="virtual-gpu" on GPU
|
||||
"""
|
||||
from .exceptions import GpuDisplayError
|
||||
from .gpudisplay import select_gpu, setup_script
|
||||
|
||||
try:
|
||||
click.echo(setup_script(select_gpu(gpu)), nl=False)
|
||||
except GpuDisplayError as e:
|
||||
raise click.ClickException(str(e)) from e
|
||||
|
||||
|
||||
class VersionInfo:
|
||||
def __init__(self):
|
||||
from rich.table import Table
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from enum import Enum
|
||||
from multiprocessing import Lock
|
||||
from threading import Lock
|
||||
from typing import List, Optional
|
||||
|
||||
from .exceptions import InvalidAddonPath
|
||||
@@ -10,6 +11,11 @@ from .pkgman import INSTALL_DIR, unzip, webdl
|
||||
# Addons are stored in a shared folder, not per-browser version
|
||||
ADDONS_DIR = INSTALL_DIR / "addons"
|
||||
|
||||
# One lock for the module: `with Lock():` on a fresh multiprocessing.Lock per call
|
||||
# guarded nothing, so concurrent launches in one process raced the same
|
||||
# check-then-extract.
|
||||
_EXTRACT_LOCK = Lock()
|
||||
|
||||
|
||||
class DefaultAddons(Enum):
|
||||
"""
|
||||
@@ -44,17 +50,35 @@ def add_default_addons(
|
||||
|
||||
addons = [addon for addon in DefaultAddons if addon not in exclude_list]
|
||||
|
||||
with Lock():
|
||||
with _EXTRACT_LOCK:
|
||||
maybe_download_addons(addons, addons_list)
|
||||
|
||||
|
||||
def download_and_extract(url: str, extract_path: str, name: str) -> None:
|
||||
"""
|
||||
Downloads and extracts an addon from a given URL to a specified path
|
||||
Downloads and extracts an addon from a given URL to a specified path.
|
||||
|
||||
Extracts into a sibling temp dir and renames it into place last, so a
|
||||
concurrent launch (another process, which the lock cannot see) never finds
|
||||
a manifest.json in a directory whose other files are still being written.
|
||||
"""
|
||||
# Create a temporary file to store the downloaded zip
|
||||
buffer = webdl(url, desc=f"Downloading addon ({name})", bar=False)
|
||||
unzip(buffer, extract_path, f"Extracting addon ({name})", bar=False)
|
||||
parent = os.path.dirname(extract_path)
|
||||
tmp_dir = tempfile.mkdtemp(prefix=f".{name}.tmp-", dir=parent)
|
||||
try:
|
||||
unzip(buffer, tmp_dir, f"Extracting addon ({name})", bar=False)
|
||||
try:
|
||||
os.replace(tmp_dir, extract_path)
|
||||
except OSError:
|
||||
# The target is not empty (or exists at all, on Windows). A complete
|
||||
# copy came from another process and may be loading in its browser,
|
||||
# so keep it; only an incomplete one is replaced.
|
||||
if os.path.exists(os.path.join(extract_path, 'manifest.json')):
|
||||
return
|
||||
shutil.rmtree(extract_path)
|
||||
os.replace(tmp_dir, extract_path)
|
||||
finally:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def get_addon_path(addon_name: str) -> str:
|
||||
@@ -92,6 +116,8 @@ def maybe_download_addons(
|
||||
if addons_list is not None:
|
||||
addons_list.append(addon_path)
|
||||
except Exception as e:
|
||||
# Drop the partial directory so the next run re-downloads.
|
||||
shutil.rmtree(addon_path, ignore_errors=True)
|
||||
# Drop the partial directory so the next run re-downloads. A complete
|
||||
# one came from another process and may be loading in its browser.
|
||||
if not os.path.exists(os.path.join(addon_path, 'manifest.json')):
|
||||
shutil.rmtree(addon_path, ignore_errors=True)
|
||||
print(f"Failed to download and extract {addon.name}: {e}")
|
||||
|
||||
@@ -23,6 +23,7 @@ from .utils import (
|
||||
new_context_options,
|
||||
persistent_context_options,
|
||||
spoofs_window_dimensions,
|
||||
start_virtual_display,
|
||||
)
|
||||
|
||||
|
||||
@@ -82,7 +83,7 @@ async def AsyncNewBrowser(
|
||||
async def AsyncNewBrowser(
|
||||
playwright: Playwright,
|
||||
*,
|
||||
headless: Optional[Union[bool, Literal['virtual']]] = None,
|
||||
headless: Optional[Union[bool, Literal['virtual', 'virtual-gpu']]] = None,
|
||||
from_options: Optional[Dict[str, Any]] = None,
|
||||
persistent_context: bool = False,
|
||||
debug: Optional[bool] = None,
|
||||
@@ -92,6 +93,11 @@ async def AsyncNewBrowser(
|
||||
Launches a new browser instance for Camoufox given a set of launch options.
|
||||
|
||||
Parameters:
|
||||
headless (Optional[Union[bool, Literal['virtual', 'virtual-gpu']]]):
|
||||
True for Firefox headless. On Linux, 'virtual' runs a headed browser on
|
||||
an Xvfb display, and 'virtual-gpu' on a headless Xorg server bound to
|
||||
the GPU `gpu` (or CAMOUFOX_GPU) names; `require_gpu=False` warns instead
|
||||
of failing when the browser does not render on it.
|
||||
from_options (Dict[str, Any]):
|
||||
A set of launch options generated by `launch_options()` to use
|
||||
persistent_context (bool):
|
||||
@@ -99,13 +105,32 @@ async def AsyncNewBrowser(
|
||||
**kwargs:
|
||||
All other keyword arguments passed to `launch_options()`.
|
||||
"""
|
||||
if headless == 'virtual':
|
||||
virtual_display = VirtualDisplay(debug=debug)
|
||||
kwargs['virtual_display'] = virtual_display.get()
|
||||
headless = False
|
||||
else:
|
||||
virtual_display = None
|
||||
virtual_display, headless = await asyncio.get_running_loop().run_in_executor(
|
||||
None, start_virtual_display, headless, debug, kwargs
|
||||
)
|
||||
# Everything after the display exists can fail (bad proxy, missing binary,
|
||||
# unsatisfiable constraints) before sync/async_attach_vd wires kill() into
|
||||
# close, which used to strand one Xvfb and its /tmp/.X<n> lock per failed
|
||||
# launch.
|
||||
try:
|
||||
return await _new_browser(
|
||||
playwright, virtual_display, headless, from_options, persistent_context, debug, kwargs
|
||||
)
|
||||
except BaseException:
|
||||
if virtual_display:
|
||||
virtual_display.kill()
|
||||
raise
|
||||
|
||||
|
||||
async def _new_browser(
|
||||
playwright: Playwright,
|
||||
virtual_display: Optional[VirtualDisplay],
|
||||
headless: Any,
|
||||
from_options: Optional[Dict[str, Any]],
|
||||
persistent_context: bool,
|
||||
debug: Optional[bool],
|
||||
kwargs: Dict[str, Any],
|
||||
) -> Union[Browser, BrowserContext]:
|
||||
# Options the caller built leave its contexts to the caller (host_identity).
|
||||
canvas_noise: Optional[bool] = False
|
||||
if not from_options:
|
||||
|
||||
@@ -172,6 +172,70 @@ class VirtualDisplayNotSupported(VirtualDisplayError):
|
||||
...
|
||||
|
||||
|
||||
class GpuDisplayError(VirtualDisplayError):
|
||||
"""
|
||||
Raised when the headless='virtual-gpu' display cannot be used.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class NoGpuFound(GpuDisplayError):
|
||||
"""
|
||||
Raised when no GPU render node with a PCI address is found.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class AmbiguousGpuSelection(GpuDisplayError):
|
||||
"""
|
||||
Raised when several GPUs match, or several exist and none was named.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class GpuNotFound(GpuDisplayError):
|
||||
"""
|
||||
Raised when the GPU selector matches no GPU.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class CannotFindXorg(GpuDisplayError):
|
||||
"""
|
||||
Raised when Xorg cannot be found.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class CannotExecuteXorg(GpuDisplayError):
|
||||
"""
|
||||
Raised when Xorg cannot be started, or exits right after starting.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class GpuVtTakeover(GpuDisplayError):
|
||||
"""
|
||||
Raised when the Xorg server switched the active console VT.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class GpuSoftwareFallback(GpuDisplayError):
|
||||
"""
|
||||
Raised when the browser on a GPU display never rendered on the GPU.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
|
||||
class CamoufoxNotInstalled(FileNotFoundError):
|
||||
"""
|
||||
Raised when camoufox is not installed.
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
"""headless='virtual-gpu': a headless Xorg server pinned to one GPU.
|
||||
|
||||
Xvfb has no GPU path, so a browser on it renders WebGL and canvas with Mesa's
|
||||
software rasterizer. This display is a real Xorg server bound to one card by
|
||||
its PCI address, drawing a virtual screen with no monitor attached, so the
|
||||
browser renders on that card. The card is always named, never guessed: the
|
||||
identity claims the GPU that renders, and two cards render differently.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import subprocess # nosec
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from shutil import which
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ._warnings import LeakWarning
|
||||
from .exceptions import (
|
||||
AmbiguousGpuSelection,
|
||||
CannotExecuteXorg,
|
||||
CannotFindXorg,
|
||||
GpuNotFound,
|
||||
GpuSoftwareFallback,
|
||||
GpuVtTakeover,
|
||||
NoGpuFound,
|
||||
)
|
||||
from .virtdisplay import VirtualDisplay
|
||||
|
||||
# Names the card when the `gpu` launch option is not given.
|
||||
GPU_ENV_VAR = 'CAMOUFOX_GPU'
|
||||
|
||||
# How long the browser has to render on the GPU after the probe page starts
|
||||
# drawing. A card busy with other work took 5-10s in Holo's measurements.
|
||||
GPU_WAIT_S = 30.0
|
||||
GPU_WAIT_ENV_VAR = 'CAMOUFOX_GPU_WAIT_S'
|
||||
|
||||
# The root window is not observable (screen.* is spoofed per context), but the
|
||||
# GPU compositor needs a real surface, so it is not 1x1 like Xvfb's.
|
||||
GPU_SCREEN = '1920x1080x24'
|
||||
|
||||
# Where Xorg looks up a relative -config when the setuid wrapper elevates it.
|
||||
SYSTEM_CONFIG_DIR = Path('/etc/X11')
|
||||
|
||||
# Without root, Xorg.wrap must let any user start Xorg (a launch from ssh,
|
||||
# systemd or a container is not a console login) and keep it root, which a
|
||||
# server with no VT of its own needs to open the card. Stock Debian and Ubuntu
|
||||
# ship allowed_users=console and needs_root_rights=auto.
|
||||
XWRAPPER_CONFIG = Path('/etc/X11/Xwrapper.config')
|
||||
XWRAPPER_SETTINGS = {'allowed_users': 'anybody', 'needs_root_rights': 'yes'}
|
||||
XORG_WRAPPERS = (Path('/usr/lib/xorg/Xorg.wrap'), Path('/usr/libexec/Xorg.wrap'))
|
||||
|
||||
# -terminate: the server exits this many seconds after its last client leaves.
|
||||
# -sharevts: never take a VT of its own, which blanks a live desktop.
|
||||
SERVER_FLAGS = ('-terminate', '2', '-nolisten', 'tcp', '-sharevts', '-novtswitch')
|
||||
|
||||
# Firefox's GPU paths, on by default for this display only; the caller's prefs
|
||||
# win. webgl.force-enabled is set by every launch already. Forcing the
|
||||
# accelerated canvas made 2D canvas match Windows Firefox on the same card.
|
||||
GPU_PREFS: Dict[str, Any] = {
|
||||
'gfx.webrender.all': True,
|
||||
'layers.acceleration.force-enabled': True,
|
||||
'gfx.x11-egl.enabled': True,
|
||||
'gfx.canvas.accelerated': True,
|
||||
'gfx.canvas.accelerated.force-enabled': True,
|
||||
}
|
||||
|
||||
# Draws one WebGL frame per animation frame: Firefox opens the GPU only when
|
||||
# something renders.
|
||||
PROBE_PAGE = (
|
||||
"data:text/html,<canvas id=c width=32 height=32></canvas><script>"
|
||||
"const gl=c.getContext('webgl2')||c.getContext('webgl');"
|
||||
"function f(){if(gl){gl.clear(gl.COLOR_BUFFER_BIT);gl.flush();}requestAnimationFrame(f);}f();"
|
||||
"</script>"
|
||||
)
|
||||
|
||||
ACTIVE_VT = Path('/sys/class/tty/tty0/active')
|
||||
# A crash right after startup (a driver fault under VRAM pressure) and a VT
|
||||
# switch both land just after the display is reported.
|
||||
SETTLE_S = 1.0
|
||||
|
||||
# An X11 connection setup request: little-endian, protocol 11.0, no auth.
|
||||
X11_SETUP = bytes([0x6C, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0])
|
||||
|
||||
VENDORS = {'0x10de': 'NVIDIA', '0x1002': 'AMD', '0x8086': 'Intel'}
|
||||
|
||||
_PCI_ADDRESS = re.compile(r'^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$')
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GpuDevice:
|
||||
"""A GPU with a DRM render node, as sysfs describes it."""
|
||||
|
||||
pci_address: str
|
||||
render_node: str
|
||||
card_node: Optional[str]
|
||||
vendor_id: str
|
||||
device_id: str
|
||||
kernel_driver: str
|
||||
# /dev/nvidia<minor> for the NVIDIA driver, which renders through it.
|
||||
nvidia_node: Optional[str] = None
|
||||
|
||||
@property
|
||||
def drm_nodes(self) -> List[str]:
|
||||
"""The card's render node and its primary node. Firefox's GPU work is
|
||||
accounted to whichever it was handed: a second launch from one process
|
||||
on an RX 7900 XTX drew through /dev/dri/card0, not renderD128."""
|
||||
return [node for node in (self.render_node, self.card_node) if node]
|
||||
|
||||
@property
|
||||
def vendor(self) -> str:
|
||||
return VENDORS.get(self.vendor_id, self.vendor_id)
|
||||
|
||||
@property
|
||||
def bus_id(self) -> str:
|
||||
"""Xorg's BusID: decimal bus:device:function, without the domain."""
|
||||
_, bus, dev_fn = self.pci_address.split(':')
|
||||
dev, fn = dev_fn.split('.')
|
||||
return f'PCI:{int(bus, 16)}:{int(dev, 16)}:{int(fn, 16)}'
|
||||
|
||||
@property
|
||||
def xorg_driver(self) -> str:
|
||||
# modesetting drives any KMS device with glamor; NVIDIA needs its own.
|
||||
return 'nvidia' if self.kernel_driver == 'nvidia' else 'modesetting'
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f'{self.pci_address} {self.vendor} [{self.vendor_id}:{self.device_id}] '
|
||||
f'driver={self.kernel_driver} {self.render_node}'
|
||||
)
|
||||
|
||||
|
||||
def drew_on_gpu(fdinfo: str) -> bool:
|
||||
"""Whether a DRM file's usage stats (its fdinfo) show work on a GPU engine.
|
||||
|
||||
Drivers report busy time (drm-engine-<engine>: <n> ns; amdgpu, i915) or busy
|
||||
cycles (drm-cycles-<engine>: <n>; xe reports only these). drm-total-cycles
|
||||
counts the GPU's clock whether this client used it or not, so it is not work.
|
||||
"""
|
||||
busy = re.findall(r'^drm-(?:engine-[\w-]+:\s*(\d+) ns|cycles-[\w-]+:\s*(\d+))$', fdinfo, re.M)
|
||||
return any(int(ns or cycles) > 0 for ns, cycles in busy)
|
||||
|
||||
|
||||
def _read(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text().strip()
|
||||
except OSError:
|
||||
return ''
|
||||
|
||||
|
||||
def _uevent(path: Path) -> Dict[str, str]:
|
||||
return dict(line.split('=', 1) for line in _read(path).splitlines() if '=' in line)
|
||||
|
||||
|
||||
def _nvidia_minor(root: Path, pci_address: str) -> Optional[str]:
|
||||
info = _read(root / 'proc/driver/nvidia/gpus' / pci_address / 'information')
|
||||
match = re.search(r'^Device Minor:\s*(\d+)', info, re.M)
|
||||
return f'/dev/nvidia{match.group(1)}' if match else None
|
||||
|
||||
|
||||
def enumerate_gpus(root: Union[str, Path] = '/') -> List[GpuDevice]:
|
||||
"""The GPUs with a render node and a PCI address, in PCI address order.
|
||||
|
||||
Ordered by PCI address, not render node number: render nodes are numbered
|
||||
in the order the drivers loaded, which can change from one boot to the next.
|
||||
"""
|
||||
root = Path(root)
|
||||
drm = root / 'sys/class/drm'
|
||||
cards = {}
|
||||
for card in drm.glob('card*'):
|
||||
if re.fullmatch(r'card\d+', card.name):
|
||||
try:
|
||||
cards[(card / 'device').resolve(strict=True)] = f'/dev/dri/{card.name}'
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
gpus = []
|
||||
for node in (root / 'dev/dri').glob('renderD*'):
|
||||
device = drm / node.name / 'device'
|
||||
try:
|
||||
real = device.resolve(strict=True)
|
||||
except OSError:
|
||||
continue
|
||||
if not _PCI_ADDRESS.match(real.name):
|
||||
continue
|
||||
driver = _uevent(device / 'uevent').get('DRIVER', '')
|
||||
gpus.append(
|
||||
GpuDevice(
|
||||
pci_address=real.name,
|
||||
render_node=f'/dev/dri/{node.name}',
|
||||
card_node=cards.get(real),
|
||||
vendor_id=_read(device / 'vendor').lower(),
|
||||
device_id=_read(device / 'device').lower(),
|
||||
kernel_driver=driver,
|
||||
nvidia_node=_nvidia_minor(root, real.name) if driver == 'nvidia' else None,
|
||||
)
|
||||
)
|
||||
return sorted(gpus, key=lambda gpu: gpu.pci_address)
|
||||
|
||||
|
||||
def _matches(index: int, gpu: GpuDevice, selector: str) -> bool:
|
||||
if selector.isdigit():
|
||||
return int(selector) == index
|
||||
ids = f"{gpu.vendor_id}:{gpu.device_id}".replace('0x', '')
|
||||
names = {gpu.render_node, os.path.basename(gpu.render_node), gpu.vendor.lower(), ids}
|
||||
if gpu.card_node:
|
||||
names |= {gpu.card_node, os.path.basename(gpu.card_node)}
|
||||
return selector.replace('0x', '') in {name.lower() for name in names} or selector in gpu.pci_address
|
||||
|
||||
|
||||
def _listing(gpus: List[GpuDevice], shown: Optional[List[GpuDevice]] = None) -> str:
|
||||
return '\n'.join(f' [{i}] {gpu}' for i, gpu in enumerate(gpus) if shown is None or gpu in shown)
|
||||
|
||||
|
||||
def select_gpu(selector: Optional[Union[str, int]] = None, gpus: Optional[List[GpuDevice]] = None) -> GpuDevice:
|
||||
"""The one GPU `selector` (or CAMOUFOX_GPU) names.
|
||||
|
||||
A selector is an index into enumerate_gpus(), a render or card node (name or
|
||||
/dev/dri path), a PCI address or part of one, vendor:device ids, or a vendor
|
||||
name. Without one, the host must have exactly one GPU.
|
||||
"""
|
||||
if gpus is None:
|
||||
gpus = enumerate_gpus()
|
||||
if not gpus:
|
||||
raise NoGpuFound(
|
||||
'virtual-gpu: no GPU render node with a PCI address. Is a GPU present, is its driver '
|
||||
'loaded, and are /dev/dri and /sys visible (--device /dev/dri in a container)?'
|
||||
)
|
||||
if selector is None:
|
||||
selector = os.environ.get(GPU_ENV_VAR, '')
|
||||
selector = str(selector).strip().lower()
|
||||
if not selector:
|
||||
if len(gpus) == 1:
|
||||
return gpus[0]
|
||||
raise AmbiguousGpuSelection(
|
||||
f'virtual-gpu: {len(gpus)} GPUs found. Name one with gpu= or {GPU_ENV_VAR}, '
|
||||
f'preferably by PCI address:\n{_listing(gpus)}'
|
||||
)
|
||||
matched = [gpu for i, gpu in enumerate(gpus) if _matches(i, gpu, selector)]
|
||||
if not matched:
|
||||
raise GpuNotFound(f'virtual-gpu: no GPU matches {selector!r}. Available:\n{_listing(gpus)}')
|
||||
if len(matched) > 1:
|
||||
raise AmbiguousGpuSelection(
|
||||
f'virtual-gpu: {selector!r} matches {len(matched)} GPUs:\n{_listing(gpus, matched)}'
|
||||
)
|
||||
return matched[0]
|
||||
|
||||
|
||||
def xorg_config(gpu: GpuDevice, screen: str = GPU_SCREEN) -> str:
|
||||
"""An xorg.conf that drives `gpu` with a virtual screen and no outputs.
|
||||
|
||||
`UseDisplayDevice "None"` keeps the NVIDIA driver off every connector, so it
|
||||
needs no DRM master and coexists with a desktop on the same card.
|
||||
"""
|
||||
width, height, depth = screen.split('x')
|
||||
offscreen = '\tOption "UseDisplayDevice" "None"\n' if gpu.xorg_driver == 'nvidia' else ''
|
||||
return f'''# Generated by camoufox for headless="virtual-gpu".
|
||||
# GPU: {gpu.pci_address} {gpu.vendor} [{gpu.vendor_id}:{gpu.device_id}] driver={gpu.kernel_driver}
|
||||
Section "ServerFlags"
|
||||
\tOption "AutoAddDevices" "false"
|
||||
\tOption "AutoAddGPU" "false"
|
||||
\tOption "DontVTSwitch" "true"
|
||||
\tOption "AllowMouseOpenFail" "true"
|
||||
EndSection
|
||||
|
||||
Section "Device"
|
||||
\tIdentifier "camoufox-gpu"
|
||||
\tDriver "{gpu.xorg_driver}"
|
||||
\tBusID "{gpu.bus_id}"
|
||||
{offscreen}\tOption "AllowEmptyInitialConfiguration" "true"
|
||||
EndSection
|
||||
|
||||
Section "Monitor"
|
||||
\tIdentifier "camoufox-monitor"
|
||||
\tHorizSync 5.0 - 1000.0
|
||||
\tVertRefresh 5.0 - 200.0
|
||||
EndSection
|
||||
|
||||
Section "Screen"
|
||||
\tIdentifier "camoufox-screen"
|
||||
\tDevice "camoufox-gpu"
|
||||
\tMonitor "camoufox-monitor"
|
||||
\tDefaultDepth {depth}
|
||||
\tSubSection "Display"
|
||||
\t\tDepth {depth}
|
||||
\t\tVirtual {width} {height}
|
||||
\tEndSubSection
|
||||
EndSection
|
||||
|
||||
Section "ServerLayout"
|
||||
\tIdentifier "camoufox-layout"
|
||||
\tScreen "camoufox-screen"
|
||||
EndSection
|
||||
'''
|
||||
|
||||
|
||||
def config_name(gpu: GpuDevice) -> str:
|
||||
"""The config's file name in /etc/X11, keyed by PCI address."""
|
||||
return f"camoufox-gpu-{re.sub(r'[:.]', '_', gpu.pci_address)}.conf"
|
||||
|
||||
|
||||
def setup_command(gpu: GpuDevice) -> str:
|
||||
"""The one command that sets this machine up for `gpu`."""
|
||||
sudo = '' if os.geteuid() == 0 else 'sudo '
|
||||
return f'python -m camoufox gpu-setup {gpu.pci_address} | {sudo}sh'
|
||||
|
||||
|
||||
def setup_script(gpu: GpuDevice) -> str:
|
||||
"""A root shell script that sets this machine up for headless="virtual-gpu"
|
||||
on `gpu`: Xorg and its setuid wrapper, the wrapper's settings, and the
|
||||
card's config. Safe to run again."""
|
||||
config = SYSTEM_CONFIG_DIR / config_name(gpu)
|
||||
xwrapper = XWRAPPER_CONFIG
|
||||
settings = ''.join(f'{key}={value}\\n' for key, value in XWRAPPER_SETTINGS.items())
|
||||
deletes = ' '.join(
|
||||
f"-e '/^[[:space:]]*{key}[[:space:]]*=/d'" for key in XWRAPPER_SETTINGS
|
||||
)
|
||||
nvidia = (
|
||||
'\n'
|
||||
'if ! find /usr/lib /usr/lib64 -name nvidia_drv.so 2>/dev/null | grep -q .; then\n'
|
||||
"\techo 'camoufox gpu-setup: no nvidia_drv.so found. Install the NVIDIA driver'\\''s Xorg "
|
||||
"module, matching the kernel driver (on Ubuntu, xserver-xorg-video-nvidia-<version>).' >&2\n"
|
||||
'fi\n'
|
||||
if gpu.xorg_driver == 'nvidia'
|
||||
else ''
|
||||
)
|
||||
return f'''#!/bin/sh
|
||||
# Generated by camoufox gpu-setup for headless="virtual-gpu".
|
||||
# GPU: {gpu}
|
||||
# Run once as root: {setup_command(gpu)}
|
||||
set -e
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
\techo 'camoufox gpu-setup: run this as root (pipe it to sudo sh).' >&2
|
||||
\texit 1
|
||||
fi
|
||||
|
||||
# 1. The Xorg server and its setuid wrapper, Xorg.wrap.
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
\tdpkg -s xserver-xorg-core xserver-xorg-legacy >/dev/null 2>&1 \\
|
||||
\t\t|| DEBIAN_FRONTEND=noninteractive apt-get install -y xserver-xorg-core xserver-xorg-legacy
|
||||
elif ! command -v Xorg >/dev/null 2>&1; then
|
||||
\techo 'camoufox gpu-setup: install the Xorg server and its setuid wrapper (Xorg.wrap), then run this again.' >&2
|
||||
\texit 1
|
||||
fi
|
||||
{nvidia}
|
||||
# 2. Let Xorg.wrap start Xorg as root for any user. A launch from ssh, systemd
|
||||
# or a container is not a console login. Any local user can then start a
|
||||
# root X server.
|
||||
touch {xwrapper}
|
||||
sed -i {deletes} {xwrapper}
|
||||
printf '{settings}' >> {xwrapper}
|
||||
|
||||
# 3. The card's config, bound by PCI address.
|
||||
cat > {config} <<'CAMOUFOX_XORG_CONF'
|
||||
{xorg_config(gpu)}CAMOUFOX_XORG_CONF
|
||||
|
||||
echo 'headless="virtual-gpu" is set up for {gpu.pci_address}.'
|
||||
'''
|
||||
|
||||
|
||||
def setup_problems() -> List[str]:
|
||||
"""What keeps Xorg.wrap from starting Xorg for this user, if anything."""
|
||||
problems = []
|
||||
if not any(path.exists() for path in XORG_WRAPPERS):
|
||||
problems.append(f"no setuid Xorg.wrap ({' or '.join(map(str, XORG_WRAPPERS))})")
|
||||
settings = {}
|
||||
for line in _read(XWRAPPER_CONFIG).splitlines():
|
||||
key, _, value = line.partition('=')
|
||||
settings[key.strip()] = value.strip()
|
||||
for key, wanted in XWRAPPER_SETTINGS.items():
|
||||
have = settings.get(key) or 'unset'
|
||||
if have != wanted:
|
||||
problems.append(f'{XWRAPPER_CONFIG} has {key}={have}, needs {key}={wanted}')
|
||||
return problems
|
||||
|
||||
|
||||
def _session_on_vt(vt: str) -> Optional[str]:
|
||||
"""The logind session on VT `vt`, which its user may activate without root."""
|
||||
try:
|
||||
sessions = subprocess.run( # nosec
|
||||
['loginctl', 'list-sessions', '--no-legend'], capture_output=True, text=True, check=True
|
||||
).stdout
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
for line in sessions.splitlines():
|
||||
session = line.split()[0]
|
||||
number = subprocess.run( # nosec
|
||||
['loginctl', 'show-session', session, '-p', 'VTNr', '--value'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
if number == vt:
|
||||
return session
|
||||
return None
|
||||
|
||||
|
||||
class GpuDisplay(VirtualDisplay):
|
||||
"""A headless Xorg server on one GPU, for headless='virtual-gpu'.
|
||||
|
||||
As root, Xorg takes a temporary config and log. Otherwise it runs through
|
||||
the setuid Xorg.wrap, which accepts only a config name relative to
|
||||
/etc/X11 and no -logfile, so the config is installed there once.
|
||||
"""
|
||||
|
||||
_name = 'Xorg'
|
||||
|
||||
def _error(self, message: str) -> CannotExecuteXorg: # type: ignore[override]
|
||||
"""A failed start, with what the machine is missing when that is why."""
|
||||
if os.geteuid() != 0:
|
||||
problems = setup_problems()
|
||||
if problems:
|
||||
message += (
|
||||
'\nheadless="virtual-gpu" without root needs Xorg.wrap to start Xorg as root:\n'
|
||||
+ ''.join(f' - {problem}\n' for problem in problems)
|
||||
+ f'Set this machine up with:\n {setup_command(self.gpu)}'
|
||||
)
|
||||
return CannotExecuteXorg(message)
|
||||
|
||||
def __init__(self, gpu: GpuDevice, debug: bool = False, require_gpu: bool = True) -> None:
|
||||
super().__init__(debug=debug, screen=GPU_SCREEN)
|
||||
self.gpu = gpu
|
||||
self.require_gpu = require_gpu
|
||||
self._client: Optional[socket.socket] = None
|
||||
self._temp_files: List[str] = []
|
||||
self._vt: Optional[str] = None
|
||||
|
||||
@property
|
||||
def xorg_path(self) -> str:
|
||||
path = which('Xorg')
|
||||
if not path:
|
||||
raise CannotFindXorg(
|
||||
'headless="virtual-gpu" needs the Xorg server. Set this machine up with:\n'
|
||||
f' {setup_command(self.gpu)}'
|
||||
)
|
||||
return path
|
||||
|
||||
def _command(self) -> List[str]:
|
||||
if os.geteuid() == 0:
|
||||
fd, config = tempfile.mkstemp(prefix='camoufox-xorg-', suffix='.conf')
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
f.write(xorg_config(self.gpu, self.screen))
|
||||
log = config[: -len('.conf')] + '.log'
|
||||
self._temp_files = [config, log]
|
||||
files = ['-config', config, '-logfile', log]
|
||||
else:
|
||||
name = config_name(self.gpu)
|
||||
if not (SYSTEM_CONFIG_DIR / name).is_file():
|
||||
raise CannotExecuteXorg(
|
||||
f'headless="virtual-gpu" without root needs {SYSTEM_CONFIG_DIR / name}. '
|
||||
f'Set this machine up with:\n {setup_command(self.gpu)}'
|
||||
)
|
||||
files = ['-config', name]
|
||||
return [self.xorg_path, *files, *SERVER_FLAGS]
|
||||
|
||||
def _wrapper(self) -> List[str]:
|
||||
# No parent-death signal: the setuid Xorg.wrap drops it on exec. The
|
||||
# held client and -terminate end the server instead (_ready).
|
||||
return []
|
||||
|
||||
def _server_env(self) -> Optional[Dict[str, str]]:
|
||||
return None
|
||||
|
||||
def _log(self) -> str:
|
||||
return self._temp_files[1] if self._temp_files else f'/var/log/Xorg.{self._display}.log'
|
||||
|
||||
def get(self) -> str:
|
||||
self._vt = _read(ACTIVE_VT) or None
|
||||
return super().get()
|
||||
|
||||
def _starting(self) -> None:
|
||||
if self._vt and _read(ACTIVE_VT) != self._vt:
|
||||
self._vt_taken()
|
||||
|
||||
def _ready(self) -> None:
|
||||
deadline = time.monotonic() + SETTLE_S
|
||||
while time.monotonic() < deadline:
|
||||
self._starting()
|
||||
if self.proc is not None and self.proc.poll() is not None:
|
||||
error = self._error(
|
||||
f'Xorg exited ({self.proc.returncode}) right after starting. See {self._log()}.'
|
||||
)
|
||||
self.kill()
|
||||
raise error
|
||||
time.sleep(0.1)
|
||||
# The server exits at its last client's disconnect (-terminate), and a
|
||||
# server no client ever reached never does. Holding one client from here
|
||||
# on means it always has a last client: this one, closed by kill() or by
|
||||
# the kernel when this process dies. An unprivileged launcher may not be
|
||||
# able to signal a root Xorg, so this is the teardown that always works.
|
||||
self._client = socket.socket(socket.AF_UNIX)
|
||||
try:
|
||||
self._client.connect(f'/tmp/.X11-unix/X{self._display}')
|
||||
self._client.sendall(X11_SETUP)
|
||||
except OSError as e:
|
||||
self.kill()
|
||||
raise self._error(f'Could not connect to the Xorg display it reported: {e}') from e
|
||||
|
||||
def _vt_taken(self) -> None:
|
||||
vt = self._vt or ''
|
||||
number = vt.removeprefix('tty')
|
||||
session = _session_on_vt(number)
|
||||
if session:
|
||||
subprocess.run(['loginctl', 'activate', session], capture_output=True) # nosec
|
||||
if _read(ACTIVE_VT) != vt:
|
||||
subprocess.run(['chvt', number], capture_output=True) # nosec
|
||||
restored = _read(ACTIVE_VT) == vt
|
||||
self.kill()
|
||||
raise GpuVtTakeover(
|
||||
f'Xorg on {self.gpu} switched the active VT away from {vt}. '
|
||||
+ ('It was switched back.' if restored else f'Switch back with `sudo chvt {number}`.')
|
||||
)
|
||||
|
||||
def _display_processes(self) -> List[int]:
|
||||
"""Every process started with DISPLAY set to this display, except Xorg and us."""
|
||||
needle = f'\0DISPLAY=:{self._display}\0'.encode()
|
||||
skip = {os.getpid(), self.proc.pid if self.proc else None}
|
||||
pids = []
|
||||
for entry in Path('/proc').iterdir():
|
||||
if not entry.name.isdigit() or int(entry.name) in skip:
|
||||
continue
|
||||
try:
|
||||
if needle in b'\0' + (entry / 'environ').read_bytes():
|
||||
pids.append(int(entry.name))
|
||||
except OSError:
|
||||
continue
|
||||
return pids
|
||||
|
||||
def uses_gpu(self) -> bool:
|
||||
"""Whether a process on this display renders on the GPU.
|
||||
|
||||
Firefox opens the card's DRM nodes for DMABuf even when it renders in
|
||||
software, so an open render node proves nothing. Rendering is the NVIDIA
|
||||
driver's /dev/nvidia<minor> held open, or work the kernel accounted to a
|
||||
GPU engine on one of the card's DRM nodes.
|
||||
"""
|
||||
nodes = self.gpu.drm_nodes
|
||||
for pid in self._display_processes():
|
||||
try:
|
||||
fds = list(Path(f'/proc/{pid}/fd').iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
for fd in fds:
|
||||
try:
|
||||
target = os.readlink(fd)
|
||||
if target == self.gpu.nvidia_node or (
|
||||
target in nodes
|
||||
and drew_on_gpu(Path(f'/proc/{pid}/fdinfo/{fd.name}').read_text())
|
||||
):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
def wait_for_gpu(self) -> bool:
|
||||
deadline = time.monotonic() + float(os.environ.get(GPU_WAIT_ENV_VAR) or GPU_WAIT_S)
|
||||
while time.monotonic() < deadline:
|
||||
if self.uses_gpu():
|
||||
return True
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
def check_gpu(self, active: bool) -> None:
|
||||
"""Raise, or warn with require_gpu=False, when the browser did not render on the GPU."""
|
||||
if active:
|
||||
return
|
||||
if not self.require_gpu:
|
||||
LeakWarning.warn('gpu_software_rendering')
|
||||
return
|
||||
raise GpuSoftwareFallback(
|
||||
f'The browser on :{self._display} never rendered on {self.gpu}, so it renders in software. '
|
||||
f'Firefox may have blocklisted the driver or failed to create a GPU context. '
|
||||
f'See {self._log()}.'
|
||||
)
|
||||
|
||||
def kill(self) -> None:
|
||||
if self._client:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
if self._display is not None:
|
||||
# The browser's helpers (dbus, at-spi) are not in its process group.
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
for pid in self._display_processes():
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
except OSError:
|
||||
pass
|
||||
deadline = time.monotonic() + 1.5
|
||||
while self._display_processes() and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
# SIGTERM lets Xorg remove its root-owned socket, which this process
|
||||
# cannot; base kill() SIGKILLs only a server still running after that.
|
||||
if self.proc is not None and self.proc.poll() is None:
|
||||
try:
|
||||
self.proc.terminate()
|
||||
self.proc.wait(timeout=10)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
super().kill()
|
||||
for path in self._temp_files:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
self._temp_files = []
|
||||
self._display = None
|
||||
|
||||
|
||||
def prove_gpu(target: Any, display: GpuDisplay) -> None:
|
||||
"""Draw in a page of `target` and check the browser rendered on the GPU."""
|
||||
page = target.new_page()
|
||||
try:
|
||||
page.goto(PROBE_PAGE)
|
||||
active = display.wait_for_gpu()
|
||||
finally:
|
||||
page.close()
|
||||
display.check_gpu(active)
|
||||
|
||||
|
||||
async def async_prove_gpu(target: Any, display: GpuDisplay) -> None:
|
||||
import asyncio
|
||||
|
||||
page = await target.new_page()
|
||||
try:
|
||||
await page.goto(PROBE_PAGE)
|
||||
active = await asyncio.get_running_loop().run_in_executor(None, display.wait_for_gpu)
|
||||
finally:
|
||||
await page.close()
|
||||
display.check_gpu(active)
|
||||
@@ -16,6 +16,7 @@ from functools import lru_cache
|
||||
from typing import Any, Dict, Mapping, Optional, Tuple
|
||||
|
||||
from .fingerprints import is_software_renderer
|
||||
from .gpudisplay import GPU_PREFS
|
||||
|
||||
BASELINE_OVERRIDES_PREF = 'privacy.baselineFingerprintingProtection.overrides'
|
||||
|
||||
@@ -76,22 +77,31 @@ def set_webgpu(prefs: Dict[str, Any], target_os: str, gpu: Gpu, host: Optional[G
|
||||
prefs.setdefault(WEBGPU_BLOCKLIST_PREF, FEATURE_STATUS_OK if own_adapter else FEATURE_BLOCKED_DEVICE)
|
||||
|
||||
|
||||
def host_gpu(executable_path: str, headless: bool, env: Mapping[str, Any]) -> Optional[Gpu]:
|
||||
def host_gpu(
|
||||
executable_path: str, headless: bool, env: Mapping[str, Any], prefs: Mapping[str, Any]
|
||||
) -> Optional[Gpu]:
|
||||
"""The (vendor, renderer) this machine's Firefox reports, or None without WebGL.
|
||||
|
||||
Read from the binary about to launch, with no identity applied, under the
|
||||
same display: headless, Xvfb and a real display can each render on a
|
||||
different device. The strings are the ones Firefox sanitizes itself.
|
||||
same display and the same rendering prefs: headless, Xvfb, a GPU display
|
||||
and a real display can each render on a different device. The strings are
|
||||
the ones Firefox sanitizes itself.
|
||||
"""
|
||||
return _host_gpu(
|
||||
str(executable_path),
|
||||
bool(headless),
|
||||
tuple(sorted((k, str(v)) for k, v in env.items() if not k.startswith(_LAUNCHER_ENV))),
|
||||
tuple(sorted((k, prefs[k]) for k in GPU_PREFS if k in prefs)),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _host_gpu(executable_path: str, headless: bool, env: Tuple[Tuple[str, str], ...]) -> Optional[Gpu]:
|
||||
def _host_gpu(
|
||||
executable_path: str,
|
||||
headless: bool,
|
||||
env: Tuple[Tuple[str, str], ...],
|
||||
prefs: Tuple[Tuple[str, Any], ...],
|
||||
) -> Optional[Gpu]:
|
||||
def read() -> Optional[Gpu]:
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
@@ -101,7 +111,7 @@ def _host_gpu(executable_path: str, headless: bool, env: Tuple[Tuple[str, str],
|
||||
headless=headless,
|
||||
env=dict(env),
|
||||
# The launch forces WebGL on, which can choose another device.
|
||||
firefox_user_prefs={'webgl.force-enabled': True},
|
||||
firefox_user_prefs={**dict(prefs), 'webgl.force-enabled': True},
|
||||
)
|
||||
try:
|
||||
gpu = browser.new_page().evaluate(_READ_GPU)
|
||||
|
||||
@@ -85,6 +85,13 @@ def validate_ip(ip: str) -> None:
|
||||
raise InvalidIP(f"Invalid IP address: {ip}")
|
||||
|
||||
|
||||
# Probing the exit IP through a proxy adds a hop and a TLS handshake behind it; 5 s
|
||||
# is not enough for a proxy that routes the long way round, and every URL timing out
|
||||
# fails the whole launch. Direct probes keep 5 s.
|
||||
DIRECT_IP_TIMEOUT = 5
|
||||
PROXIED_IP_TIMEOUT = 15
|
||||
|
||||
|
||||
def proxy_exit_geo(proxy: str) -> Tuple[str, str]:
|
||||
"""
|
||||
The exit IP of `proxy` and that IP's timezone, looked up through the proxy.
|
||||
@@ -95,7 +102,7 @@ def proxy_exit_geo(proxy: str) -> Tuple[str, str]:
|
||||
resp = requests.get(
|
||||
"http://ip-api.com/json?fields=status,message,query,timezone",
|
||||
proxies=Proxy.as_requests_proxy(proxy),
|
||||
timeout=10,
|
||||
timeout=PROXIED_IP_TIMEOUT,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
@@ -121,7 +128,7 @@ def public_ipv6() -> Optional[str]:
|
||||
"""
|
||||
for url in ("https://api6.ipify.org", "https://v6.ident.me"):
|
||||
try:
|
||||
resp = requests.get(url, timeout=5, verify=True)
|
||||
resp = requests.get(url, timeout=DIRECT_IP_TIMEOUT, verify=True)
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException:
|
||||
continue
|
||||
@@ -153,7 +160,7 @@ def public_ip(proxy: Optional[str] = None) -> str:
|
||||
resp = requests.get(
|
||||
url,
|
||||
proxies=Proxy.as_requests_proxy(proxy) if proxy else None,
|
||||
timeout=5,
|
||||
timeout=PROXIED_IP_TIMEOUT if proxy else DIRECT_IP_TIMEOUT,
|
||||
verify=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, NoReturn, Tuple, Union
|
||||
from typing import Any, Dict, NoReturn, Optional, Tuple, Union
|
||||
|
||||
import base64
|
||||
import orjson
|
||||
from playwright._impl._driver import compute_driver_executable
|
||||
|
||||
from camoufox.pkgman import LOCAL_DATA
|
||||
from camoufox.utils import launch_options
|
||||
from camoufox.gpudisplay import GpuDisplay, prove_gpu
|
||||
from camoufox.utils import launch_options, start_virtual_display
|
||||
from camoufox.virtdisplay import VirtualDisplay
|
||||
|
||||
LAUNCH_SCRIPT: Path = LOCAL_DATA / "launchServer.js"
|
||||
@@ -61,21 +67,46 @@ def launch_server(**kwargs) -> NoReturn:
|
||||
)
|
||||
kwargs.pop(unsupported, None)
|
||||
|
||||
virtual_display = None
|
||||
if kwargs.get('headless') == 'virtual':
|
||||
virtual_display = VirtualDisplay(debug=kwargs.get('debug'))
|
||||
kwargs['virtual_display'] = virtual_display.get()
|
||||
kwargs['headless'] = False
|
||||
virtual_display, kwargs['headless'] = start_virtual_display(
|
||||
kwargs.get('headless'), kwargs.get('debug'), kwargs
|
||||
)
|
||||
try:
|
||||
_serve(launch_options(**kwargs))
|
||||
_serve(launch_options(**kwargs), virtual_display)
|
||||
finally:
|
||||
if virtual_display:
|
||||
virtual_display.kill()
|
||||
|
||||
|
||||
def _serve(config: Dict[str, Any]) -> NoReturn:
|
||||
def _prove_served_gpu(process: subprocess.Popen, display: GpuDisplay) -> None:
|
||||
"""Pass the server's output through, and once it prints its endpoint, check
|
||||
that its browser renders on the GPU."""
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
def prove(endpoint: str) -> None:
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.firefox.connect(endpoint)
|
||||
try:
|
||||
prove_gpu(browser, display)
|
||||
finally:
|
||||
browser.close()
|
||||
|
||||
assert process.stdout is not None
|
||||
for line in process.stdout:
|
||||
sys.stdout.write(line)
|
||||
sys.stdout.flush()
|
||||
endpoint = re.search(r'ws://\S+', line)
|
||||
if endpoint:
|
||||
# The sync API refuses to start inside a running asyncio loop.
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
pool.submit(prove, endpoint.group(0)).result()
|
||||
break
|
||||
threading.Thread(target=shutil.copyfileobj, args=(process.stdout, sys.stdout), daemon=True).start()
|
||||
|
||||
|
||||
def _serve(config: Dict[str, Any], virtual_display: Optional[VirtualDisplay] = None) -> NoReturn:
|
||||
"""Run launchServer.js with `config` until the Node process exits."""
|
||||
nodejs = get_nodejs()
|
||||
gpu_display = virtual_display if isinstance(virtual_display, GpuDisplay) else None
|
||||
|
||||
data = orjson.dumps(to_camel_case_dict(config))
|
||||
|
||||
@@ -90,6 +121,7 @@ def _serve(config: Dict[str, Any]) -> NoReturn:
|
||||
],
|
||||
cwd=driver_package,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE if gpu_display else None,
|
||||
text=True,
|
||||
)
|
||||
# Send one newline-delimited configuration frame, then keep the pipe open.
|
||||
@@ -126,6 +158,8 @@ def _serve(config: Dict[str, Any]) -> NoReturn:
|
||||
try:
|
||||
stdin.write(encoded_config)
|
||||
stdin.flush()
|
||||
if gpu_display:
|
||||
_prove_served_gpu(process, gpu_display)
|
||||
process.wait()
|
||||
except OSError as error:
|
||||
# If Node failed before reading the frame, preserve its exit status
|
||||
|
||||
@@ -20,6 +20,7 @@ from .utils import (
|
||||
new_context_options,
|
||||
persistent_context_options,
|
||||
spoofs_window_dimensions,
|
||||
start_virtual_display,
|
||||
sync_attach_vd,
|
||||
)
|
||||
|
||||
@@ -84,7 +85,7 @@ def NewBrowser(
|
||||
def NewBrowser(
|
||||
playwright: Playwright,
|
||||
*,
|
||||
headless: Optional[Union[bool, Literal['virtual']]] = None,
|
||||
headless: Optional[Union[bool, Literal['virtual', 'virtual-gpu']]] = None,
|
||||
from_options: Optional[Dict[str, Any]] = None,
|
||||
persistent_context: bool = False,
|
||||
debug: Optional[bool] = None,
|
||||
@@ -94,6 +95,11 @@ def NewBrowser(
|
||||
Launches a new browser instance for Camoufox given a set of launch options.
|
||||
|
||||
Parameters:
|
||||
headless (Optional[Union[bool, Literal['virtual', 'virtual-gpu']]]):
|
||||
True for Firefox headless. On Linux, 'virtual' runs a headed browser on
|
||||
an Xvfb display, and 'virtual-gpu' on a headless Xorg server bound to
|
||||
the GPU `gpu` (or CAMOUFOX_GPU) names; `require_gpu=False` warns instead
|
||||
of failing when the browser does not render on it.
|
||||
from_options (Dict[str, Any]):
|
||||
A set of launch options generated by `launch_options()` to use
|
||||
persistent_context (bool):
|
||||
@@ -101,13 +107,28 @@ def NewBrowser(
|
||||
**kwargs:
|
||||
All other keyword arguments passed to `launch_options()`.
|
||||
"""
|
||||
if headless == 'virtual':
|
||||
virtual_display = VirtualDisplay(debug=debug)
|
||||
kwargs['virtual_display'] = virtual_display.get()
|
||||
headless = False
|
||||
else:
|
||||
virtual_display = None
|
||||
virtual_display, headless = start_virtual_display(headless, debug, kwargs)
|
||||
# Everything after the display exists can fail (bad proxy, missing binary,
|
||||
# unsatisfiable constraints) before sync/async_attach_vd wires kill() into
|
||||
# close, which used to strand one Xvfb and its /tmp/.X<n> lock per failed
|
||||
# launch.
|
||||
try:
|
||||
return _launch(playwright, virtual_display, headless, from_options, persistent_context, debug, kwargs)
|
||||
except BaseException:
|
||||
if virtual_display:
|
||||
virtual_display.kill()
|
||||
raise
|
||||
|
||||
|
||||
def _launch(
|
||||
playwright: Playwright,
|
||||
virtual_display: Optional[VirtualDisplay],
|
||||
headless: Any,
|
||||
from_options: Optional[Dict[str, Any]],
|
||||
persistent_context: bool,
|
||||
debug: Optional[bool],
|
||||
kwargs: Dict[str, Any],
|
||||
) -> Union[Browser, BrowserContext]:
|
||||
# Options the caller built leave its contexts to the caller (host_identity).
|
||||
canvas_noise: Optional[bool] = False
|
||||
if not from_options:
|
||||
|
||||
@@ -43,6 +43,7 @@ from .pkgman import (
|
||||
selected_launch_path,
|
||||
warn_if_package_outdated,
|
||||
)
|
||||
from .gpudisplay import GPU_PREFS, GpuDisplay, async_prove_gpu, prove_gpu, select_gpu
|
||||
from .virtdisplay import VirtualDisplay
|
||||
from ._warnings import FallbackWarning, LeakWarning
|
||||
from .host_rendering import Gpu, add_canvas_placeholder, has_canvas_placeholder, host_gpu, renders_on_hardware, set_webgpu
|
||||
@@ -796,7 +797,7 @@ def host_identity(from_options: Dict[str, Any], canvas_noise: Optional[bool]) ->
|
||||
host_os = _host_os_key()
|
||||
covered, gpu = plan_rendering(
|
||||
host_os, None, False, bool(prefs.get('webgl.disabled')),
|
||||
lambda: host_gpu(from_options['executable_path'], from_options['headless'], from_options['env']),
|
||||
lambda: host_gpu(from_options['executable_path'], from_options['headless'], from_options['env'], prefs),
|
||||
)
|
||||
return None if covered else (host_os, gpu)
|
||||
|
||||
@@ -846,11 +847,37 @@ def new_context_options(
|
||||
return opts
|
||||
|
||||
|
||||
def start_virtual_display(
|
||||
headless: Any, debug: Optional[bool], kwargs: Dict[str, Any]
|
||||
) -> Tuple[Optional[VirtualDisplay], Any]:
|
||||
"""Start the display headless='virtual' or 'virtual-gpu' asks for.
|
||||
|
||||
Pops the display's own options from `kwargs` and points the launch at the
|
||||
display. Returns the display, or None, and the headless value to launch with.
|
||||
"""
|
||||
gpu = kwargs.pop('gpu', None)
|
||||
require_gpu = kwargs.pop('require_gpu', None)
|
||||
if headless == 'virtual-gpu':
|
||||
display: VirtualDisplay = GpuDisplay(
|
||||
select_gpu(gpu), debug=bool(debug), require_gpu=require_gpu is not False
|
||||
)
|
||||
kwargs['firefox_user_prefs'] = {**GPU_PREFS, **(kwargs.get('firefox_user_prefs') or {})}
|
||||
elif gpu is not None or require_gpu is not None:
|
||||
raise ValueError("gpu and require_gpu apply to headless='virtual-gpu' only")
|
||||
elif headless == 'virtual':
|
||||
display = VirtualDisplay(debug=debug)
|
||||
else:
|
||||
return None, headless
|
||||
kwargs['virtual_display'] = display.get()
|
||||
return display, False
|
||||
|
||||
|
||||
async def async_attach_vd(
|
||||
browser: Any, virtual_display: Optional[VirtualDisplay] = None
|
||||
) -> Any: # type: ignore
|
||||
"""
|
||||
Attaches the virtual display to the async browser cleanup
|
||||
Attaches the virtual display to the async browser cleanup, and on a GPU
|
||||
display checks the browser renders on the GPU.
|
||||
"""
|
||||
if not virtual_display: # Skip if no virtual display is provided
|
||||
return browser
|
||||
@@ -865,6 +892,13 @@ async def async_attach_vd(
|
||||
|
||||
browser.close = new_close
|
||||
|
||||
if isinstance(virtual_display, GpuDisplay):
|
||||
try:
|
||||
await async_prove_gpu(browser, virtual_display)
|
||||
except BaseException:
|
||||
await browser.close()
|
||||
raise
|
||||
|
||||
return browser
|
||||
|
||||
|
||||
@@ -872,7 +906,8 @@ def sync_attach_vd(
|
||||
browser: Any, virtual_display: Optional[VirtualDisplay] = None
|
||||
) -> Any: # type: ignore
|
||||
"""
|
||||
Attaches the virtual display to the sync browser cleanup
|
||||
Attaches the virtual display to the sync browser cleanup, and on a GPU
|
||||
display checks the browser renders on the GPU.
|
||||
"""
|
||||
if not virtual_display: # Skip if no virtual display is provided
|
||||
return browser
|
||||
@@ -887,6 +922,13 @@ def sync_attach_vd(
|
||||
|
||||
browser.close = new_close
|
||||
|
||||
if isinstance(virtual_display, GpuDisplay):
|
||||
try:
|
||||
prove_gpu(browser, virtual_display)
|
||||
except BaseException:
|
||||
browser.close()
|
||||
raise
|
||||
|
||||
return browser
|
||||
|
||||
|
||||
@@ -1002,7 +1044,8 @@ def launch_options(
|
||||
headless (Optional[bool]):
|
||||
Whether to run the browser in headless mode. Defaults to False.
|
||||
Note: If you are running linux, passing headless='virtual' to Camoufox, AsyncCamoufox
|
||||
or launch_server will use Xvfb.
|
||||
or launch_server will use Xvfb, and headless='virtual-gpu' a headless Xorg
|
||||
server bound to one GPU.
|
||||
main_world_eval (Optional[bool]):
|
||||
Whether to enable running scripts in the main world.
|
||||
To use this, prepend "mw:" to the script: page.evaluate("mw:" + script).
|
||||
@@ -1576,7 +1619,7 @@ def launch_options(
|
||||
canvas_noise,
|
||||
bool(webgl_config or (config.get('webGl:vendor') and config.get('webGl:renderer'))),
|
||||
bool(block_webgl),
|
||||
lambda: host_gpu(browser_binary, headless, env),
|
||||
lambda: host_gpu(browser_binary, headless, env, firefox_user_prefs),
|
||||
)
|
||||
if cover_canvas:
|
||||
add_canvas_placeholder(firefox_user_prefs)
|
||||
|
||||
@@ -4,7 +4,7 @@ import select
|
||||
import subprocess # nosec
|
||||
import time
|
||||
from shutil import which
|
||||
from typing import Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from camoufox.exceptions import (
|
||||
CannotExecuteXvfb,
|
||||
@@ -13,8 +13,10 @@ from camoufox.exceptions import (
|
||||
)
|
||||
from camoufox.pkgman import OS_NAME
|
||||
|
||||
# Safe timeout for Xvfb writing display num, prevents infinite hang.
|
||||
DISPLAYFD_READ_TIMEOUT_S = 10.0
|
||||
# Safe timeout for Xvfb writing display num, prevents infinite hang. A cold first
|
||||
# Xvfb on a fresh host took 0.6-8.4 s to report (lang315/camoufox#157), so 10 s
|
||||
# failed launches that would have succeeded.
|
||||
DISPLAYFD_READ_TIMEOUT_S = 30.0
|
||||
|
||||
# Xvfb screen geometry for headless="virtual". The root window size is not
|
||||
# observable (see virtual-display-default-1x1x24 in ci/tribal-rules.yml);
|
||||
@@ -94,24 +96,43 @@ class VirtualDisplay:
|
||||
raise CannotFindXvfb("Please install util-linux (for setpriv) to use headless mode.")
|
||||
return path
|
||||
|
||||
# What get() calls the server in its errors, and what it raises.
|
||||
_name = "Xvfb"
|
||||
_error = CannotExecuteXvfb
|
||||
|
||||
def _wrapper(self) -> List[str]:
|
||||
"""What execs the server. setpriv's parent-death signal stops Xvfb when
|
||||
this process dies without reaching kill(), even by SIGKILL; SIGTERM lets
|
||||
Xvfb remove its own lock and socket. See xvfb-dies-with-its-launcher in
|
||||
ci/tribal-rules.yml."""
|
||||
return [self.setpriv_path, "--pdeathsig", "TERM"]
|
||||
|
||||
def _command(self) -> List[str]:
|
||||
"""The server's argv, less -displayfd, which get() adds."""
|
||||
return [self.xvfb_path, *self.xvfb_args]
|
||||
|
||||
def _server_env(self) -> Optional[Dict[str, str]]:
|
||||
# Force Mesa software GLX; we don't use the GPU anyway.
|
||||
return {**os.environ, "__GLX_VENDOR_LIBRARY_NAME": "mesa", "LIBGL_ALWAYS_SOFTWARE": "1"}
|
||||
|
||||
def _starting(self) -> None:
|
||||
"""Called about every 100ms while the server has not reported a display."""
|
||||
|
||||
def _ready(self) -> None:
|
||||
"""Called once the server has reported its display."""
|
||||
|
||||
def get(self) -> str:
|
||||
self._assert_linux()
|
||||
|
||||
if self.proc is None:
|
||||
# Launch Xvfb with -displayfd so Xvfb itself picks a free display
|
||||
# Launch the server with -displayfd so it picks a free display
|
||||
# number atomically and reports it back. Avoids userspace races.
|
||||
# subprocess.Popen's pass_fds keeps an fd at its parent number in
|
||||
# the child (unlike Node's `stdio: [..., 'pipe']` which renumbers
|
||||
# to 3), so we tell Xvfb that exact number.
|
||||
# to 3), so we tell the server that exact number.
|
||||
program, *args = self._command()
|
||||
read_fd, write_fd = os.pipe()
|
||||
# setpriv's parent-death signal stops Xvfb when this process dies
|
||||
# without reaching kill(), even by SIGKILL; SIGTERM lets Xvfb remove
|
||||
# its own lock and socket. See xvfb-dies-with-its-launcher in
|
||||
# ci/tribal-rules.yml.
|
||||
cmd = [
|
||||
self.setpriv_path, "--pdeathsig", "TERM",
|
||||
self.xvfb_path, "-displayfd", str(write_fd), *self.xvfb_args,
|
||||
]
|
||||
cmd = [*self._wrapper(), program, "-displayfd", str(write_fd), *args]
|
||||
if self.debug:
|
||||
print("Starting virtual display:", " ".join(cmd))
|
||||
self.proc = subprocess.Popen( # nosec
|
||||
@@ -121,31 +142,29 @@ class VirtualDisplay:
|
||||
stderr=None if self.debug else subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
pass_fds=(write_fd,),
|
||||
env={
|
||||
**os.environ,
|
||||
# Force Mesa software GLX; we don't use the GPU anyway.
|
||||
"__GLX_VENDOR_LIBRARY_NAME": "mesa",
|
||||
"LIBGL_ALWAYS_SOFTWARE": "1",
|
||||
},
|
||||
env=self._server_env(),
|
||||
)
|
||||
os.close(write_fd) # so the read end EOFs when Xvfb closes its end
|
||||
os.close(write_fd) # so the read end EOFs when the server closes its end
|
||||
|
||||
buf = b""
|
||||
deadline = time.monotonic() + DISPLAYFD_READ_TIMEOUT_S
|
||||
try:
|
||||
while b"\n" not in buf:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0 or not select.select([read_fd], [], [], remaining)[0]:
|
||||
if remaining <= 0:
|
||||
self.kill()
|
||||
raise CannotExecuteXvfb(
|
||||
f"Xvfb did not report a display within "
|
||||
raise self._error(
|
||||
f"{self._name} did not report a display within "
|
||||
f"{int(DISPLAYFD_READ_TIMEOUT_S * 1000)}ms"
|
||||
)
|
||||
if not select.select([read_fd], [], [], min(remaining, 0.1))[0]:
|
||||
self._starting()
|
||||
continue
|
||||
chunk = os.read(read_fd, 64)
|
||||
if not chunk:
|
||||
self.kill()
|
||||
raise CannotExecuteXvfb(
|
||||
f"Xvfb did not report a display "
|
||||
raise self._error(
|
||||
f"{self._name} did not report a display "
|
||||
f"(got {buf!r}, exit={self.proc.poll()})"
|
||||
)
|
||||
buf += chunk
|
||||
@@ -156,7 +175,8 @@ class VirtualDisplay:
|
||||
self._display = int(buf.strip())
|
||||
except ValueError:
|
||||
self.kill()
|
||||
raise CannotExecuteXvfb(f"Xvfb wrote non-integer display: {buf!r}")
|
||||
raise self._error(f"{self._name} wrote non-integer display: {buf!r}")
|
||||
self._ready()
|
||||
elif self.debug:
|
||||
print(f"Using virtual display: {self._display}")
|
||||
|
||||
|
||||
@@ -63,6 +63,11 @@ canvas_noise: >-
|
||||
the identity claims another OS or GPU than this machine's, whose canvas would
|
||||
show the real one. Pass `canvas_noise=False` to keep stock canvas behaviour.
|
||||
|
||||
gpu_software_rendering: >-
|
||||
The browser on the headless="virtual-gpu" display never rendered on the GPU, so it
|
||||
renders canvas and WebGL in software, which a page can tell from the GPU the
|
||||
identity claims. Pass `require_gpu=True` (the default) to fail the launch instead.
|
||||
|
||||
block_images: >-
|
||||
Blocking image requests has been reported to cause detection issues on major WAFs.
|
||||
|
||||
|
||||
@@ -51,6 +51,13 @@ _HOST_DEPENDENT_EXTENSIONS = frozenset({'OVR_multiview2'})
|
||||
# unconditionally on D3D11 (renderer11_utils.cpp); 146's ANGLE did not.
|
||||
_BACKEND_EXTENSIONS = {'win': ('EXT_depth_clamp',)}
|
||||
|
||||
# Firefox 158 (bug 1854003) exposes EXT_polygon_offset_clamp wherever the GL
|
||||
# backend clamps polygon offsets: ANGLE's D3D11 backend from feature level 10_0,
|
||||
# which WebGL2 needs, and Linux desktop GL drivers (ARB_polygon_offset_clamp,
|
||||
# core in GL 4.6), whose recordings have EXT_depth_clamp. macOS release builds
|
||||
# run WebGL on CGL's GL 4.1, which lacks it.
|
||||
_POLYGON_OFFSET_CLAMP = 'EXT_polygon_offset_clamp'
|
||||
|
||||
_ALIASED_LINE_WIDTH_RANGE = '33902'
|
||||
|
||||
# What Firefox reports under privacy.resistFingerprinting. A Camoufox identity
|
||||
@@ -111,10 +118,22 @@ def firefox_gpus(target_os: str) -> FrozenSet[Tuple[str, str]]:
|
||||
)
|
||||
|
||||
|
||||
def _context_config(prefix: str, webgl: Dict[str, Any], target_os: str) -> Dict[str, Any]:
|
||||
def _backend_extensions(target_os: str, webgl: Dict[str, Any], webgl2: Any) -> Tuple[str, ...]:
|
||||
"""The extensions the shipped Firefox adds on this device's backend."""
|
||||
extensions = _BACKEND_EXTENSIONS.get(target_os, ())
|
||||
if (target_os == 'win' and webgl2) or (
|
||||
target_os == 'lin' and 'EXT_depth_clamp' in webgl['supportedExtensions']
|
||||
):
|
||||
extensions += (_POLYGON_OFFSET_CLAMP,)
|
||||
return extensions
|
||||
|
||||
|
||||
def _context_config(
|
||||
prefix: str, webgl: Dict[str, Any], target_os: str, backend: Tuple[str, ...]
|
||||
) -> Dict[str, Any]:
|
||||
blocked = _filtered_extensions(target_os)
|
||||
extensions = [extension for extension in webgl['supportedExtensions'] if extension not in blocked]
|
||||
for extension in _BACKEND_EXTENSIONS.get(target_os, ()):
|
||||
for extension in backend:
|
||||
if extension not in extensions:
|
||||
# Firefox lists them in WebGLExtensionID order: alphabetical, ignoring case.
|
||||
at = next((i for i, other in enumerate(extensions) if other.casefold() > extension.casefold()), None)
|
||||
@@ -134,14 +153,15 @@ def to_config(webgl: Dict[str, Any], webgl2: Any, target_os: str) -> Dict[str, A
|
||||
|
||||
`webgl2` is `[]` for a device without WebGL2.
|
||||
"""
|
||||
backend = _backend_extensions(target_os, webgl, webgl2)
|
||||
config = {
|
||||
'webGl:vendor': webgl['vendor'],
|
||||
'webGl:renderer': webgl['renderer'],
|
||||
**_context_config('webGl', webgl, target_os),
|
||||
**_context_config('webGl', webgl, target_os, backend),
|
||||
'webGl2Enabled': bool(webgl2),
|
||||
}
|
||||
if webgl2:
|
||||
config.update(_context_config('webGl2', webgl2, target_os))
|
||||
config.update(_context_config('webGl2', webgl2, target_os, backend))
|
||||
# Firefox 156 creates WebGL1 on the same GL profile as WebGL2, so both
|
||||
# report one line width range. 146 forced WebGL1 onto a compatibility
|
||||
# profile, and fpgen recorded that profile's driver range.
|
||||
|
||||
Reference in New Issue
Block a user