From 7f0e52e7920725115e9e90197c38afa42269a188 Mon Sep 17 00:00:00 2001 From: coffeegrind123 <103264516+coffeegrind123@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:10:00 +0300 Subject: [PATCH] Install the pinned fpgen model before fpgen is imported pythonlib left the model to fpgen, whose first import downloads the first release the GitHub API lists: model-4/2025, whose WebGL records have no vendor or renderer. Every generated launch on a fresh install failed with KeyError: 'vendor'. scripts/pin-fpgen-model.py pinned the model for CI only, and fpgen's five-week refresh replaced even that pin, under a stamp the script's --check still trusted. fpgen is now imported only through fpgen_model.load_fpgen(), which installs the release named by the pin, checks the archive and each file against their sha256, and dates the files past fpgen's refresh. It writes the layout and stamp the script and the TypeScript launcher use, verifies an install by hashing it, and leaves FPGEN_MODEL_URL to fpgen. `camoufox fetch` installs the model as well. The pin gains each file's sha256; the package carries a copy of it, and the script now calls the module. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit f043de359878571278b7f2e50374e94279eb809b) --- pythonlib/README.md | 2 + pythonlib/camoufox/__main__.py | 2 + pythonlib/camoufox/exceptions.py | 6 + pythonlib/camoufox/fingerprints.py | 4 +- pythonlib/camoufox/fpgen-model.json | 19 ++ pythonlib/camoufox/fpgen_model.py | 168 ++++++++++++++ pythonlib/camoufox/webgl.py | 9 +- pythonlib/tests/test_fpgen_model.py | 336 ++++++++++++++++++++++++++++ scripts/data/fpgen-model.json | 7 +- scripts/pin-fpgen-model.py | 138 ++---------- typescript/src/fpgen/pin.ts | 12 +- 11 files changed, 569 insertions(+), 134 deletions(-) create mode 100644 pythonlib/camoufox/fpgen-model.json create mode 100644 pythonlib/camoufox/fpgen_model.py create mode 100644 pythonlib/tests/test_fpgen_model.py diff --git a/pythonlib/README.md b/pythonlib/README.md index 0659c4f0..8871e2cd 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -37,6 +37,8 @@ Next, download the Camoufox browser: camoufox fetch ``` +`fetch` also installs fpgen's model, pinned by sha256, into fpgen's package directory. Otherwise the first generated fingerprint installs it. Run `fetch` as that directory's owner if the browser will run as another user, e.g. while building a Docker image. + To uninstall, run `camoufox remove`. --- diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index de1dd11c..81e44ff7 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -11,6 +11,7 @@ from typing import Any, List, Optional, Tuple import rich_click as click from .addons import DefaultAddons, maybe_download_addons +from .fpgen_model import ensure_fpgen_model from .geolocation import ( ALLOW_GEOIP, GEOIP_DIR, @@ -323,6 +324,7 @@ def fetch(version): if ALLOW_GEOIP: download_mmdb() maybe_download_addons(list(DefaultAddons)) + ensure_fpgen_model() def _set_channel(repo_name: str, channel_type: str): diff --git a/pythonlib/camoufox/exceptions.py b/pythonlib/camoufox/exceptions.py index 292e992e..9be0e780 100644 --- a/pythonlib/camoufox/exceptions.py +++ b/pythonlib/camoufox/exceptions.py @@ -184,3 +184,9 @@ class ProfileDirectoryError(RuntimeError): """Raised when Camoufox's required runtime directory cannot be prepared.""" ... + + +class FpgenModelError(RuntimeError): + """Raised when fpgen's pinned model cannot be installed where fpgen reads it.""" + + ... diff --git a/pythonlib/camoufox/fingerprints.py b/pythonlib/camoufox/fingerprints.py index b2b4f891..ae6fb92a 100644 --- a/pythonlib/camoufox/fingerprints.py +++ b/pythonlib/camoufox/fingerprints.py @@ -29,9 +29,9 @@ _FP_GENERATOR = None def _generator(): global _FP_GENERATOR if _FP_GENERATOR is None: - from fpgen import Generator + from .fpgen_model import load_fpgen - _FP_GENERATOR = Generator() + _FP_GENERATOR = load_fpgen().Generator() return _FP_GENERATOR diff --git a/pythonlib/camoufox/fpgen-model.json b/pythonlib/camoufox/fpgen-model.json new file mode 100644 index 00000000..7016ee11 --- /dev/null +++ b/pythonlib/camoufox/fpgen-model.json @@ -0,0 +1,19 @@ +{ + "tag": "model-2/2026", + "asset": "model-release.zip", + "size": 1564571, + "sha256": "6530b8322cdaa4ec042921c8d9a0369a0e6e0269ba636c01a7203e4a2f109936", + "url": "https://github.com/scrapfly/fingerprint-generator/releases/download/model-2/2026/model-release.zip", + "repo": "scrapfly/fingerprint-generator", + "files": [ + "fingerprint-network.json.zst", + "values.dat.zst", + "values.json.zst" + ], + "file_sha256": { + "fingerprint-network.json.zst": "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4", + "values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", + "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f" + }, + "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate." +} diff --git a/pythonlib/camoufox/fpgen_model.py b/pythonlib/camoufox/fpgen_model.py new file mode 100644 index 00000000..28d39b88 --- /dev/null +++ b/pythonlib/camoufox/fpgen_model.py @@ -0,0 +1,168 @@ +"""fpgen's model, installed from the pinned release before fpgen is imported. + +Left to itself, `import fpgen` downloads its model from the first release the +GitHub API lists, with TLS verification off and no checksum, and downloads it +again whenever the files are five weeks old (fpgen/pkgman.py). The first +release listed is model-4/2025, whose WebGL records have no vendor or renderer, +so every generated launch failed with KeyError: 'vendor'. + +So fpgen is only ever imported through load_fpgen(), which first installs the +release named by fpgen-model.json (a copy of scripts/data/fpgen-model.json), +checks the archive and each file against its sha256, and dates the files in the +future so fpgen's five-week refresh never replaces them. The data directory +layout, including the `.pinned-model` stamp, is the one +scripts/pin-fpgen-model.py and the TypeScript launcher write. +""" + +import hashlib +import importlib.util +import json +import os +import shutil +import sys +import tempfile +import time +from pathlib import Path +from threading import Lock +from types import ModuleType +from typing import Optional +from zipfile import ZipFile + +from .exceptions import CorruptedDownload, FpgenModelError +from .pkgman import verify_sha256, webdl + +PIN = json.loads((Path(__file__).parent / 'fpgen-model.json').read_text(encoding='utf-8')) + +# Holds the archive's sha256, written last: a stamp means the install finished. +STAMP = '.pinned-model' + +# `python -m fpgen decompress` replaces the archive's files with these, which +# fpgen then reads in preference to them. +DECOMPRESSED_FILES = ('fingerprint-network.json', 'values.json', 'values.dat') + +# 2100-01-01T00:00:00Z. fpgen refetches files whose mtime is older than this +# window (pkgman.files_are_recent). +PINNED_MTIME = 4102444800 +FPGEN_MAX_AGE_S = 5 * 7 * 24 * 3600 + +# fpgen fetches a model from this URL instead of GitHub. A caller who set it +# chose their model, so it is left alone. +CUSTOM_MODEL_ENV = 'FPGEN_MODEL_URL' + +_LOCK = Lock() +_ready = False + + +def fpgen_data_dir() -> Path: + """Where fpgen reads its model, found without importing fpgen (which fetches).""" + spec = importlib.util.find_spec('fpgen') + if spec is None or spec.origin is None: + raise FpgenModelError('fpgen is not installed.') + return Path(spec.origin).parent / 'data' + + +def load_fpgen() -> ModuleType: + """fpgen, imported only once the pinned model is in place.""" + global _ready + if not _ready: + ensure_fpgen_model() + _ready = True + import fpgen + import fpgen.utils # noqa: F401 -- callers use fpgen.utils._lookup_possibilities + + return fpgen + + +def is_pinned(data_dir: Optional[Path] = None) -> bool: + """Whether the pinned model's files are in place, whatever wrote them.""" + data_dir = data_dir or fpgen_data_dir() + return all( + (data_dir / name).exists() and _hash(data_dir / name) == digest + for name, digest in PIN['file_sha256'].items() + ) + + +def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> None: + """Install the pinned model unless it is already there, and keep fpgen from + replacing it. Hashing the installed files takes ~2 ms, and catches a model + fpgen fetched over an earlier install.""" + if os.getenv(CUSTOM_MODEL_ENV): + return + data_dir = data_dir or fpgen_data_dir() + with _LOCK: + if any((data_dir / name).exists() for name in DECOMPRESSED_FILES): + raise FpgenModelError( + f"fpgen's model in {data_dir} is decompressed, so it cannot be checked against " + f"the pinned {PIN['tag']}. Remove it with `python -m fpgen remove`; Camoufox " + 'then installs the pinned model.' + ) + try: + if force or not is_pinned(data_dir): + _install(data_dir) + _stamp(data_dir) + except PermissionError as e: + raise FpgenModelError( + f"fpgen's model directory is not writable by this user: {data_dir}\n" + 'Install the model as its owner once, e.g. while building an image: ' + 'camoufox fetch' + ) from e + + +def _install(data_dir: Path) -> None: + print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr) + buffer = webdl(PIN['url'], progress_callback=lambda done, total: None) + verify_sha256(buffer, PIN['sha256'], 'fpgen model') + + data_dir.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix='.model-', dir=data_dir)) + try: + with ZipFile(buffer) as zf: + if sorted(zf.namelist()) != sorted(PIN['file_sha256']): + raise CorruptedDownload( + f"fpgen model archive holds {sorted(zf.namelist())}, " + f"expected {sorted(PIN['file_sha256'])}." + ) + # Members are read by name, never extracted by path. + for name, expected in PIN['file_sha256'].items(): + data = zf.read(name) + if hashlib.sha256(data).hexdigest() != expected: + raise CorruptedDownload(f'fpgen model member {name} does not match its pinned sha256.') + (staging / name).write_bytes(data) + (data_dir / STAMP).unlink(missing_ok=True) + for name in PIN['file_sha256']: + # Dated before the move, so fpgen never sees it as stale; each + # replace is atomic, so a concurrent reader never sees a torn file. + os.utime(staging / name, (PINNED_MTIME, PINNED_MTIME)) + os.replace(staging / name, data_dir / name) + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def _stamp(data_dir: Path) -> None: + """Date the files past fpgen's refresh and write the stamp. Files seeded by + another user (a root-built image) may be left as they are while fpgen still + considers them current.""" + for name in PIN['file_sha256']: + path = data_dir / name + if int(path.stat().st_mtime) == PINNED_MTIME: + continue + try: + os.utime(path, (PINNED_MTIME, PINNED_MTIME)) + except PermissionError: + if path.stat().st_mtime < time.time() - FPGEN_MAX_AGE_S: + raise + stamp = data_dir / STAMP + try: + if stamp.read_text(encoding='utf-8').strip() == PIN['sha256']: + return + except OSError: + pass + stamp.write_text(PIN['sha256'] + '\n', encoding='utf-8') + + +def _hash(path: Path) -> str: + digest = hashlib.sha256() + with open(path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() diff --git a/pythonlib/camoufox/webgl.py b/pythonlib/camoufox/webgl.py index d31c6bc0..08fde48d 100644 --- a/pythonlib/camoufox/webgl.py +++ b/pythonlib/camoufox/webgl.py @@ -18,6 +18,7 @@ import orjson from .coherence import gpu_fits_os from .fingerprints import _FPGEN_OS, gpu_screen_is_plausible, is_software_renderer +from .fpgen_model import load_fpgen # Extensions a release Firefox never exposes (draft extensions behind # webgl.enable-draft-extensions, or mobile-only): fpgen's corpus carries some @@ -60,9 +61,7 @@ def _filtered_extensions(target_os: str) -> FrozenSet[str]: @lru_cache(maxsize=None) def _lookup_index(node: str) -> Dict[str, str]: """fpgen's lookup index for every value of `node`, keyed by its JSON.""" - from fpgen.utils import _lookup_possibilities - - return _lookup_possibilities(node, casefold=False) + return load_fpgen().utils._lookup_possibilities(node, casefold=False) def _pin(node: str, value: Any) -> Tuple[str, str]: @@ -79,10 +78,8 @@ def _pin(node: str, value: Any) -> Tuple[str, str]: @lru_cache(maxsize=None) def _trace(target: str, target_os: str, pinned: Tuple[Tuple[str, str], ...] = ()) -> Tuple[Any, ...]: """fpgen's distribution of `target` for Firefox on `target_os`, in its order.""" - import fpgen - return tuple( - fpgen.trace( + load_fpgen().trace( target=target, browser='Firefox', os=_FPGEN_OS[target_os], diff --git a/pythonlib/tests/test_fpgen_model.py b/pythonlib/tests/test_fpgen_model.py new file mode 100644 index 00000000..04614caf --- /dev/null +++ b/pythonlib/tests/test_fpgen_model.py @@ -0,0 +1,336 @@ +"""fpgen's model is the pinned release, installed before fpgen is first imported. + +Left to itself, `import fpgen` downloads the first release the GitHub API +lists -- model-4/2025, whose WebGL records carry no vendor or renderer -- so a +pip-installed camoufox failed every generated launch with KeyError: 'vendor' in +webgl.py. scripts/pin-fpgen-model.py pinned the model for CI only. +""" + +import ast +import hashlib +import io +import json +import os +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from camoufox import fpgen_model # noqa: E402 +from camoufox.exceptions import CorruptedDownload, FpgenModelError # noqa: E402 +from camoufox.fpgen_model import FPGEN_MAX_AGE_S, PINNED_MTIME, STAMP, ensure_fpgen_model # noqa: E402 + +PACKAGE = Path(fpgen_model.__file__).resolve().parent +MEMBERS = { + "fingerprint-network.json.zst": b"network" * 1000, + "values.json.zst": b"values-json" * 1000, + "values.dat.zst": b"values-dat" * 1000, +} +STALE = time.time() - FPGEN_MAX_AGE_S - 86400 + + +def _archive(members): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, data in members.items(): + zf.writestr(name, data) + return buf.getvalue() + + +def _sha256(data): + return hashlib.sha256(data).hexdigest() + + +@pytest.fixture +def pinned(monkeypatch): + """Pin a small synthetic model instead of the real release, and record the + downloads. Set `.payload` to serve other bytes than the pin describes.""" + archive = _archive(MEMBERS) + pin = { + **fpgen_model.PIN, + "size": len(archive), + "sha256": _sha256(archive), + "files": sorted(MEMBERS), + "file_sha256": {name: _sha256(data) for name, data in MEMBERS.items()}, + } + monkeypatch.setattr(fpgen_model, "PIN", pin) + monkeypatch.delenv("FPGEN_MODEL_URL", raising=False) + + class Downloads(list): + payload = archive + + downloads = Downloads() + + def fake_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None): + downloads.append(url) + return io.BytesIO(downloads.payload) + + monkeypatch.setattr(fpgen_model, "webdl", fake_webdl) + return downloads + + +def _write(data_dir, members, mtime=None, stamp=None): + data_dir.mkdir(parents=True, exist_ok=True) + for name, data in members.items(): + path = data_dir / name + path.write_bytes(data) + if mtime is not None: + os.utime(path, (mtime, mtime)) + if stamp is not None: + (data_dir / STAMP).write_text(stamp + "\n") + + +def _mtimes(data_dir): + return {int((data_dir / name).stat().st_mtime) for name in MEMBERS} + + +def test_only_the_model_module_imports_fpgen(): + """Every fpgen import goes through fpgen_model.load_fpgen(), which installs + the pinned model first. A direct import anywhere else lets fpgen fetch + model-4/2025 before the pin is in place.""" + offenders = [] + for path in sorted(PACKAGE.rglob("*.py")): + if path.name == "fpgen_model.py": + continue + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ""] + else: + continue + if any(name == "fpgen" or name.startswith("fpgen.") for name in names): + offenders.append(f"{path.relative_to(PACKAGE)}:{node.lineno}") + assert offenders == [] + + +def test_missing_model_is_installed_verified_and_stamped(tmp_path, pinned): + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + + assert pinned == [fpgen_model.PIN["url"]] + for name, data in MEMBERS.items(): + assert (data_dir / name).read_bytes() == data + assert _mtimes(data_dir) == {PINNED_MTIME} + # The same stamp scripts/pin-fpgen-model.py and the TypeScript launcher write. + assert (data_dir / STAMP).read_text().strip() == fpgen_model.PIN["sha256"] + assert sorted(p.name for p in data_dir.iterdir()) == sorted([*MEMBERS, STAMP]) + + +def test_pinned_model_is_not_downloaded_again(tmp_path, pinned): + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + ensure_fpgen_model(data_dir) + + assert len(pinned) == 1 + + +def test_stale_pinned_model_is_restamped_not_refetched(tmp_path, pinned): + """Five weeks after install fpgen would fetch model-4/2025 over the pin.""" + data_dir = tmp_path / "data" + _write(data_dir, MEMBERS, mtime=STALE, stamp=fpgen_model.PIN["sha256"]) + + ensure_fpgen_model(data_dir) + + assert pinned == [] + assert _mtimes(data_dir) == {PINNED_MTIME} + + +def test_a_model_fpgen_overwrote_under_a_valid_stamp_is_replaced(tmp_path, pinned): + """The stamp outlives fpgen's own five-week refetch, so files are hashed.""" + data_dir = tmp_path / "data" + _write(data_dir, {**MEMBERS, "values.dat.zst": b"model-4/2025"}, stamp=fpgen_model.PIN["sha256"]) + + ensure_fpgen_model(data_dir) + + assert len(pinned) == 1 + assert (data_dir / "values.dat.zst").read_bytes() == MEMBERS["values.dat.zst"] + + +def test_matching_files_without_a_stamp_are_adopted(tmp_path, pinned): + data_dir = tmp_path / "data" + _write(data_dir, MEMBERS) + + ensure_fpgen_model(data_dir) + + assert pinned == [] + assert (data_dir / STAMP).read_text().strip() == fpgen_model.PIN["sha256"] + + +def test_partial_model_is_completed(tmp_path, pinned): + data_dir = tmp_path / "data" + _write(data_dir, {"values.json.zst": MEMBERS["values.json.zst"]}) + + ensure_fpgen_model(data_dir) + + assert len(pinned) == 1 + assert all((data_dir / name).exists() for name in MEMBERS) + + +def test_force_reinstalls(tmp_path, pinned): + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + ensure_fpgen_model(data_dir, force=True) + + assert len(pinned) == 2 + + +@pytest.mark.parametrize( + "tamper", + [ + pytest.param(lambda a: a[:-1], id="truncated-archive"), + pytest.param(lambda a: _archive({**MEMBERS, "values.dat.zst": b"x"}), id="member-changed"), + ], +) +def test_tampered_download_installs_nothing(tmp_path, pinned, tamper): + data_dir = tmp_path / "data" + pinned.payload = tamper(_archive(MEMBERS)) + + with pytest.raises(CorruptedDownload): + ensure_fpgen_model(data_dir) + + assert not data_dir.exists() or list(data_dir.iterdir()) == [] + + +def test_member_hashes_are_checked_even_when_the_archive_hash_matches(tmp_path, pinned): + bad = _archive({**MEMBERS, "values.dat.zst": b"x"}) + pinned.payload = bad + fpgen_model.PIN.update(size=len(bad), sha256=_sha256(bad)) + + with pytest.raises(CorruptedDownload, match="values.dat.zst"): + ensure_fpgen_model(tmp_path / "data") + + +def test_decompressed_model_fails_loudly(tmp_path, pinned): + """`python -m fpgen decompress` leaves files the pin cannot verify, and fpgen + reads them in preference to the verified ones.""" + data_dir = tmp_path / "data" + _write(data_dir, {name: b"{}" for name in fpgen_model.DECOMPRESSED_FILES}) + + with pytest.raises(FpgenModelError, match="fpgen remove"): + ensure_fpgen_model(data_dir) + assert pinned == [] + + +def test_custom_model_url_is_left_to_fpgen(tmp_path, pinned, monkeypatch): + monkeypatch.setenv("FPGEN_MODEL_URL", "https://example.invalid/model.zip") + data_dir = tmp_path / "data" + + ensure_fpgen_model(data_dir) + + assert pinned == [] + assert not data_dir.exists() + + +needs_non_root = pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores file permissions" +) + + +@pytest.fixture +def read_only(tmp_path): + data_dir = tmp_path / "data" + yield data_dir + if data_dir.exists(): + data_dir.chmod(0o755) + + +@needs_non_root +def test_read_only_missing_model_names_the_fix(read_only, pinned): + read_only.mkdir() + read_only.chmod(0o555) + + with pytest.raises(FpgenModelError, match="camoufox fetch"): + ensure_fpgen_model(read_only) + + +@needs_non_root +def test_read_only_current_pinned_model_is_used(read_only, pinned): + """A root-seeded image run as another user: fpgen accepts the files as they + are for five weeks, so the launch does not need to stamp them.""" + _write(read_only, MEMBERS, stamp=fpgen_model.PIN["sha256"]) + read_only.chmod(0o555) + for name in MEMBERS: + (read_only / name).chmod(0o444) + + ensure_fpgen_model(read_only) + + assert pinned == [] + + +def test_install_writes_nothing_to_stdout(tmp_path, pinned, capfd): + """A caller's stdout may be data (a probe printing JSON).""" + ensure_fpgen_model(tmp_path / "data") + + out, err = capfd.readouterr() + assert out == "" + assert "fpgen model" in err + + +def test_the_packaged_pin_is_the_repo_pin(): + """The package cannot ship scripts/, so it carries a copy. Bump both.""" + repo_pin = PACKAGE.parents[1] / "scripts" / "data" / "fpgen-model.json" + assert fpgen_model.PIN == json.loads(repo_pin.read_text(encoding="utf-8")) + + +def test_the_real_pin_is_complete(): + pin = fpgen_model.PIN + assert pin["url"].startswith("https://github.com/") + assert f"/download/{pin['tag']}/" in pin["url"] + assert sorted(pin["file_sha256"]) == sorted(pin["files"]) + + +def test_constants_match_the_installed_fpgen(): + """fpgen's file names, data directory and five-week window are its own; if + a release changes them, this module would install a model fpgen cannot see. + FPGEN_NO_INIT keeps fpgen from loading (or fetching) while it is read.""" + probe = ( + "import json, fpgen.pkgman as p;" + "print(json.dumps({" + "'dir': str(p.DATA_DIR)," + "'compressed': sorted(v.name for v in p.FILE_PAIRS.values())," + "'decompressed': sorted(k.name for k in p.FILE_PAIRS.keys())}))" + ) + env = {**os.environ, "FPGEN_NO_INIT": "1"} + out = subprocess.run([sys.executable, "-c", probe], env=env, capture_output=True, text=True, check=True) + seen = json.loads(out.stdout.strip().splitlines()[-1]) + + assert seen["dir"] == str(fpgen_model.fpgen_data_dir()) + assert seen["compressed"] == sorted(fpgen_model.PIN["files"]) + assert seen["decompressed"] == sorted(fpgen_model.DECOMPRESSED_FILES) + assert "timedelta(weeks=5)" in (fpgen_model.fpgen_data_dir().parent / "pkgman.py").read_text() + + +_IMPORT_ORDER = """ +import sys +from camoufox import fpgen_model +real = fpgen_model.ensure_fpgen_model +seen = [] +def spy(*args, **kwargs): + seen.append("fpgen" in sys.modules) + return real(*args, **kwargs) +fpgen_model.ensure_fpgen_model = spy +from camoufox import {module} +{call} +print(seen) +""" + + +@pytest.mark.parametrize( + "module, call", + [ + pytest.param("fingerprints", "fingerprints._generator()", id="generate"), + pytest.param("webgl", "webgl.firefox_gpus('win')", id="preset-or-webgl_config"), + ], +) +def test_each_entry_point_installs_the_model_before_fpgen_loads(module, call): + script = _IMPORT_ORDER.format(module=module, call=call) + out = subprocess.run( + [sys.executable, "-c", script], cwd=PACKAGE.parent, capture_output=True, text=True, check=True + ) + assert out.stdout.strip().splitlines()[-1] == "[False]" diff --git a/scripts/data/fpgen-model.json b/scripts/data/fpgen-model.json index 93d041cf..7016ee11 100644 --- a/scripts/data/fpgen-model.json +++ b/scripts/data/fpgen-model.json @@ -10,5 +10,10 @@ "values.dat.zst", "values.json.zst" ], - "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see scripts/pin-fpgen-model.py. Bump by editing this file; the sha256 is the gate." + "file_sha256": { + "fingerprint-network.json.zst": "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4", + "values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", + "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f" + }, + "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate." } diff --git a/scripts/pin-fpgen-model.py b/scripts/pin-fpgen-model.py index ec3a1847..67b00c98 100755 --- a/scripts/pin-fpgen-model.py +++ b/scripts/pin-fpgen-model.py @@ -1,38 +1,10 @@ #!/usr/bin/env python3 -"""Install a pinned fpgen model, instead of letting fpgen fetch one itself. +"""Install the pinned fpgen model before anything imports fpgen. -WHY THIS EXISTS ---------------- -`fpgen` (scrapfly/fingerprint-generator) is where Camoufox's synthetic -fingerprints come from. It does not ship its model; it downloads one the first -time it is imported, and again whenever the files on disk are over five weeks -old. That fetch has four problems, all of them in fpgen/pkgman.py: - - 1. TLS verification is DISABLED on both the API call and the download - (`httpx.get(..., verify=False)`, `httpx.stream(..., verify=False)`), so - anyone on the network path can serve the model. - 2. The archive is never checksummed -- it goes straight into - `zipfile.ZipFile(...).extractall(DATA_DIR)`, which is also not guarded - against path traversal. - 3. The GitHub API is called unauthenticated, sharing a 60-request/hour limit - with every other job on the runner's IP. - 4. It takes the FIRST release the API lists and that release's first `.zip`. - GitHub sorts releases by `created_at`, and `model-4/2025` and - `model-2/2026` carry an identical `created_at` (2025-03-22T11:41:12Z, - inherited from the tag's commit), so the tie breaks toward the lower id - and fpgen always lands on the 2025 model. The 2026 model is unreachable - through that path. - -So an unpinned Camoufox generates from an April-2025 corpus -- newest Firefox -137, newest GPU an RTX 40 -- and cannot be talked into anything newer. - -This script puts the model named by scripts/data/fpgen-model.json where fpgen -looks, with TLS verification on and the sha256 checked, before anything imports -fpgen. fpgen then finds recent files and never calls the network. - -Locating the data directory must NOT import fpgen: importing is what triggers -the download this script exists to prevent. importlib.util.find_spec() reads the -module's origin without executing it. +The launcher does this itself on first use (pythonlib/camoufox/fpgen_model.py, +which explains why fpgen's own download is not used). This entry point exists +for jobs that need the model in place before they start: CI, image builds, and +the TypeScript golden recorder. Usage: python3 scripts/pin-fpgen-model.py # install if not already pinned @@ -41,84 +13,12 @@ Usage: """ import argparse -import hashlib -import importlib.util -import json -import os -import ssl import sys -import tempfile -import urllib.request -import zipfile -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SPEC = os.path.join(REPO, 'scripts', 'data', 'fpgen-model.json') -STAMP = '.pinned-model' - - -def load_spec(): - with open(SPEC, encoding='utf-8') as fh: - return json.load(fh) - - -def data_dir(): - """fpgen's data directory, found WITHOUT importing fpgen.""" - spec = importlib.util.find_spec('fpgen') - if spec is None or not spec.origin: - sys.exit('fpgen is not installed; `pip install -e pythonlib` first') - return os.path.join(os.path.dirname(spec.origin), 'data') - - -def digest(path, chunk=1 << 20): - h = hashlib.sha256() - with open(path, 'rb') as fh: - for block in iter(lambda: fh.read(chunk), b''): - h.update(block) - return h.hexdigest() - - -def is_pinned(spec, d): - """True when the pinned model is already in place.""" - try: - with open(os.path.join(d, STAMP), encoding='utf-8') as fh: - if fh.read().strip() != spec['sha256']: - return False - except OSError: - return False - return all(os.path.exists(os.path.join(d, f)) for f in spec['files']) - - -def download(spec, dest): - # Verification ON, unlike fpgen's own fetch. create_default_context() - # verifies the chain and the hostname. - ctx = ssl.create_default_context() - print(f'fetching {spec["url"]}', file=sys.stderr) - req = urllib.request.Request(spec['url'], headers={'User-Agent': 'camoufox-build'}) - with urllib.request.urlopen(req, context=ctx, timeout=60) as r, open(dest, 'wb') as fh: - while chunk := r.read(1 << 20): - fh.write(chunk) - size = os.path.getsize(dest) - if spec.get('size') and size != spec['size']: - sys.exit(f'size mismatch: got {size}, expected {spec["size"]}') - got = digest(dest) - if got != spec['sha256']: - sys.exit(f'sha256 mismatch:\n got {got}\n expected {spec["sha256"]}') - - -def install(spec, archive, d): - os.makedirs(d, exist_ok=True) - with zipfile.ZipFile(archive) as z: - # fpgen extracts without checking; do not copy that. A member escaping - # the data directory would write anywhere the build user can. - for name in z.namelist(): - target = os.path.realpath(os.path.join(d, name)) - if not target.startswith(os.path.realpath(d) + os.sep): - sys.exit(f'refusing to extract outside the data dir: {name}') - z.extractall(d) - # Written last: an interrupted extract leaves no stamp, so the next run - # reinstalls rather than trusting a partial model. - with open(os.path.join(d, STAMP), 'w', encoding='utf-8') as fh: - fh.write(spec['sha256'] + '\n') +try: + from camoufox.fpgen_model import PIN, ensure_fpgen_model, fpgen_data_dir, is_pinned +except ImportError: + sys.exit('camoufox is not installed; `pip install -e pythonlib` first') def main(): @@ -128,26 +28,16 @@ def main(): ap.add_argument('--force', action='store_true', help='reinstall even if already pinned') args = ap.parse_args() - spec = load_spec() - d = data_dir() - if args.check: - if is_pinned(spec, d): - print(f'OK: fpgen model pinned to {spec["tag"]}') + if is_pinned(): + print(f'OK: fpgen model pinned to {PIN["tag"]}') return 0 - print(f'fpgen model is NOT pinned to {spec["tag"]}; run scripts/pin-fpgen-model.py', + print(f'fpgen model is NOT pinned to {PIN["tag"]}; run scripts/pin-fpgen-model.py', file=sys.stderr) return 1 - if is_pinned(spec, d) and not args.force: - print(f'OK: fpgen model already pinned to {spec["tag"]}', file=sys.stderr) - return 0 - - with tempfile.TemporaryDirectory() as tmp: - archive = os.path.join(tmp, 'model-release.zip') - download(spec, archive) - install(spec, archive, d) - print(f'OK: pinned fpgen model {spec["tag"]} -> {d}', file=sys.stderr) + ensure_fpgen_model(force=args.force) + print(f'OK: fpgen model {PIN["tag"]} pinned in {fpgen_data_dir()}', file=sys.stderr) return 0 diff --git a/typescript/src/fpgen/pin.ts b/typescript/src/fpgen/pin.ts index db31d4b4..8e2ff973 100644 --- a/typescript/src/fpgen/pin.ts +++ b/typescript/src/fpgen/pin.ts @@ -6,7 +6,7 @@ * tests/fpgen-model.test.ts fails if the two ever disagree, so bumping the * model means editing both (the sha256 is the gate in each). * - * See scripts/pin-fpgen-model.py for why fpgen's own downloader is not used: + * See pythonlib/camoufox/fpgen_model.py for why fpgen's own downloader is not used: * it disables TLS verification, never checks a digest, and its "first listed * release" rule cannot reach this tag. */ @@ -19,6 +19,8 @@ export interface ModelPin { readonly url: string; readonly repo: string; readonly files: readonly string[]; + /** Each file's sha256, for checking an installed model without its archive. */ + readonly file_sha256: Readonly>; } export const MODEL_PIN: ModelPin = { @@ -29,4 +31,12 @@ export const MODEL_PIN: ModelPin = { url: "https://github.com/scrapfly/fingerprint-generator/releases/download/model-2/2026/model-release.zip", repo: "scrapfly/fingerprint-generator", files: ["fingerprint-network.json.zst", "values.dat.zst", "values.json.zst"], + file_sha256: { + "fingerprint-network.json.zst": + "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4", + "values.dat.zst": + "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", + "values.json.zst": + "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f", + }, };