diff --git a/pythonlib/README.md b/pythonlib/README.md index 0659c4f0..8871e2cd 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -37,6 +37,8 @@ Next, download the Camoufox browser: camoufox fetch ``` +`fetch` also installs fpgen's model, pinned by sha256, into fpgen's package directory. Otherwise the first generated fingerprint installs it. Run `fetch` as that directory's owner if the browser will run as another user, e.g. while building a Docker image. + To uninstall, run `camoufox remove`. --- diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index de1dd11c..81e44ff7 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -11,6 +11,7 @@ from typing import Any, List, Optional, Tuple import rich_click as click from .addons import DefaultAddons, maybe_download_addons +from .fpgen_model import ensure_fpgen_model from .geolocation import ( ALLOW_GEOIP, GEOIP_DIR, @@ -323,6 +324,7 @@ def fetch(version): if ALLOW_GEOIP: download_mmdb() maybe_download_addons(list(DefaultAddons)) + ensure_fpgen_model() def _set_channel(repo_name: str, channel_type: str): diff --git a/pythonlib/camoufox/exceptions.py b/pythonlib/camoufox/exceptions.py index 292e992e..9be0e780 100644 --- a/pythonlib/camoufox/exceptions.py +++ b/pythonlib/camoufox/exceptions.py @@ -184,3 +184,9 @@ class ProfileDirectoryError(RuntimeError): """Raised when Camoufox's required runtime directory cannot be prepared.""" ... + + +class FpgenModelError(RuntimeError): + """Raised when fpgen's pinned model cannot be installed where fpgen reads it.""" + + ... diff --git a/pythonlib/camoufox/fingerprints.py b/pythonlib/camoufox/fingerprints.py index b2b4f891..ae6fb92a 100644 --- a/pythonlib/camoufox/fingerprints.py +++ b/pythonlib/camoufox/fingerprints.py @@ -29,9 +29,9 @@ _FP_GENERATOR = None def _generator(): global _FP_GENERATOR if _FP_GENERATOR is None: - from fpgen import Generator + from .fpgen_model import load_fpgen - _FP_GENERATOR = Generator() + _FP_GENERATOR = load_fpgen().Generator() return _FP_GENERATOR diff --git a/pythonlib/camoufox/fpgen-model.json b/pythonlib/camoufox/fpgen-model.json new file mode 100644 index 00000000..7016ee11 --- /dev/null +++ b/pythonlib/camoufox/fpgen-model.json @@ -0,0 +1,19 @@ +{ + "tag": "model-2/2026", + "asset": "model-release.zip", + "size": 1564571, + "sha256": "6530b8322cdaa4ec042921c8d9a0369a0e6e0269ba636c01a7203e4a2f109936", + "url": "https://github.com/scrapfly/fingerprint-generator/releases/download/model-2/2026/model-release.zip", + "repo": "scrapfly/fingerprint-generator", + "files": [ + "fingerprint-network.json.zst", + "values.dat.zst", + "values.json.zst" + ], + "file_sha256": { + "fingerprint-network.json.zst": "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4", + "values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", + "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f" + }, + "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate." +} diff --git a/pythonlib/camoufox/fpgen_model.py b/pythonlib/camoufox/fpgen_model.py new file mode 100644 index 00000000..28d39b88 --- /dev/null +++ b/pythonlib/camoufox/fpgen_model.py @@ -0,0 +1,168 @@ +"""fpgen's model, installed from the pinned release before fpgen is imported. + +Left to itself, `import fpgen` downloads its model from the first release the +GitHub API lists, with TLS verification off and no checksum, and downloads it +again whenever the files are five weeks old (fpgen/pkgman.py). The first +release listed is model-4/2025, whose WebGL records have no vendor or renderer, +so every generated launch failed with KeyError: 'vendor'. + +So fpgen is only ever imported through load_fpgen(), which first installs the +release named by fpgen-model.json (a copy of scripts/data/fpgen-model.json), +checks the archive and each file against its sha256, and dates the files in the +future so fpgen's five-week refresh never replaces them. The data directory +layout, including the `.pinned-model` stamp, is the one +scripts/pin-fpgen-model.py and the TypeScript launcher write. +""" + +import hashlib +import importlib.util +import json +import os +import shutil +import sys +import tempfile +import time +from pathlib import Path +from threading import Lock +from types import ModuleType +from typing import Optional +from zipfile import ZipFile + +from .exceptions import CorruptedDownload, FpgenModelError +from .pkgman import verify_sha256, webdl + +PIN = json.loads((Path(__file__).parent / 'fpgen-model.json').read_text(encoding='utf-8')) + +# Holds the archive's sha256, written last: a stamp means the install finished. +STAMP = '.pinned-model' + +# `python -m fpgen decompress` replaces the archive's files with these, which +# fpgen then reads in preference to them. +DECOMPRESSED_FILES = ('fingerprint-network.json', 'values.json', 'values.dat') + +# 2100-01-01T00:00:00Z. fpgen refetches files whose mtime is older than this +# window (pkgman.files_are_recent). +PINNED_MTIME = 4102444800 +FPGEN_MAX_AGE_S = 5 * 7 * 24 * 3600 + +# fpgen fetches a model from this URL instead of GitHub. A caller who set it +# chose their model, so it is left alone. +CUSTOM_MODEL_ENV = 'FPGEN_MODEL_URL' + +_LOCK = Lock() +_ready = False + + +def fpgen_data_dir() -> Path: + """Where fpgen reads its model, found without importing fpgen (which fetches).""" + spec = importlib.util.find_spec('fpgen') + if spec is None or spec.origin is None: + raise FpgenModelError('fpgen is not installed.') + return Path(spec.origin).parent / 'data' + + +def load_fpgen() -> ModuleType: + """fpgen, imported only once the pinned model is in place.""" + global _ready + if not _ready: + ensure_fpgen_model() + _ready = True + import fpgen + import fpgen.utils # noqa: F401 -- callers use fpgen.utils._lookup_possibilities + + return fpgen + + +def is_pinned(data_dir: Optional[Path] = None) -> bool: + """Whether the pinned model's files are in place, whatever wrote them.""" + data_dir = data_dir or fpgen_data_dir() + return all( + (data_dir / name).exists() and _hash(data_dir / name) == digest + for name, digest in PIN['file_sha256'].items() + ) + + +def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> None: + """Install the pinned model unless it is already there, and keep fpgen from + replacing it. Hashing the installed files takes ~2 ms, and catches a model + fpgen fetched over an earlier install.""" + if os.getenv(CUSTOM_MODEL_ENV): + return + data_dir = data_dir or fpgen_data_dir() + with _LOCK: + if any((data_dir / name).exists() for name in DECOMPRESSED_FILES): + raise FpgenModelError( + f"fpgen's model in {data_dir} is decompressed, so it cannot be checked against " + f"the pinned {PIN['tag']}. Remove it with `python -m fpgen remove`; Camoufox " + 'then installs the pinned model.' + ) + try: + if force or not is_pinned(data_dir): + _install(data_dir) + _stamp(data_dir) + except PermissionError as e: + raise FpgenModelError( + f"fpgen's model directory is not writable by this user: {data_dir}\n" + 'Install the model as its owner once, e.g. while building an image: ' + 'camoufox fetch' + ) from e + + +def _install(data_dir: Path) -> None: + print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr) + buffer = webdl(PIN['url'], progress_callback=lambda done, total: None) + verify_sha256(buffer, PIN['sha256'], 'fpgen model') + + data_dir.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix='.model-', dir=data_dir)) + try: + with ZipFile(buffer) as zf: + if sorted(zf.namelist()) != sorted(PIN['file_sha256']): + raise CorruptedDownload( + f"fpgen model archive holds {sorted(zf.namelist())}, " + f"expected {sorted(PIN['file_sha256'])}." + ) + # Members are read by name, never extracted by path. + for name, expected in PIN['file_sha256'].items(): + data = zf.read(name) + if hashlib.sha256(data).hexdigest() != expected: + raise CorruptedDownload(f'fpgen model member {name} does not match its pinned sha256.') + (staging / name).write_bytes(data) + (data_dir / STAMP).unlink(missing_ok=True) + for name in PIN['file_sha256']: + # Dated before the move, so fpgen never sees it as stale; each + # replace is atomic, so a concurrent reader never sees a torn file. + os.utime(staging / name, (PINNED_MTIME, PINNED_MTIME)) + os.replace(staging / name, data_dir / name) + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def _stamp(data_dir: Path) -> None: + """Date the files past fpgen's refresh and write the stamp. Files seeded by + another user (a root-built image) may be left as they are while fpgen still + considers them current.""" + for name in PIN['file_sha256']: + path = data_dir / name + if int(path.stat().st_mtime) == PINNED_MTIME: + continue + try: + os.utime(path, (PINNED_MTIME, PINNED_MTIME)) + except PermissionError: + if path.stat().st_mtime < time.time() - FPGEN_MAX_AGE_S: + raise + stamp = data_dir / STAMP + try: + if stamp.read_text(encoding='utf-8').strip() == PIN['sha256']: + return + except OSError: + pass + stamp.write_text(PIN['sha256'] + '\n', encoding='utf-8') + + +def _hash(path: Path) -> str: + digest = hashlib.sha256() + with open(path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() diff --git a/pythonlib/camoufox/webgl.py b/pythonlib/camoufox/webgl.py index d31c6bc0..08fde48d 100644 --- a/pythonlib/camoufox/webgl.py +++ b/pythonlib/camoufox/webgl.py @@ -18,6 +18,7 @@ import orjson from .coherence import gpu_fits_os from .fingerprints import _FPGEN_OS, gpu_screen_is_plausible, is_software_renderer +from .fpgen_model import load_fpgen # Extensions a release Firefox never exposes (draft extensions behind # webgl.enable-draft-extensions, or mobile-only): fpgen's corpus carries some @@ -60,9 +61,7 @@ def _filtered_extensions(target_os: str) -> FrozenSet[str]: @lru_cache(maxsize=None) def _lookup_index(node: str) -> Dict[str, str]: """fpgen's lookup index for every value of `node`, keyed by its JSON.""" - from fpgen.utils import _lookup_possibilities - - return _lookup_possibilities(node, casefold=False) + return load_fpgen().utils._lookup_possibilities(node, casefold=False) def _pin(node: str, value: Any) -> Tuple[str, str]: @@ -79,10 +78,8 @@ def _pin(node: str, value: Any) -> Tuple[str, str]: @lru_cache(maxsize=None) def _trace(target: str, target_os: str, pinned: Tuple[Tuple[str, str], ...] = ()) -> Tuple[Any, ...]: """fpgen's distribution of `target` for Firefox on `target_os`, in its order.""" - import fpgen - return tuple( - fpgen.trace( + load_fpgen().trace( target=target, browser='Firefox', os=_FPGEN_OS[target_os], diff --git a/pythonlib/tests/test_fpgen_model.py b/pythonlib/tests/test_fpgen_model.py new file mode 100644 index 00000000..04614caf --- /dev/null +++ b/pythonlib/tests/test_fpgen_model.py @@ -0,0 +1,336 @@ +"""fpgen's model is the pinned release, installed before fpgen is first imported. + +Left to itself, `import fpgen` downloads the first release the GitHub API +lists -- model-4/2025, whose WebGL records carry no vendor or renderer -- so a +pip-installed camoufox failed every generated launch with KeyError: 'vendor' in +webgl.py. scripts/pin-fpgen-model.py pinned the model for CI only. +""" + +import ast +import hashlib +import io +import json +import os +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from camoufox import fpgen_model # noqa: E402 +from camoufox.exceptions import CorruptedDownload, FpgenModelError # noqa: E402 +from camoufox.fpgen_model import FPGEN_MAX_AGE_S, PINNED_MTIME, STAMP, ensure_fpgen_model # noqa: E402 + +PACKAGE = Path(fpgen_model.__file__).resolve().parent +MEMBERS = { + "fingerprint-network.json.zst": b"network" * 1000, + "values.json.zst": b"values-json" * 1000, + "values.dat.zst": b"values-dat" * 1000, +} +STALE = time.time() - FPGEN_MAX_AGE_S - 86400 + + +def _archive(members): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, data in members.items(): + zf.writestr(name, data) + return buf.getvalue() + + +def _sha256(data): + return hashlib.sha256(data).hexdigest() + + +@pytest.fixture +def pinned(monkeypatch): + """Pin a small synthetic model instead of the real release, and record the + downloads. Set `.payload` to serve other bytes than the pin describes.""" + archive = _archive(MEMBERS) + pin = { + **fpgen_model.PIN, + "size": len(archive), + "sha256": _sha256(archive), + "files": sorted(MEMBERS), + "file_sha256": {name: _sha256(data) for name, data in MEMBERS.items()}, + } + monkeypatch.setattr(fpgen_model, "PIN", pin) + monkeypatch.delenv("FPGEN_MODEL_URL", raising=False) + + class Downloads(list): + payload = archive + + downloads = Downloads() + + def fake_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None): + downloads.append(url) + return io.BytesIO(downloads.payload) + + monkeypatch.setattr(fpgen_model, "webdl", fake_webdl) + return downloads + + +def _write(data_dir, members, mtime=None, stamp=None): + data_dir.mkdir(parents=True, exist_ok=True) + for name, data in members.items(): + path = data_dir / name + path.write_bytes(data) + if mtime is not None: + os.utime(path, (mtime, mtime)) + if stamp is not None: + (data_dir / STAMP).write_text(stamp + "\n") + + +def _mtimes(data_dir): + return {int((data_dir / name).stat().st_mtime) for name in MEMBERS} + + +def test_only_the_model_module_imports_fpgen(): + """Every fpgen import goes through fpgen_model.load_fpgen(), which installs + the pinned model first. A direct import anywhere else lets fpgen fetch + model-4/2025 before the pin is in place.""" + offenders = [] + for path in sorted(PACKAGE.rglob("*.py")): + if path.name == "fpgen_model.py": + continue + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ""] + else: + continue + if any(name == "fpgen" or name.startswith("fpgen.") for name in names): + offenders.append(f"{path.relative_to(PACKAGE)}:{node.lineno}") + assert offenders == [] + + +def test_missing_model_is_installed_verified_and_stamped(tmp_path, pinned): + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + + assert pinned == [fpgen_model.PIN["url"]] + for name, data in MEMBERS.items(): + assert (data_dir / name).read_bytes() == data + assert _mtimes(data_dir) == {PINNED_MTIME} + # The same stamp scripts/pin-fpgen-model.py and the TypeScript launcher write. + assert (data_dir / STAMP).read_text().strip() == fpgen_model.PIN["sha256"] + assert sorted(p.name for p in data_dir.iterdir()) == sorted([*MEMBERS, STAMP]) + + +def test_pinned_model_is_not_downloaded_again(tmp_path, pinned): + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + ensure_fpgen_model(data_dir) + + assert len(pinned) == 1 + + +def test_stale_pinned_model_is_restamped_not_refetched(tmp_path, pinned): + """Five weeks after install fpgen would fetch model-4/2025 over the pin.""" + data_dir = tmp_path / "data" + _write(data_dir, MEMBERS, mtime=STALE, stamp=fpgen_model.PIN["sha256"]) + + ensure_fpgen_model(data_dir) + + assert pinned == [] + assert _mtimes(data_dir) == {PINNED_MTIME} + + +def test_a_model_fpgen_overwrote_under_a_valid_stamp_is_replaced(tmp_path, pinned): + """The stamp outlives fpgen's own five-week refetch, so files are hashed.""" + data_dir = tmp_path / "data" + _write(data_dir, {**MEMBERS, "values.dat.zst": b"model-4/2025"}, stamp=fpgen_model.PIN["sha256"]) + + ensure_fpgen_model(data_dir) + + assert len(pinned) == 1 + assert (data_dir / "values.dat.zst").read_bytes() == MEMBERS["values.dat.zst"] + + +def test_matching_files_without_a_stamp_are_adopted(tmp_path, pinned): + data_dir = tmp_path / "data" + _write(data_dir, MEMBERS) + + ensure_fpgen_model(data_dir) + + assert pinned == [] + assert (data_dir / STAMP).read_text().strip() == fpgen_model.PIN["sha256"] + + +def test_partial_model_is_completed(tmp_path, pinned): + data_dir = tmp_path / "data" + _write(data_dir, {"values.json.zst": MEMBERS["values.json.zst"]}) + + ensure_fpgen_model(data_dir) + + assert len(pinned) == 1 + assert all((data_dir / name).exists() for name in MEMBERS) + + +def test_force_reinstalls(tmp_path, pinned): + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + ensure_fpgen_model(data_dir, force=True) + + assert len(pinned) == 2 + + +@pytest.mark.parametrize( + "tamper", + [ + pytest.param(lambda a: a[:-1], id="truncated-archive"), + pytest.param(lambda a: _archive({**MEMBERS, "values.dat.zst": b"x"}), id="member-changed"), + ], +) +def test_tampered_download_installs_nothing(tmp_path, pinned, tamper): + data_dir = tmp_path / "data" + pinned.payload = tamper(_archive(MEMBERS)) + + with pytest.raises(CorruptedDownload): + ensure_fpgen_model(data_dir) + + assert not data_dir.exists() or list(data_dir.iterdir()) == [] + + +def test_member_hashes_are_checked_even_when_the_archive_hash_matches(tmp_path, pinned): + bad = _archive({**MEMBERS, "values.dat.zst": b"x"}) + pinned.payload = bad + fpgen_model.PIN.update(size=len(bad), sha256=_sha256(bad)) + + with pytest.raises(CorruptedDownload, match="values.dat.zst"): + ensure_fpgen_model(tmp_path / "data") + + +def test_decompressed_model_fails_loudly(tmp_path, pinned): + """`python -m fpgen decompress` leaves files the pin cannot verify, and fpgen + reads them in preference to the verified ones.""" + data_dir = tmp_path / "data" + _write(data_dir, {name: b"{}" for name in fpgen_model.DECOMPRESSED_FILES}) + + with pytest.raises(FpgenModelError, match="fpgen remove"): + ensure_fpgen_model(data_dir) + assert pinned == [] + + +def test_custom_model_url_is_left_to_fpgen(tmp_path, pinned, monkeypatch): + monkeypatch.setenv("FPGEN_MODEL_URL", "https://example.invalid/model.zip") + data_dir = tmp_path / "data" + + ensure_fpgen_model(data_dir) + + assert pinned == [] + assert not data_dir.exists() + + +needs_non_root = pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores file permissions" +) + + +@pytest.fixture +def read_only(tmp_path): + data_dir = tmp_path / "data" + yield data_dir + if data_dir.exists(): + data_dir.chmod(0o755) + + +@needs_non_root +def test_read_only_missing_model_names_the_fix(read_only, pinned): + read_only.mkdir() + read_only.chmod(0o555) + + with pytest.raises(FpgenModelError, match="camoufox fetch"): + ensure_fpgen_model(read_only) + + +@needs_non_root +def test_read_only_current_pinned_model_is_used(read_only, pinned): + """A root-seeded image run as another user: fpgen accepts the files as they + are for five weeks, so the launch does not need to stamp them.""" + _write(read_only, MEMBERS, stamp=fpgen_model.PIN["sha256"]) + read_only.chmod(0o555) + for name in MEMBERS: + (read_only / name).chmod(0o444) + + ensure_fpgen_model(read_only) + + assert pinned == [] + + +def test_install_writes_nothing_to_stdout(tmp_path, pinned, capfd): + """A caller's stdout may be data (a probe printing JSON).""" + ensure_fpgen_model(tmp_path / "data") + + out, err = capfd.readouterr() + assert out == "" + assert "fpgen model" in err + + +def test_the_packaged_pin_is_the_repo_pin(): + """The package cannot ship scripts/, so it carries a copy. Bump both.""" + repo_pin = PACKAGE.parents[1] / "scripts" / "data" / "fpgen-model.json" + assert fpgen_model.PIN == json.loads(repo_pin.read_text(encoding="utf-8")) + + +def test_the_real_pin_is_complete(): + pin = fpgen_model.PIN + assert pin["url"].startswith("https://github.com/") + assert f"/download/{pin['tag']}/" in pin["url"] + assert sorted(pin["file_sha256"]) == sorted(pin["files"]) + + +def test_constants_match_the_installed_fpgen(): + """fpgen's file names, data directory and five-week window are its own; if + a release changes them, this module would install a model fpgen cannot see. + FPGEN_NO_INIT keeps fpgen from loading (or fetching) while it is read.""" + probe = ( + "import json, fpgen.pkgman as p;" + "print(json.dumps({" + "'dir': str(p.DATA_DIR)," + "'compressed': sorted(v.name for v in p.FILE_PAIRS.values())," + "'decompressed': sorted(k.name for k in p.FILE_PAIRS.keys())}))" + ) + env = {**os.environ, "FPGEN_NO_INIT": "1"} + out = subprocess.run([sys.executable, "-c", probe], env=env, capture_output=True, text=True, check=True) + seen = json.loads(out.stdout.strip().splitlines()[-1]) + + assert seen["dir"] == str(fpgen_model.fpgen_data_dir()) + assert seen["compressed"] == sorted(fpgen_model.PIN["files"]) + assert seen["decompressed"] == sorted(fpgen_model.DECOMPRESSED_FILES) + assert "timedelta(weeks=5)" in (fpgen_model.fpgen_data_dir().parent / "pkgman.py").read_text() + + +_IMPORT_ORDER = """ +import sys +from camoufox import fpgen_model +real = fpgen_model.ensure_fpgen_model +seen = [] +def spy(*args, **kwargs): + seen.append("fpgen" in sys.modules) + return real(*args, **kwargs) +fpgen_model.ensure_fpgen_model = spy +from camoufox import {module} +{call} +print(seen) +""" + + +@pytest.mark.parametrize( + "module, call", + [ + pytest.param("fingerprints", "fingerprints._generator()", id="generate"), + pytest.param("webgl", "webgl.firefox_gpus('win')", id="preset-or-webgl_config"), + ], +) +def test_each_entry_point_installs_the_model_before_fpgen_loads(module, call): + script = _IMPORT_ORDER.format(module=module, call=call) + out = subprocess.run( + [sys.executable, "-c", script], cwd=PACKAGE.parent, capture_output=True, text=True, check=True + ) + assert out.stdout.strip().splitlines()[-1] == "[False]" diff --git a/scripts/data/fpgen-model.json b/scripts/data/fpgen-model.json index 93d041cf..7016ee11 100644 --- a/scripts/data/fpgen-model.json +++ b/scripts/data/fpgen-model.json @@ -10,5 +10,10 @@ "values.dat.zst", "values.json.zst" ], - "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see scripts/pin-fpgen-model.py. Bump by editing this file; the sha256 is the gate." + "file_sha256": { + "fingerprint-network.json.zst": "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4", + "values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", + "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f" + }, + "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate." } diff --git a/scripts/pin-fpgen-model.py b/scripts/pin-fpgen-model.py index ec3a1847..67b00c98 100755 --- a/scripts/pin-fpgen-model.py +++ b/scripts/pin-fpgen-model.py @@ -1,38 +1,10 @@ #!/usr/bin/env python3 -"""Install a pinned fpgen model, instead of letting fpgen fetch one itself. +"""Install the pinned fpgen model before anything imports fpgen. -WHY THIS EXISTS ---------------- -`fpgen` (scrapfly/fingerprint-generator) is where Camoufox's synthetic -fingerprints come from. It does not ship its model; it downloads one the first -time it is imported, and again whenever the files on disk are over five weeks -old. That fetch has four problems, all of them in fpgen/pkgman.py: - - 1. TLS verification is DISABLED on both the API call and the download - (`httpx.get(..., verify=False)`, `httpx.stream(..., verify=False)`), so - anyone on the network path can serve the model. - 2. The archive is never checksummed -- it goes straight into - `zipfile.ZipFile(...).extractall(DATA_DIR)`, which is also not guarded - against path traversal. - 3. The GitHub API is called unauthenticated, sharing a 60-request/hour limit - with every other job on the runner's IP. - 4. It takes the FIRST release the API lists and that release's first `.zip`. - GitHub sorts releases by `created_at`, and `model-4/2025` and - `model-2/2026` carry an identical `created_at` (2025-03-22T11:41:12Z, - inherited from the tag's commit), so the tie breaks toward the lower id - and fpgen always lands on the 2025 model. The 2026 model is unreachable - through that path. - -So an unpinned Camoufox generates from an April-2025 corpus -- newest Firefox -137, newest GPU an RTX 40 -- and cannot be talked into anything newer. - -This script puts the model named by scripts/data/fpgen-model.json where fpgen -looks, with TLS verification on and the sha256 checked, before anything imports -fpgen. fpgen then finds recent files and never calls the network. - -Locating the data directory must NOT import fpgen: importing is what triggers -the download this script exists to prevent. importlib.util.find_spec() reads the -module's origin without executing it. +The launcher does this itself on first use (pythonlib/camoufox/fpgen_model.py, +which explains why fpgen's own download is not used). This entry point exists +for jobs that need the model in place before they start: CI, image builds, and +the TypeScript golden recorder. Usage: python3 scripts/pin-fpgen-model.py # install if not already pinned @@ -41,84 +13,12 @@ Usage: """ import argparse -import hashlib -import importlib.util -import json -import os -import ssl import sys -import tempfile -import urllib.request -import zipfile -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SPEC = os.path.join(REPO, 'scripts', 'data', 'fpgen-model.json') -STAMP = '.pinned-model' - - -def load_spec(): - with open(SPEC, encoding='utf-8') as fh: - return json.load(fh) - - -def data_dir(): - """fpgen's data directory, found WITHOUT importing fpgen.""" - spec = importlib.util.find_spec('fpgen') - if spec is None or not spec.origin: - sys.exit('fpgen is not installed; `pip install -e pythonlib` first') - return os.path.join(os.path.dirname(spec.origin), 'data') - - -def digest(path, chunk=1 << 20): - h = hashlib.sha256() - with open(path, 'rb') as fh: - for block in iter(lambda: fh.read(chunk), b''): - h.update(block) - return h.hexdigest() - - -def is_pinned(spec, d): - """True when the pinned model is already in place.""" - try: - with open(os.path.join(d, STAMP), encoding='utf-8') as fh: - if fh.read().strip() != spec['sha256']: - return False - except OSError: - return False - return all(os.path.exists(os.path.join(d, f)) for f in spec['files']) - - -def download(spec, dest): - # Verification ON, unlike fpgen's own fetch. create_default_context() - # verifies the chain and the hostname. - ctx = ssl.create_default_context() - print(f'fetching {spec["url"]}', file=sys.stderr) - req = urllib.request.Request(spec['url'], headers={'User-Agent': 'camoufox-build'}) - with urllib.request.urlopen(req, context=ctx, timeout=60) as r, open(dest, 'wb') as fh: - while chunk := r.read(1 << 20): - fh.write(chunk) - size = os.path.getsize(dest) - if spec.get('size') and size != spec['size']: - sys.exit(f'size mismatch: got {size}, expected {spec["size"]}') - got = digest(dest) - if got != spec['sha256']: - sys.exit(f'sha256 mismatch:\n got {got}\n expected {spec["sha256"]}') - - -def install(spec, archive, d): - os.makedirs(d, exist_ok=True) - with zipfile.ZipFile(archive) as z: - # fpgen extracts without checking; do not copy that. A member escaping - # the data directory would write anywhere the build user can. - for name in z.namelist(): - target = os.path.realpath(os.path.join(d, name)) - if not target.startswith(os.path.realpath(d) + os.sep): - sys.exit(f'refusing to extract outside the data dir: {name}') - z.extractall(d) - # Written last: an interrupted extract leaves no stamp, so the next run - # reinstalls rather than trusting a partial model. - with open(os.path.join(d, STAMP), 'w', encoding='utf-8') as fh: - fh.write(spec['sha256'] + '\n') +try: + from camoufox.fpgen_model import PIN, ensure_fpgen_model, fpgen_data_dir, is_pinned +except ImportError: + sys.exit('camoufox is not installed; `pip install -e pythonlib` first') def main(): @@ -128,26 +28,16 @@ def main(): ap.add_argument('--force', action='store_true', help='reinstall even if already pinned') args = ap.parse_args() - spec = load_spec() - d = data_dir() - if args.check: - if is_pinned(spec, d): - print(f'OK: fpgen model pinned to {spec["tag"]}') + if is_pinned(): + print(f'OK: fpgen model pinned to {PIN["tag"]}') return 0 - print(f'fpgen model is NOT pinned to {spec["tag"]}; run scripts/pin-fpgen-model.py', + print(f'fpgen model is NOT pinned to {PIN["tag"]}; run scripts/pin-fpgen-model.py', file=sys.stderr) return 1 - if is_pinned(spec, d) and not args.force: - print(f'OK: fpgen model already pinned to {spec["tag"]}', file=sys.stderr) - return 0 - - with tempfile.TemporaryDirectory() as tmp: - archive = os.path.join(tmp, 'model-release.zip') - download(spec, archive) - install(spec, archive, d) - print(f'OK: pinned fpgen model {spec["tag"]} -> {d}', file=sys.stderr) + ensure_fpgen_model(force=args.force) + print(f'OK: fpgen model {PIN["tag"]} pinned in {fpgen_data_dir()}', file=sys.stderr) return 0 diff --git a/typescript/src/fpgen/pin.ts b/typescript/src/fpgen/pin.ts index db31d4b4..8e2ff973 100644 --- a/typescript/src/fpgen/pin.ts +++ b/typescript/src/fpgen/pin.ts @@ -6,7 +6,7 @@ * tests/fpgen-model.test.ts fails if the two ever disagree, so bumping the * model means editing both (the sha256 is the gate in each). * - * See scripts/pin-fpgen-model.py for why fpgen's own downloader is not used: + * See pythonlib/camoufox/fpgen_model.py for why fpgen's own downloader is not used: * it disables TLS verification, never checks a digest, and its "first listed * release" rule cannot reach this tag. */ @@ -19,6 +19,8 @@ export interface ModelPin { readonly url: string; readonly repo: string; readonly files: readonly string[]; + /** Each file's sha256, for checking an installed model without its archive. */ + readonly file_sha256: Readonly>; } export const MODEL_PIN: ModelPin = { @@ -29,4 +31,12 @@ export const MODEL_PIN: ModelPin = { url: "https://github.com/scrapfly/fingerprint-generator/releases/download/model-2/2026/model-release.zip", repo: "scrapfly/fingerprint-generator", files: ["fingerprint-network.json.zst", "values.dat.zst", "values.json.zst"], + file_sha256: { + "fingerprint-network.json.zst": + "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4", + "values.dat.zst": + "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", + "values.json.zst": + "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f", + }, };