Install the pinned fpgen model before fpgen is imported

pythonlib left the model to fpgen, whose first import downloads the
first release the GitHub API lists: model-4/2025, whose WebGL records
have no vendor or renderer. Every generated launch on a fresh install
failed with KeyError: 'vendor'. scripts/pin-fpgen-model.py pinned the
model for CI only, and fpgen's five-week refresh replaced even that
pin, under a stamp the script's --check still trusted.

fpgen is now imported only through fpgen_model.load_fpgen(), which
installs the release named by the pin, checks the archive and each
file against their sha256, and dates the files past fpgen's refresh.
It writes the layout and stamp the script and the TypeScript launcher
use, verifies an install by hashing it, and leaves FPGEN_MODEL_URL to
fpgen. `camoufox fetch` installs the model as well. The pin gains each
file's sha256; the package carries a copy of it, and the script now
calls the module.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
(cherry picked from commit f043de3598)
This commit is contained in:
coffeegrind123
2026-09-27 20:09:03 +00:00
committed by Jake Writer
parent 0c6cc0a397
commit 7f0e52e792
11 changed files with 569 additions and 134 deletions
+2
View File
@@ -37,6 +37,8 @@ Next, download the Camoufox browser:
camoufox fetch
```
`fetch` also installs fpgen's model, pinned by sha256, into fpgen's package directory. Otherwise the first generated fingerprint installs it. Run `fetch` as that directory's owner if the browser will run as another user, e.g. while building a Docker image.
To uninstall, run `camoufox remove`.
---
+2
View File
@@ -11,6 +11,7 @@ from typing import Any, List, Optional, Tuple
import rich_click as click
from .addons import DefaultAddons, maybe_download_addons
from .fpgen_model import ensure_fpgen_model
from .geolocation import (
ALLOW_GEOIP,
GEOIP_DIR,
@@ -323,6 +324,7 @@ def fetch(version):
if ALLOW_GEOIP:
download_mmdb()
maybe_download_addons(list(DefaultAddons))
ensure_fpgen_model()
def _set_channel(repo_name: str, channel_type: str):
+6
View File
@@ -184,3 +184,9 @@ class ProfileDirectoryError(RuntimeError):
"""Raised when Camoufox's required runtime directory cannot be prepared."""
...
class FpgenModelError(RuntimeError):
"""Raised when fpgen's pinned model cannot be installed where fpgen reads it."""
...
+2 -2
View File
@@ -29,9 +29,9 @@ _FP_GENERATOR = None
def _generator():
global _FP_GENERATOR
if _FP_GENERATOR is None:
from fpgen import Generator
from .fpgen_model import load_fpgen
_FP_GENERATOR = Generator()
_FP_GENERATOR = load_fpgen().Generator()
return _FP_GENERATOR
+19
View File
@@ -0,0 +1,19 @@
{
"tag": "model-2/2026",
"asset": "model-release.zip",
"size": 1564571,
"sha256": "6530b8322cdaa4ec042921c8d9a0369a0e6e0269ba636c01a7203e4a2f109936",
"url": "https://github.com/scrapfly/fingerprint-generator/releases/download/model-2/2026/model-release.zip",
"repo": "scrapfly/fingerprint-generator",
"files": [
"fingerprint-network.json.zst",
"values.dat.zst",
"values.json.zst"
],
"file_sha256": {
"fingerprint-network.json.zst": "e1b0a7e60837c347f4b7d5dad4a20c356d521e0593e1bf4a8be39ea1e6a41ac4",
"values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122",
"values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f"
},
"note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate."
}
+168
View File
@@ -0,0 +1,168 @@
"""fpgen's model, installed from the pinned release before fpgen is imported.
Left to itself, `import fpgen` downloads its model from the first release the
GitHub API lists, with TLS verification off and no checksum, and downloads it
again whenever the files are five weeks old (fpgen/pkgman.py). The first
release listed is model-4/2025, whose WebGL records have no vendor or renderer,
so every generated launch failed with KeyError: 'vendor'.
So fpgen is only ever imported through load_fpgen(), which first installs the
release named by fpgen-model.json (a copy of scripts/data/fpgen-model.json),
checks the archive and each file against its sha256, and dates the files in the
future so fpgen's five-week refresh never replaces them. The data directory
layout, including the `.pinned-model` stamp, is the one
scripts/pin-fpgen-model.py and the TypeScript launcher write.
"""
import hashlib
import importlib.util
import json
import os
import shutil
import sys
import tempfile
import time
from pathlib import Path
from threading import Lock
from types import ModuleType
from typing import Optional
from zipfile import ZipFile
from .exceptions import CorruptedDownload, FpgenModelError
from .pkgman import verify_sha256, webdl
PIN = json.loads((Path(__file__).parent / 'fpgen-model.json').read_text(encoding='utf-8'))
# Holds the archive's sha256, written last: a stamp means the install finished.
STAMP = '.pinned-model'
# `python -m fpgen decompress` replaces the archive's files with these, which
# fpgen then reads in preference to them.
DECOMPRESSED_FILES = ('fingerprint-network.json', 'values.json', 'values.dat')
# 2100-01-01T00:00:00Z. fpgen refetches files whose mtime is older than this
# window (pkgman.files_are_recent).
PINNED_MTIME = 4102444800
FPGEN_MAX_AGE_S = 5 * 7 * 24 * 3600
# fpgen fetches a model from this URL instead of GitHub. A caller who set it
# chose their model, so it is left alone.
CUSTOM_MODEL_ENV = 'FPGEN_MODEL_URL'
_LOCK = Lock()
_ready = False
def fpgen_data_dir() -> Path:
"""Where fpgen reads its model, found without importing fpgen (which fetches)."""
spec = importlib.util.find_spec('fpgen')
if spec is None or spec.origin is None:
raise FpgenModelError('fpgen is not installed.')
return Path(spec.origin).parent / 'data'
def load_fpgen() -> ModuleType:
"""fpgen, imported only once the pinned model is in place."""
global _ready
if not _ready:
ensure_fpgen_model()
_ready = True
import fpgen
import fpgen.utils # noqa: F401 -- callers use fpgen.utils._lookup_possibilities
return fpgen
def is_pinned(data_dir: Optional[Path] = None) -> bool:
"""Whether the pinned model's files are in place, whatever wrote them."""
data_dir = data_dir or fpgen_data_dir()
return all(
(data_dir / name).exists() and _hash(data_dir / name) == digest
for name, digest in PIN['file_sha256'].items()
)
def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> None:
"""Install the pinned model unless it is already there, and keep fpgen from
replacing it. Hashing the installed files takes ~2 ms, and catches a model
fpgen fetched over an earlier install."""
if os.getenv(CUSTOM_MODEL_ENV):
return
data_dir = data_dir or fpgen_data_dir()
with _LOCK:
if any((data_dir / name).exists() for name in DECOMPRESSED_FILES):
raise FpgenModelError(
f"fpgen's model in {data_dir} is decompressed, so it cannot be checked against "
f"the pinned {PIN['tag']}. Remove it with `python -m fpgen remove`; Camoufox "
'then installs the pinned model.'
)
try:
if force or not is_pinned(data_dir):
_install(data_dir)
_stamp(data_dir)
except PermissionError as e:
raise FpgenModelError(
f"fpgen's model directory is not writable by this user: {data_dir}\n"
'Install the model as its owner once, e.g. while building an image: '
'camoufox fetch'
) from e
def _install(data_dir: Path) -> None:
print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr)
buffer = webdl(PIN['url'], progress_callback=lambda done, total: None)
verify_sha256(buffer, PIN['sha256'], 'fpgen model')
data_dir.mkdir(parents=True, exist_ok=True)
staging = Path(tempfile.mkdtemp(prefix='.model-', dir=data_dir))
try:
with ZipFile(buffer) as zf:
if sorted(zf.namelist()) != sorted(PIN['file_sha256']):
raise CorruptedDownload(
f"fpgen model archive holds {sorted(zf.namelist())}, "
f"expected {sorted(PIN['file_sha256'])}."
)
# Members are read by name, never extracted by path.
for name, expected in PIN['file_sha256'].items():
data = zf.read(name)
if hashlib.sha256(data).hexdigest() != expected:
raise CorruptedDownload(f'fpgen model member {name} does not match its pinned sha256.')
(staging / name).write_bytes(data)
(data_dir / STAMP).unlink(missing_ok=True)
for name in PIN['file_sha256']:
# Dated before the move, so fpgen never sees it as stale; each
# replace is atomic, so a concurrent reader never sees a torn file.
os.utime(staging / name, (PINNED_MTIME, PINNED_MTIME))
os.replace(staging / name, data_dir / name)
finally:
shutil.rmtree(staging, ignore_errors=True)
def _stamp(data_dir: Path) -> None:
"""Date the files past fpgen's refresh and write the stamp. Files seeded by
another user (a root-built image) may be left as they are while fpgen still
considers them current."""
for name in PIN['file_sha256']:
path = data_dir / name
if int(path.stat().st_mtime) == PINNED_MTIME:
continue
try:
os.utime(path, (PINNED_MTIME, PINNED_MTIME))
except PermissionError:
if path.stat().st_mtime < time.time() - FPGEN_MAX_AGE_S:
raise
stamp = data_dir / STAMP
try:
if stamp.read_text(encoding='utf-8').strip() == PIN['sha256']:
return
except OSError:
pass
stamp.write_text(PIN['sha256'] + '\n', encoding='utf-8')
def _hash(path: Path) -> str:
digest = hashlib.sha256()
with open(path, 'rb') as f:
for block in iter(lambda: f.read(1 << 20), b''):
digest.update(block)
return digest.hexdigest()
+3 -6
View File
@@ -18,6 +18,7 @@ import orjson
from .coherence import gpu_fits_os
from .fingerprints import _FPGEN_OS, gpu_screen_is_plausible, is_software_renderer
from .fpgen_model import load_fpgen
# Extensions a release Firefox never exposes (draft extensions behind
# webgl.enable-draft-extensions, or mobile-only): fpgen's corpus carries some
@@ -60,9 +61,7 @@ def _filtered_extensions(target_os: str) -> FrozenSet[str]:
@lru_cache(maxsize=None)
def _lookup_index(node: str) -> Dict[str, str]:
"""fpgen's lookup index for every value of `node`, keyed by its JSON."""
from fpgen.utils import _lookup_possibilities
return _lookup_possibilities(node, casefold=False)
return load_fpgen().utils._lookup_possibilities(node, casefold=False)
def _pin(node: str, value: Any) -> Tuple[str, str]:
@@ -79,10 +78,8 @@ def _pin(node: str, value: Any) -> Tuple[str, str]:
@lru_cache(maxsize=None)
def _trace(target: str, target_os: str, pinned: Tuple[Tuple[str, str], ...] = ()) -> Tuple[Any, ...]:
"""fpgen's distribution of `target` for Firefox on `target_os`, in its order."""
import fpgen
return tuple(
fpgen.trace(
load_fpgen().trace(
target=target,
browser='Firefox',
os=_FPGEN_OS[target_os],
+336
View File
@@ -0,0 +1,336 @@
"""fpgen's model is the pinned release, installed before fpgen is first imported.
Left to itself, `import fpgen` downloads the first release the GitHub API
lists -- model-4/2025, whose WebGL records carry no vendor or renderer -- so a
pip-installed camoufox failed every generated launch with KeyError: 'vendor' in
webgl.py. scripts/pin-fpgen-model.py pinned the model for CI only.
"""
import ast
import hashlib
import io
import json
import os
import subprocess
import sys
import time
import zipfile
from pathlib import Path
import pytest
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from camoufox import fpgen_model # noqa: E402
from camoufox.exceptions import CorruptedDownload, FpgenModelError # noqa: E402
from camoufox.fpgen_model import FPGEN_MAX_AGE_S, PINNED_MTIME, STAMP, ensure_fpgen_model # noqa: E402
PACKAGE = Path(fpgen_model.__file__).resolve().parent
MEMBERS = {
"fingerprint-network.json.zst": b"network" * 1000,
"values.json.zst": b"values-json" * 1000,
"values.dat.zst": b"values-dat" * 1000,
}
STALE = time.time() - FPGEN_MAX_AGE_S - 86400
def _archive(members):
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
for name, data in members.items():
zf.writestr(name, data)
return buf.getvalue()
def _sha256(data):
return hashlib.sha256(data).hexdigest()
@pytest.fixture
def pinned(monkeypatch):
"""Pin a small synthetic model instead of the real release, and record the
downloads. Set `.payload` to serve other bytes than the pin describes."""
archive = _archive(MEMBERS)
pin = {
**fpgen_model.PIN,
"size": len(archive),
"sha256": _sha256(archive),
"files": sorted(MEMBERS),
"file_sha256": {name: _sha256(data) for name, data in MEMBERS.items()},
}
monkeypatch.setattr(fpgen_model, "PIN", pin)
monkeypatch.delenv("FPGEN_MODEL_URL", raising=False)
class Downloads(list):
payload = archive
downloads = Downloads()
def fake_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None):
downloads.append(url)
return io.BytesIO(downloads.payload)
monkeypatch.setattr(fpgen_model, "webdl", fake_webdl)
return downloads
def _write(data_dir, members, mtime=None, stamp=None):
data_dir.mkdir(parents=True, exist_ok=True)
for name, data in members.items():
path = data_dir / name
path.write_bytes(data)
if mtime is not None:
os.utime(path, (mtime, mtime))
if stamp is not None:
(data_dir / STAMP).write_text(stamp + "\n")
def _mtimes(data_dir):
return {int((data_dir / name).stat().st_mtime) for name in MEMBERS}
def test_only_the_model_module_imports_fpgen():
"""Every fpgen import goes through fpgen_model.load_fpgen(), which installs
the pinned model first. A direct import anywhere else lets fpgen fetch
model-4/2025 before the pin is in place."""
offenders = []
for path in sorted(PACKAGE.rglob("*.py")):
if path.name == "fpgen_model.py":
continue
for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
if isinstance(node, ast.Import):
names = [alias.name for alias in node.names]
elif isinstance(node, ast.ImportFrom) and node.level == 0:
names = [node.module or ""]
else:
continue
if any(name == "fpgen" or name.startswith("fpgen.") for name in names):
offenders.append(f"{path.relative_to(PACKAGE)}:{node.lineno}")
assert offenders == []
def test_missing_model_is_installed_verified_and_stamped(tmp_path, pinned):
data_dir = tmp_path / "data"
ensure_fpgen_model(data_dir)
assert pinned == [fpgen_model.PIN["url"]]
for name, data in MEMBERS.items():
assert (data_dir / name).read_bytes() == data
assert _mtimes(data_dir) == {PINNED_MTIME}
# The same stamp scripts/pin-fpgen-model.py and the TypeScript launcher write.
assert (data_dir / STAMP).read_text().strip() == fpgen_model.PIN["sha256"]
assert sorted(p.name for p in data_dir.iterdir()) == sorted([*MEMBERS, STAMP])
def test_pinned_model_is_not_downloaded_again(tmp_path, pinned):
data_dir = tmp_path / "data"
ensure_fpgen_model(data_dir)
ensure_fpgen_model(data_dir)
assert len(pinned) == 1
def test_stale_pinned_model_is_restamped_not_refetched(tmp_path, pinned):
"""Five weeks after install fpgen would fetch model-4/2025 over the pin."""
data_dir = tmp_path / "data"
_write(data_dir, MEMBERS, mtime=STALE, stamp=fpgen_model.PIN["sha256"])
ensure_fpgen_model(data_dir)
assert pinned == []
assert _mtimes(data_dir) == {PINNED_MTIME}
def test_a_model_fpgen_overwrote_under_a_valid_stamp_is_replaced(tmp_path, pinned):
"""The stamp outlives fpgen's own five-week refetch, so files are hashed."""
data_dir = tmp_path / "data"
_write(data_dir, {**MEMBERS, "values.dat.zst": b"model-4/2025"}, stamp=fpgen_model.PIN["sha256"])
ensure_fpgen_model(data_dir)
assert len(pinned) == 1
assert (data_dir / "values.dat.zst").read_bytes() == MEMBERS["values.dat.zst"]
def test_matching_files_without_a_stamp_are_adopted(tmp_path, pinned):
data_dir = tmp_path / "data"
_write(data_dir, MEMBERS)
ensure_fpgen_model(data_dir)
assert pinned == []
assert (data_dir / STAMP).read_text().strip() == fpgen_model.PIN["sha256"]
def test_partial_model_is_completed(tmp_path, pinned):
data_dir = tmp_path / "data"
_write(data_dir, {"values.json.zst": MEMBERS["values.json.zst"]})
ensure_fpgen_model(data_dir)
assert len(pinned) == 1
assert all((data_dir / name).exists() for name in MEMBERS)
def test_force_reinstalls(tmp_path, pinned):
data_dir = tmp_path / "data"
ensure_fpgen_model(data_dir)
ensure_fpgen_model(data_dir, force=True)
assert len(pinned) == 2
@pytest.mark.parametrize(
"tamper",
[
pytest.param(lambda a: a[:-1], id="truncated-archive"),
pytest.param(lambda a: _archive({**MEMBERS, "values.dat.zst": b"x"}), id="member-changed"),
],
)
def test_tampered_download_installs_nothing(tmp_path, pinned, tamper):
data_dir = tmp_path / "data"
pinned.payload = tamper(_archive(MEMBERS))
with pytest.raises(CorruptedDownload):
ensure_fpgen_model(data_dir)
assert not data_dir.exists() or list(data_dir.iterdir()) == []
def test_member_hashes_are_checked_even_when_the_archive_hash_matches(tmp_path, pinned):
bad = _archive({**MEMBERS, "values.dat.zst": b"x"})
pinned.payload = bad
fpgen_model.PIN.update(size=len(bad), sha256=_sha256(bad))
with pytest.raises(CorruptedDownload, match="values.dat.zst"):
ensure_fpgen_model(tmp_path / "data")
def test_decompressed_model_fails_loudly(tmp_path, pinned):
"""`python -m fpgen decompress` leaves files the pin cannot verify, and fpgen
reads them in preference to the verified ones."""
data_dir = tmp_path / "data"
_write(data_dir, {name: b"{}" for name in fpgen_model.DECOMPRESSED_FILES})
with pytest.raises(FpgenModelError, match="fpgen remove"):
ensure_fpgen_model(data_dir)
assert pinned == []
def test_custom_model_url_is_left_to_fpgen(tmp_path, pinned, monkeypatch):
monkeypatch.setenv("FPGEN_MODEL_URL", "https://example.invalid/model.zip")
data_dir = tmp_path / "data"
ensure_fpgen_model(data_dir)
assert pinned == []
assert not data_dir.exists()
needs_non_root = pytest.mark.skipif(
hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores file permissions"
)
@pytest.fixture
def read_only(tmp_path):
data_dir = tmp_path / "data"
yield data_dir
if data_dir.exists():
data_dir.chmod(0o755)
@needs_non_root
def test_read_only_missing_model_names_the_fix(read_only, pinned):
read_only.mkdir()
read_only.chmod(0o555)
with pytest.raises(FpgenModelError, match="camoufox fetch"):
ensure_fpgen_model(read_only)
@needs_non_root
def test_read_only_current_pinned_model_is_used(read_only, pinned):
"""A root-seeded image run as another user: fpgen accepts the files as they
are for five weeks, so the launch does not need to stamp them."""
_write(read_only, MEMBERS, stamp=fpgen_model.PIN["sha256"])
read_only.chmod(0o555)
for name in MEMBERS:
(read_only / name).chmod(0o444)
ensure_fpgen_model(read_only)
assert pinned == []
def test_install_writes_nothing_to_stdout(tmp_path, pinned, capfd):
"""A caller's stdout may be data (a probe printing JSON)."""
ensure_fpgen_model(tmp_path / "data")
out, err = capfd.readouterr()
assert out == ""
assert "fpgen model" in err
def test_the_packaged_pin_is_the_repo_pin():
"""The package cannot ship scripts/, so it carries a copy. Bump both."""
repo_pin = PACKAGE.parents[1] / "scripts" / "data" / "fpgen-model.json"
assert fpgen_model.PIN == json.loads(repo_pin.read_text(encoding="utf-8"))
def test_the_real_pin_is_complete():
pin = fpgen_model.PIN
assert pin["url"].startswith("https://github.com/")
assert f"/download/{pin['tag']}/" in pin["url"]
assert sorted(pin["file_sha256"]) == sorted(pin["files"])
def test_constants_match_the_installed_fpgen():
"""fpgen's file names, data directory and five-week window are its own; if
a release changes them, this module would install a model fpgen cannot see.
FPGEN_NO_INIT keeps fpgen from loading (or fetching) while it is read."""
probe = (
"import json, fpgen.pkgman as p;"
"print(json.dumps({"
"'dir': str(p.DATA_DIR),"
"'compressed': sorted(v.name for v in p.FILE_PAIRS.values()),"
"'decompressed': sorted(k.name for k in p.FILE_PAIRS.keys())}))"
)
env = {**os.environ, "FPGEN_NO_INIT": "1"}
out = subprocess.run([sys.executable, "-c", probe], env=env, capture_output=True, text=True, check=True)
seen = json.loads(out.stdout.strip().splitlines()[-1])
assert seen["dir"] == str(fpgen_model.fpgen_data_dir())
assert seen["compressed"] == sorted(fpgen_model.PIN["files"])
assert seen["decompressed"] == sorted(fpgen_model.DECOMPRESSED_FILES)
assert "timedelta(weeks=5)" in (fpgen_model.fpgen_data_dir().parent / "pkgman.py").read_text()
_IMPORT_ORDER = """
import sys
from camoufox import fpgen_model
real = fpgen_model.ensure_fpgen_model
seen = []
def spy(*args, **kwargs):
seen.append("fpgen" in sys.modules)
return real(*args, **kwargs)
fpgen_model.ensure_fpgen_model = spy
from camoufox import {module}
{call}
print(seen)
"""
@pytest.mark.parametrize(
"module, call",
[
pytest.param("fingerprints", "fingerprints._generator()", id="generate"),
pytest.param("webgl", "webgl.firefox_gpus('win')", id="preset-or-webgl_config"),
],
)
def test_each_entry_point_installs_the_model_before_fpgen_loads(module, call):
script = _IMPORT_ORDER.format(module=module, call=call)
out = subprocess.run(
[sys.executable, "-c", script], cwd=PACKAGE.parent, capture_output=True, text=True, check=True
)
assert out.stdout.strip().splitlines()[-1] == "[False]"