From 8823d399b81920c011cc2db14ef7edc8653ae260 Mon Sep 17 00:00:00 2001 From: Jake Writer Date: Sun, 27 Sep 2026 11:55:21 -0600 Subject: [PATCH] fix(fpgen): keep a values.dat the TypeScript launcher decompressed from the pin CAMOUFOX_FPGEN_DATA may point the TS launcher at pythonlib's fpgen data/, and it decompresses values.dat there. ensure_fpgen_model() refused to run whenever values.dat existed, so sharing the directory broke every Python launch. The pin now carries values.dat's sha256 (all three twins): a matching values.dat is kept, any other is removed, since fpgen reads it in preference to the verified archive. The files `fpgen decompress` leaves still fail loudly. Co-Authored-By: Claude Opus 5.5 --- pythonlib/camoufox/fpgen-model.json | 3 +++ pythonlib/camoufox/fpgen_model.py | 20 +++++++++++++++++++- pythonlib/tests/test_fpgen_model.py | 28 ++++++++++++++++++++++++++++ scripts/data/fpgen-model.json | 3 +++ typescript/src/fpgen/pin.ts | 10 ++++++++++ 5 files changed, 63 insertions(+), 1 deletion(-) diff --git a/pythonlib/camoufox/fpgen-model.json b/pythonlib/camoufox/fpgen-model.json index 7016ee1..d0ce1e5 100644 --- a/pythonlib/camoufox/fpgen-model.json +++ b/pythonlib/camoufox/fpgen-model.json @@ -15,5 +15,8 @@ "values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f" }, + "decompressed_sha256": { + "values.dat": "9ce7d16630e91654e73988f7d37e8c99987c09b9c779a7bb9dc7ad6355dcc2f0" + }, "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate." } diff --git a/pythonlib/camoufox/fpgen_model.py b/pythonlib/camoufox/fpgen_model.py index 28d39b8..4f98fda 100644 --- a/pythonlib/camoufox/fpgen_model.py +++ b/pythonlib/camoufox/fpgen_model.py @@ -39,6 +39,9 @@ STAMP = '.pinned-model' # `python -m fpgen decompress` replaces the archive's files with these, which # fpgen then reads in preference to them. DECOMPRESSED_FILES = ('fingerprint-network.json', 'values.json', 'values.dat') +# The TypeScript launcher decompresses values.dat beside the archive's files, +# and CAMOUFOX_FPGEN_DATA may point it at this directory. The pin carries its +# hash, so a values.dat from the pinned model is kept and any other is dropped. # 2100-01-01T00:00:00Z. fpgen refetches files whose mtime is older than this # window (pkgman.files_are_recent). @@ -90,7 +93,11 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> return data_dir = data_dir or fpgen_data_dir() with _LOCK: - if any((data_dir / name).exists() for name in DECOMPRESSED_FILES): + if any( + (data_dir / name).exists() + for name in DECOMPRESSED_FILES + if name not in PIN['decompressed_sha256'] + ): raise FpgenModelError( f"fpgen's model in {data_dir} is decompressed, so it cannot be checked against " f"the pinned {PIN['tag']}. Remove it with `python -m fpgen remove`; Camoufox " @@ -99,6 +106,7 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> try: if force or not is_pinned(data_dir): _install(data_dir) + _drop_foreign_decompressed(data_dir) _stamp(data_dir) except PermissionError as e: raise FpgenModelError( @@ -138,6 +146,16 @@ def _install(data_dir: Path) -> None: shutil.rmtree(staging, ignore_errors=True) +def _drop_foreign_decompressed(data_dir: Path) -> None: + """Remove a decompressed file that is not the pinned model's: fpgen would + read it in preference to the verified archive. Hashing values.dat (~210 MB) + takes ~0.2 s, once per process, and only when the file is there.""" + for name, digest in PIN['decompressed_sha256'].items(): + path = data_dir / name + if path.exists() and _hash(path) != digest: + path.unlink(missing_ok=True) + + def _stamp(data_dir: Path) -> None: """Date the files past fpgen's refresh and write the stamp. Files seeded by another user (a root-built image) may be left as they are while fpgen still diff --git a/pythonlib/tests/test_fpgen_model.py b/pythonlib/tests/test_fpgen_model.py index 04614ca..08b140f 100644 --- a/pythonlib/tests/test_fpgen_model.py +++ b/pythonlib/tests/test_fpgen_model.py @@ -31,6 +31,8 @@ MEMBERS = { "values.json.zst": b"values-json" * 1000, "values.dat.zst": b"values-dat" * 1000, } +# What the synthetic values.dat.zst decompresses to, as far as the pin says. +VALUES_DAT = b"decompressed-values" * 1000 STALE = time.time() - FPGEN_MAX_AGE_S - 86400 @@ -57,6 +59,7 @@ def pinned(monkeypatch): "sha256": _sha256(archive), "files": sorted(MEMBERS), "file_sha256": {name: _sha256(data) for name, data in MEMBERS.items()}, + "decompressed_sha256": {"values.dat": _sha256(VALUES_DAT)}, } monkeypatch.setattr(fpgen_model, "PIN", pin) monkeypatch.delenv("FPGEN_MODEL_URL", raising=False) @@ -217,6 +220,31 @@ def test_decompressed_model_fails_loudly(tmp_path, pinned): assert pinned == [] +def test_a_pinned_values_dat_is_shared_with_the_typescript_launcher(tmp_path, pinned): + """CAMOUFOX_FPGEN_DATA may point the TS launcher at this directory, and it + decompresses values.dat there. The pinned model's values.dat stays.""" + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + _write(data_dir, {"values.dat": VALUES_DAT}) + + ensure_fpgen_model(data_dir) + + assert (data_dir / "values.dat").read_bytes() == VALUES_DAT + assert pinned == [fpgen_model.PIN["url"]] + + +def test_a_values_dat_from_another_model_is_dropped(tmp_path, pinned): + """fpgen reads values.dat in preference to values.dat.zst, so one left over + from another model would pair its values with the pinned network.""" + data_dir = tmp_path / "data" + ensure_fpgen_model(data_dir) + _write(data_dir, {"values.dat": b"another model"}) + + ensure_fpgen_model(data_dir) + + assert not (data_dir / "values.dat").exists() + + def test_custom_model_url_is_left_to_fpgen(tmp_path, pinned, monkeypatch): monkeypatch.setenv("FPGEN_MODEL_URL", "https://example.invalid/model.zip") data_dir = tmp_path / "data" diff --git a/scripts/data/fpgen-model.json b/scripts/data/fpgen-model.json index 7016ee1..d0ce1e5 100644 --- a/scripts/data/fpgen-model.json +++ b/scripts/data/fpgen-model.json @@ -15,5 +15,8 @@ "values.dat.zst": "3da2cf0891a4a85ef6458f0fbdf9346acfcd04e5fd279a0ea22d7919e4eaf122", "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f" }, + "decompressed_sha256": { + "values.dat": "9ce7d16630e91654e73988f7d37e8c99987c09b9c779a7bb9dc7ad6355dcc2f0" + }, "note": "fpgen's model data, pinned. fpgen fetches this itself on first import with TLS verification DISABLED, no checksum, and a release selection that cannot reach this tag -- see pythonlib/camoufox/fpgen_model.py. Bump by editing this file and its twins, pythonlib/camoufox/fpgen-model.json and typescript/src/fpgen/pin.ts (tests fail if they disagree); the sha256 values are the gate." } diff --git a/typescript/src/fpgen/pin.ts b/typescript/src/fpgen/pin.ts index 8e2ff97..567bc54 100644 --- a/typescript/src/fpgen/pin.ts +++ b/typescript/src/fpgen/pin.ts @@ -21,6 +21,12 @@ export interface ModelPin { readonly files: readonly string[]; /** Each file's sha256, for checking an installed model without its archive. */ readonly file_sha256: Readonly>; + /** + * sha256 of each file the model decompresses to. values.dat is shared with + * pythonlib when CAMOUFOX_FPGEN_DATA points at its fpgen `data/`, and this + * is how pythonlib tells one decompressed from the pinned model. + */ + readonly decompressed_sha256: Readonly>; } export const MODEL_PIN: ModelPin = { @@ -39,4 +45,8 @@ export const MODEL_PIN: ModelPin = { "values.json.zst": "294decde5b6a1a52ed53d50a894130fa5c58f7cc804b78198f5c33f55b3ba66f", }, + decompressed_sha256: { + "values.dat": + "9ce7d16630e91654e73988f7d37e8c99987c09b9c779a7bb9dc7ad6355dcc2f0", + }, };