Cache the pinned fpgen model archive across CI runs

Nine jobs downloaded the fpgen model from its GitHub release on every
run, so a 504 from GitHub failed pythonlib and skipped every browser
job behind it. pin-fpgen-model.py --archive keeps the downloaded archive
in a file and installs from it when it is there, checked against the
pin either way. The fpgen-model action caches that file, keyed on
fpgen-model.json, and every job installs the model through it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jake Writer
2026-11-09 23:02:24 +00:00
co-authored by Claude Opus 5.5
parent 2ac91c0dc4
commit b8db95563b
5 changed files with 90 additions and 60 deletions
+6 -1
View File
@@ -10,10 +10,12 @@ Usage:
python3 python/scripts/pin-fpgen-model.py # install if not already pinned
python3 python/scripts/pin-fpgen-model.py --check # verify only; non-zero if not pinned
python3 python/scripts/pin-fpgen-model.py --force # re-download and reinstall
python3 python/scripts/pin-fpgen-model.py --archive F # keep the archive in F, and install from F when it is there
"""
import argparse
import sys
from pathlib import Path
try:
from camoufox.fpgen_model import PIN, ensure_fpgen_model, fpgen_data_dir, is_pinned
@@ -26,6 +28,9 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('--check', action='store_true', help='verify only; non-zero if not pinned')
ap.add_argument('--force', action='store_true', help='reinstall even if already pinned')
ap.add_argument('--archive', type=Path,
help='keep the downloaded archive in this file, and install from it when it is '
'there (still checked against the pin), so CI can cache it')
args = ap.parse_args()
if args.check:
@@ -36,7 +41,7 @@ def main():
file=sys.stderr)
return 1
ensure_fpgen_model(force=args.force)
ensure_fpgen_model(force=args.force, archive=args.archive)
print(f'OK: fpgen model {PIN["tag"]} pinned in {fpgen_data_dir()}', file=sys.stderr)
return 0
+18 -7
View File
@@ -19,6 +19,7 @@ import json
import os
import sys
import time
from io import BytesIO
from pathlib import Path
from threading import Lock
from types import ModuleType
@@ -83,10 +84,13 @@ def is_pinned(data_dir: Optional[Path] = None) -> bool:
)
def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> None:
def ensure_fpgen_model(
data_dir: Optional[Path] = None, force: bool = False, archive: Optional[Path] = None
) -> None:
"""Install the pinned model unless it is already there, and keep fpgen from
replacing it. Hashing the installed files takes ~2 ms, and catches a model
fpgen fetched over an earlier install."""
fpgen fetched over an earlier install. `archive` is a file the downloaded
archive is kept in and installed from, checked against the pin either way."""
if os.getenv(CUSTOM_MODEL_ENV):
return
data_dir = data_dir or fpgen_data_dir()
@@ -103,7 +107,7 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) ->
)
try:
if force or not is_pinned(data_dir):
_install(data_dir)
_install(data_dir, archive)
_drop_foreign_decompressed(data_dir)
_stamp(data_dir)
except PermissionError as e:
@@ -116,10 +120,17 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) ->
) from e
def _install(data_dir: Path) -> None:
print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr)
buffer = webdl(PIN['url'], progress_callback=lambda done, total: None)
verify_sha256(buffer, PIN['sha256'], 'fpgen model')
def _install(data_dir: Path, archive: Optional[Path] = None) -> None:
kept = archive is not None and archive.exists()
if kept:
buffer = BytesIO(archive.read_bytes())
else:
print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr)
buffer = webdl(PIN['url'], progress_callback=lambda done, total: None)
verify_sha256(buffer, PIN['sha256'], f'fpgen model {archive}' if kept else 'fpgen model')
if archive and not kept:
archive.parent.mkdir(parents=True, exist_ok=True)
write_atomic(archive, buffer.getvalue())
members = {}
with ZipFile(buffer) as zf:
+23
View File
@@ -200,6 +200,29 @@ def test_tampered_download_installs_nothing(tmp_path, pinned, tamper):
assert not data_dir.exists() or list(data_dir.iterdir()) == []
def test_a_kept_archive_installs_without_downloading(tmp_path, pinned):
archive = tmp_path / "cache" / "model.zip"
ensure_fpgen_model(tmp_path / "first", archive=archive)
ensure_fpgen_model(tmp_path / "second", archive=archive)
assert len(pinned) == 1
assert archive.read_bytes() == pinned.payload
for name, data in MEMBERS.items():
assert (tmp_path / "second" / name).read_bytes() == data
def test_a_tampered_kept_archive_installs_nothing(tmp_path, pinned):
archive = tmp_path / "model.zip"
archive.write_bytes(pinned.payload[:-1])
data_dir = tmp_path / "data"
with pytest.raises(CorruptedDownload):
ensure_fpgen_model(data_dir, archive=archive)
assert pinned == []
assert not data_dir.exists()
def test_member_hashes_are_checked_even_when_the_archive_hash_matches(tmp_path, pinned):
bad = _archive({**MEMBERS, "values.dat.zst": b"x"})
pinned.payload = bad