mirror of
https://github.com/daijro/camoufox.git
synced 2026-11-19 00:07:28 +00:00
Cache the pinned fpgen model archive across CI runs
Nine jobs downloaded the fpgen model from its GitHub release on every run, so a 504 from GitHub failed pythonlib and skipped every browser job behind it. pin-fpgen-model.py --archive keeps the downloaded archive in a file and installs from it when it is there, checked against the pin either way. The fpgen-model action caches that file, keyed on fpgen-model.json, and every job installs the model through it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2ac91c0dc4
commit
b8db95563b
@@ -10,10 +10,12 @@ Usage:
|
||||
python3 python/scripts/pin-fpgen-model.py # install if not already pinned
|
||||
python3 python/scripts/pin-fpgen-model.py --check # verify only; non-zero if not pinned
|
||||
python3 python/scripts/pin-fpgen-model.py --force # re-download and reinstall
|
||||
python3 python/scripts/pin-fpgen-model.py --archive F # keep the archive in F, and install from F when it is there
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from camoufox.fpgen_model import PIN, ensure_fpgen_model, fpgen_data_dir, is_pinned
|
||||
@@ -26,6 +28,9 @@ def main():
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--check', action='store_true', help='verify only; non-zero if not pinned')
|
||||
ap.add_argument('--force', action='store_true', help='reinstall even if already pinned')
|
||||
ap.add_argument('--archive', type=Path,
|
||||
help='keep the downloaded archive in this file, and install from it when it is '
|
||||
'there (still checked against the pin), so CI can cache it')
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.check:
|
||||
@@ -36,7 +41,7 @@ def main():
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
ensure_fpgen_model(force=args.force)
|
||||
ensure_fpgen_model(force=args.force, archive=args.archive)
|
||||
print(f'OK: fpgen model {PIN["tag"]} pinned in {fpgen_data_dir()}', file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from types import ModuleType
|
||||
@@ -83,10 +84,13 @@ def is_pinned(data_dir: Optional[Path] = None) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> None:
|
||||
def ensure_fpgen_model(
|
||||
data_dir: Optional[Path] = None, force: bool = False, archive: Optional[Path] = None
|
||||
) -> None:
|
||||
"""Install the pinned model unless it is already there, and keep fpgen from
|
||||
replacing it. Hashing the installed files takes ~2 ms, and catches a model
|
||||
fpgen fetched over an earlier install."""
|
||||
fpgen fetched over an earlier install. `archive` is a file the downloaded
|
||||
archive is kept in and installed from, checked against the pin either way."""
|
||||
if os.getenv(CUSTOM_MODEL_ENV):
|
||||
return
|
||||
data_dir = data_dir or fpgen_data_dir()
|
||||
@@ -103,7 +107,7 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) ->
|
||||
)
|
||||
try:
|
||||
if force or not is_pinned(data_dir):
|
||||
_install(data_dir)
|
||||
_install(data_dir, archive)
|
||||
_drop_foreign_decompressed(data_dir)
|
||||
_stamp(data_dir)
|
||||
except PermissionError as e:
|
||||
@@ -116,10 +120,17 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) ->
|
||||
) from e
|
||||
|
||||
|
||||
def _install(data_dir: Path) -> None:
|
||||
print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr)
|
||||
buffer = webdl(PIN['url'], progress_callback=lambda done, total: None)
|
||||
verify_sha256(buffer, PIN['sha256'], 'fpgen model')
|
||||
def _install(data_dir: Path, archive: Optional[Path] = None) -> None:
|
||||
kept = archive is not None and archive.exists()
|
||||
if kept:
|
||||
buffer = BytesIO(archive.read_bytes())
|
||||
else:
|
||||
print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr)
|
||||
buffer = webdl(PIN['url'], progress_callback=lambda done, total: None)
|
||||
verify_sha256(buffer, PIN['sha256'], f'fpgen model {archive}' if kept else 'fpgen model')
|
||||
if archive and not kept:
|
||||
archive.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_atomic(archive, buffer.getvalue())
|
||||
|
||||
members = {}
|
||||
with ZipFile(buffer) as zf:
|
||||
|
||||
@@ -200,6 +200,29 @@ def test_tampered_download_installs_nothing(tmp_path, pinned, tamper):
|
||||
assert not data_dir.exists() or list(data_dir.iterdir()) == []
|
||||
|
||||
|
||||
def test_a_kept_archive_installs_without_downloading(tmp_path, pinned):
|
||||
archive = tmp_path / "cache" / "model.zip"
|
||||
ensure_fpgen_model(tmp_path / "first", archive=archive)
|
||||
ensure_fpgen_model(tmp_path / "second", archive=archive)
|
||||
|
||||
assert len(pinned) == 1
|
||||
assert archive.read_bytes() == pinned.payload
|
||||
for name, data in MEMBERS.items():
|
||||
assert (tmp_path / "second" / name).read_bytes() == data
|
||||
|
||||
|
||||
def test_a_tampered_kept_archive_installs_nothing(tmp_path, pinned):
|
||||
archive = tmp_path / "model.zip"
|
||||
archive.write_bytes(pinned.payload[:-1])
|
||||
data_dir = tmp_path / "data"
|
||||
|
||||
with pytest.raises(CorruptedDownload):
|
||||
ensure_fpgen_model(data_dir, archive=archive)
|
||||
|
||||
assert pinned == []
|
||||
assert not data_dir.exists()
|
||||
|
||||
|
||||
def test_member_hashes_are_checked_even_when_the_archive_hash_matches(tmp_path, pinned):
|
||||
bad = _archive({**MEMBERS, "values.dat.zst": b"x"})
|
||||
pinned.payload = bad
|
||||
|
||||
Reference in New Issue
Block a user