diff --git a/.github/actions/fpgen-model/action.yml b/.github/actions/fpgen-model/action.yml new file mode 100644 index 00000000..1b436e54 --- /dev/null +++ b/.github/actions/fpgen-model/action.yml @@ -0,0 +1,28 @@ +name: Install the fpgen model +description: >- + Install the fpgen model pinned by python/src/camoufox/fpgen-model.json, from + an archive cached across runs, so a job does not depend on GitHub serving the + release (a 504 there failed pythonlib and skipped every browser job). The + archive is checked against the pin whether it was cached or downloaded. + Left to itself, fpgen downloads its model on first import with TLS + verification off and no checksum, and only ever the April-2025 release + (python/scripts/pin-fpgen-model.py), so every job that installs pythonlib + runs this before anything imports fpgen. + +inputs: + python: + description: The interpreter camoufox is installed in + required: false + default: python3 + +runs: + using: composite + steps: + - uses: actions/cache@v4 + with: + path: .ci-work/fpgen-model.zip + key: fpgen-archive-${{ hashFiles('python/src/camoufox/fpgen-model.json') }} + - shell: bash + run: >- + ${{ inputs.python }} "$GITHUB_WORKSPACE/python/scripts/pin-fpgen-model.py" + --archive "$GITHUB_WORKSPACE/.ci-work/fpgen-model.zip" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 542f8949..8cc94496 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -270,11 +270,7 @@ jobs: # assert against it. It is a pure-Python install; no browser involved. - run: | pip install -r ci/requirements.txt pytest -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - name: Synthesized input goes through one chokepoint run: python3 browser/scripts/check-input-dispatch.py @@ -353,11 +349,7 @@ jobs: # The geoip extra: python/tests/test_geoip_sources.py downloads every GeoIP # source and fails when one stops publishing, which gates releases. pip install -r ci/requirements.txt pytest -e 'python[geoip]' - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - name: Run # pythonlib resolves the published release through the GitHub API # (pkgman.py honours GITHUB_TOKEN). Unauthenticated, a runner shares the @@ -405,11 +397,9 @@ jobs: # fixtures from its pythonlib before the suite runs. python3 -m venv .venv .venv/bin/pip install -r ci/requirements.txt -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - .venv/bin/python python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model + with: + python: .venv/bin/python - name: Test prerequisites # Everything tests/prereq.ts may ask for. In CI a missing prerequisite # FAILS its tests rather than skipping them, so this list is the job's @@ -780,11 +770,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | pip install -r ci/requirements.txt -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - name: Download # Same GitHub API path as the pythonlib job above, and the same anonymous @@ -929,11 +915,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | pip install -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model # Fonts and properties.json are staged into the artifact by the build job; # `make stage-fonts` here would find no source tree and do nothing. # ipv4-only-network.py builds its test network in an unprivileged user @@ -989,13 +971,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | cd browser/tests/build-tester && npm install && pip install -r requirements.txt - # build-tester pulls pythonlib -- and so fpgen -- through its own - # requirements.txt (`-e ../../../python`) rather than `pip install -e - # python`, so the pin the other jobs get by matching that line has - # to be spelled out here. Without it this job was still downloading - # fpgen's model itself: TLS verification off, no checksum, and only - # ever the April-2025 release (observed in run 36050915401). - python3 "$GITHUB_WORKSPACE/python/scripts/pin-fpgen-model.py" + # build-tester installs pythonlib through its own requirements.txt + # (`-e ../../../python`), and still needs the pin before fpgen loads + # (observed downloading the April-2025 model in run 36050915401). + - uses: ./.github/actions/fpgen-model - run: xvfb-run -a python3 -m ci.run_build_tester --binary "$CAMOUFOX_BINARY" - uses: actions/upload-artifact@v4 if: always() @@ -1035,11 +1014,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | pip install -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - name: Run # Launches browsers, kills them, and proves nothing survived -- the # failure a long-running scraper hits after six hours and no Playwright @@ -1102,11 +1077,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | pip install -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - uses: pnpm/action-setup@v4 with: package_json_file: typescript/package.json @@ -1163,11 +1134,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | pip install -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - run: xvfb-run -a python3 -m ci.run_native --subset growth --shard "${{ matrix.shard }}" --binary "$CAMOUFOX_BINARY" - uses: actions/upload-artifact@v4 if: always() @@ -1196,11 +1163,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: | pip install -e python - # fpgen downloads its model on first import with TLS verification - # OFF and no checksum, and its release picker can only ever reach the - # April-2025 model. Install the pinned one first: see - # python/scripts/pin-fpgen-model.py. - python3 python/scripts/pin-fpgen-model.py + - uses: ./.github/actions/fpgen-model - name: Run # Exits 0 with a SKIP result if sundial itself is unreachable -- the # browser was never measured, so neither a pass nor a failure would be diff --git a/python/scripts/pin-fpgen-model.py b/python/scripts/pin-fpgen-model.py index da6ac893..914aa0b7 100755 --- a/python/scripts/pin-fpgen-model.py +++ b/python/scripts/pin-fpgen-model.py @@ -10,10 +10,12 @@ Usage: python3 python/scripts/pin-fpgen-model.py # install if not already pinned python3 python/scripts/pin-fpgen-model.py --check # verify only; non-zero if not pinned python3 python/scripts/pin-fpgen-model.py --force # re-download and reinstall + python3 python/scripts/pin-fpgen-model.py --archive F # keep the archive in F, and install from F when it is there """ import argparse import sys +from pathlib import Path try: from camoufox.fpgen_model import PIN, ensure_fpgen_model, fpgen_data_dir, is_pinned @@ -26,6 +28,9 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--check', action='store_true', help='verify only; non-zero if not pinned') ap.add_argument('--force', action='store_true', help='reinstall even if already pinned') + ap.add_argument('--archive', type=Path, + help='keep the downloaded archive in this file, and install from it when it is ' + 'there (still checked against the pin), so CI can cache it') args = ap.parse_args() if args.check: @@ -36,7 +41,7 @@ def main(): file=sys.stderr) return 1 - ensure_fpgen_model(force=args.force) + ensure_fpgen_model(force=args.force, archive=args.archive) print(f'OK: fpgen model {PIN["tag"]} pinned in {fpgen_data_dir()}', file=sys.stderr) return 0 diff --git a/python/src/camoufox/fpgen_model.py b/python/src/camoufox/fpgen_model.py index f7cb3bee..cb6d5ff4 100644 --- a/python/src/camoufox/fpgen_model.py +++ b/python/src/camoufox/fpgen_model.py @@ -19,6 +19,7 @@ import json import os import sys import time +from io import BytesIO from pathlib import Path from threading import Lock from types import ModuleType @@ -83,10 +84,13 @@ def is_pinned(data_dir: Optional[Path] = None) -> bool: ) -def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> None: +def ensure_fpgen_model( + data_dir: Optional[Path] = None, force: bool = False, archive: Optional[Path] = None +) -> None: """Install the pinned model unless it is already there, and keep fpgen from replacing it. Hashing the installed files takes ~2 ms, and catches a model - fpgen fetched over an earlier install.""" + fpgen fetched over an earlier install. `archive` is a file the downloaded + archive is kept in and installed from, checked against the pin either way.""" if os.getenv(CUSTOM_MODEL_ENV): return data_dir = data_dir or fpgen_data_dir() @@ -103,7 +107,7 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> ) try: if force or not is_pinned(data_dir): - _install(data_dir) + _install(data_dir, archive) _drop_foreign_decompressed(data_dir) _stamp(data_dir) except PermissionError as e: @@ -116,10 +120,17 @@ def ensure_fpgen_model(data_dir: Optional[Path] = None, force: bool = False) -> ) from e -def _install(data_dir: Path) -> None: - print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr) - buffer = webdl(PIN['url'], progress_callback=lambda done, total: None) - verify_sha256(buffer, PIN['sha256'], 'fpgen model') +def _install(data_dir: Path, archive: Optional[Path] = None) -> None: + kept = archive is not None and archive.exists() + if kept: + buffer = BytesIO(archive.read_bytes()) + else: + print(f"Downloading fpgen model {PIN['tag']}: {PIN['url']}", file=sys.stderr) + buffer = webdl(PIN['url'], progress_callback=lambda done, total: None) + verify_sha256(buffer, PIN['sha256'], f'fpgen model {archive}' if kept else 'fpgen model') + if archive and not kept: + archive.parent.mkdir(parents=True, exist_ok=True) + write_atomic(archive, buffer.getvalue()) members = {} with ZipFile(buffer) as zf: diff --git a/python/tests/test_fpgen_model.py b/python/tests/test_fpgen_model.py index 44d591d4..d7fd19e9 100644 --- a/python/tests/test_fpgen_model.py +++ b/python/tests/test_fpgen_model.py @@ -200,6 +200,29 @@ def test_tampered_download_installs_nothing(tmp_path, pinned, tamper): assert not data_dir.exists() or list(data_dir.iterdir()) == [] +def test_a_kept_archive_installs_without_downloading(tmp_path, pinned): + archive = tmp_path / "cache" / "model.zip" + ensure_fpgen_model(tmp_path / "first", archive=archive) + ensure_fpgen_model(tmp_path / "second", archive=archive) + + assert len(pinned) == 1 + assert archive.read_bytes() == pinned.payload + for name, data in MEMBERS.items(): + assert (tmp_path / "second" / name).read_bytes() == data + + +def test_a_tampered_kept_archive_installs_nothing(tmp_path, pinned): + archive = tmp_path / "model.zip" + archive.write_bytes(pinned.payload[:-1]) + data_dir = tmp_path / "data" + + with pytest.raises(CorruptedDownload): + ensure_fpgen_model(data_dir, archive=archive) + + assert pinned == [] + assert not data_dir.exists() + + def test_member_hashes_are_checked_even_when_the_archive_hash_matches(tmp_path, pinned): bad = _archive({**MEMBERS, "values.dat.zst": b"x"}) pinned.payload = bad