diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index fb43c76..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: Build and Release - -on: - workflow_dispatch: - push: - tags: - - "*" - # The font bundle ships as its own release, in a separate tag namespace so - # it does not appear among the browser downloads. Those tags must not - # trigger a browser build (see scripts/fetch-fonts.py). - - "!font-bundle-*" - -permissions: {} - -env: - # The version tests.yml builds and tests with, so a release is compiled by the - # same interpreter every pull request's build job already exercised. - PYTHON_VERSION: "3.12" - -jobs: - build: - runs-on: ubuntu-24.04 - permissions: - contents: read - env: - # `make dir` (inside ci.run_prepare) writes the mozconfig and adds the Rust - # targets from BUILD_TARGET, defaulting to macos,arm64 when it is unset. - # multibuild.py sets the same value again before building. - BUILD_TARGET: ${{ matrix.target }},${{ matrix.arch }} - strategy: - matrix: - target: [linux, windows, macos] - arch: [x86_64, arm64, i686] - exclude: - # Fails (.mozbuild does not include clang++-cl) - - target: windows - arch: arm64 - # Unsupported - - target: macos - arch: i686 - - target: linux - arch: i686 - - steps: - - name: Maximize build space - uses: AdityaGarg8/remove-unwanted-software@8831c82abf29b34eb2caac48d5f999ecfc0d8eef # v4.1 - with: - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - remove-docker-images: "true" - remove-cached-tools: "true" - remove-swapfile: "true" - verbose: "false" - - - name: Remove unwanted tools - # Originally from here: https://github.com/AdityaGarg8/remove-unwanted-software/blob/master/action.yml - run: | - sudo apt-get remove -y '^aspnetcore-.*' > /dev/null - sudo apt-get remove -y '^dotnet-.*' > /dev/null - sudo apt-get remove -y '^llvm-.*' > /dev/null - sudo apt-get remove -y 'php.*' > /dev/null - sudo apt-get remove -y '^mongodb-.*' > /dev/null - sudo apt-get remove -y '^mysql-.*' > /dev/null - sudo apt-get remove -y azure-cli google-chrome-stable firefox ${POWERSHELL} mono-devel libgl1-mesa-dri --fix-missing > /dev/null - if [[ (${CODENAME} = focal) || (${CODENAME} = jammy) ]]; then - sudo apt-get remove -y google-cloud-sdk --fix-missing > /dev/null - sudo apt-get remove -y google-cloud-cli --fix-missing > /dev/null - fi - sudo apt-get autoremove -y > /dev/null - sudo apt-get clean > /dev/null - - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Set up LLVM - run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 18 - sudo apt-get install -y lld-18 clang-18 - if [ "${{ matrix.arch }}" != "x86_64" ]; then - sudo apt-get install -y libc6-i386 lib32gcc-s1 lib32stdc++6 gcc-multilib g++-multilib - fi - sudo update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-18 100 - - - name: Check disk space - run: df -h - - - name: Install dependencies - run: | - sudo apt-get update - # ccache: the mozconfig enables --with-ccache, but the runner image no - # longer ships it, so configure fails with "Cannot find ccache". - # fontconfig: scripts/verify-fonts.py resolves every reportable family - # through fc-list/fc-match, so the bundle is checked with the same tool - # the browser uses rather than assumed correct. - sudo apt-get install -y msitools p7zip-full aria2 ccache fontconfig - - - name: Fetch the font bundle - # The bundle is a release asset, not repo content (~2.16 GB extracted, - # 843 MB as .tar.xz -- see scripts/fetch-fonts.py). Without this the - # package would ship with NO fonts while pythonlib/camoufox/fonts.json - # still reports hundreds of families, which is a reverse leak in the - # shipped browser. After the dependency install, so the download uses - # aria2c's parallel connections rather than the curl fallback. - run: make fonts-extract - - - name: Verify the font bundle matches the manifest - # Full run, not --quick: this is the release path, and the one invariant - # that cannot be recovered after publishing is a family fonts.json - # reports that the packaged fontconfig cannot resolve. - run: python3 scripts/verify-fonts.py - - - name: Prepare the source tree - # setup-minimal (which fetches the tarball) -> dir -> mozbootstrap, with - # the two network-bound steps retried on transient failures, exactly as - # the tests.yml build job does. multibuild.py then finds _READY and - # builds without re-patching. - run: python3 -m ci.run_prepare - - - name: Create swap space - run: | - sudo fallocate -l 24G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - free -h - df -h / - - - name: Build - env: - # Cap Rust parallelism to lower peak memory (avoids OOM/SIGTERM). - # Tune to taste: higher = faster but more RAM. - CARGO_BUILD_JOBS: "1" - run: python3 ./multibuild.py --target ${{ matrix.target }} --arch ${{ matrix.arch }} - - - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: CamoufoxBuilds-${{ matrix.target }}-${{ matrix.arch }} - path: dist/* - - release: - needs: build - permissions: - contents: write - runs-on: ubuntu-latest - - steps: - - name: Download all artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: artifacts - - - name: Create Release - uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 - if: startsWith(github.ref, 'refs/tags/') - with: - files: artifacts/**/* - generate_release_notes: true - draft: true - prerelease: true diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml deleted file mode 100644 index 8f78629..0000000 --- a/.github/workflows/publish-npm.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Publish to npm - -# The npm half of a launcher release. The two packages ship together, at one -# version (scripts/check-pack.mjs refuses a package.json that differs from -# pythonlib's), and "Publish to pypi" is the one place a release starts: -# -# 1. publish-pypi.yml calls this workflow as a dry run -- every check, the -# build, the pack check and `npm publish --dry-run` -- so a broken npm -# package stops the release before anything is uploaded anywhere; -# 2. it uploads to PyPI; -# 3. its success triggers this workflow again (workflow_run), which publishes -# the same commit to npm. -# -# Started by hand, this workflow only retries step 3 -- after an npm outage, -# say -- and refuses unless PyPI already has this version, so npm can never get -# a release PyPI does not. -# -# Authentication is npm trusted publishing (OIDC): no token is stored anywhere. -# npm accepts this workflow's identity because the package's settings on -# npmjs.com name this repository and this file (publish-npm.yml). Renaming the -# file, or publishing from a fork, is rejected by the registry. The first -# version of a new package has to be published by hand, since the trusted -# publisher is configured on a package that already exists. - -on: - workflow_run: - workflows: ["Publish to pypi"] - types: [completed] - workflow_call: - inputs: - dry_run: - type: boolean - default: true - workflow_dispatch: - inputs: - dry_run: - description: "Run every check and `npm publish --dry-run`, upload nothing" - type: boolean - default: false - -permissions: - contents: read - -jobs: - publish: - # After a PyPI run, only a successful one. - if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # trusted publishing, and the provenance attestation - defaults: - run: - working-directory: typescript - env: - CAMOUFOX_FPGEN_DATA: ${{ github.workspace }}/.fpgen - - steps: - - name: Check out repository - uses: actions/checkout@v6 - with: - # The commit PyPI was released from, not whatever main is now. - ref: ${{ github.event.workflow_run.head_sha || github.sha }} - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.14" - - - name: Install pythonlib - # tests/golden-setup.ts records the golden fixtures from it, and the - # build copies its data files into the package. - working-directory: . - run: | - python3 -m venv .venv - .venv/bin/pip install -r ci/requirements.txt -e pythonlib - .venv/bin/python scripts/pin-fpgen-model.py - - - name: Set up pnpm - uses: pnpm/action-setup@v4 - with: - package_json_file: typescript/package.json - - - name: Set up Node - uses: actions/setup-node@v6 - with: - # Trusted publishing needs npm >= 11.5.1, which Node 24 ships. - node-version: "24" - registry-url: https://registry.npmjs.org - cache: pnpm - cache-dependency-path: typescript/pnpm-lock.yaml - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Type check and lint - run: | - pnpm typecheck - pnpm check - - - name: Test - run: pnpm test - - - name: Build package - run: pnpm build - - - name: Check package - run: node scripts/check-pack.mjs - - - name: PyPI has this version - # npm never gets a release PyPI does not have. Retried for a few minutes: - # straight after the upload, PyPI's JSON API can lag. - if: ${{ !inputs.dry_run }} - run: | - version=$(node -p "require('./package.json').version") - for attempt in $(seq 1 20); do - if curl -fsS -o /dev/null "https://pypi.org/pypi/camoufox/$version/json"; then - echo "camoufox $version is on PyPI" - exit 0 - fi - sleep 15 - done - echo "::error::camoufox $version is not on PyPI. Release through 'Publish to pypi', which publishes to npm after it." - exit 1 - - - name: Publish to npm - # A dry run uploads nothing, so it has nothing to attest. - run: npm publish --access public ${{ inputs.dry_run && '--dry-run --provenance=false' || '' }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml deleted file mode 100644 index d704133..0000000 --- a/.github/workflows/publish-pypi.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Publish to pypi - -on: - workflow_dispatch: - -permissions: - contents: read - -# The one place a launcher release starts. The npm package is checked first -- -# as a dry run of publish-npm.yml -- so a broken one stops the release before -# PyPI has it; PyPI's success then triggers publish-npm.yml to publish the same -# commit to npm (see the comment there). - -jobs: - npm-preflight: - permissions: - contents: read - id-token: write # publish-npm.yml's job declares it; a dry run does not use it - uses: ./.github/workflows/publish-npm.yml - with: - dry_run: true - - publish: - needs: npm-preflight - runs-on: ubuntu-latest - defaults: - run: - working-directory: pythonlib - - steps: - - name: Check out repository - uses: actions/checkout@v6 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install publishing tools - run: pip install vermin build twine - - - name: Check Python compatibility - run: vermin . --eval-annotations --target=3.8 --violations camoufox/ || exit 1 - - - name: Build package - run: python -m build - - - name: Check distribution - run: twine check dist/* - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_NON_INTERACTIVE: "1" - run: twine upload dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9ed0467 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,463 @@ +name: Release + +# Every release of the browser, the Python package and the npm package, from one +# workflow. Two ways in: +# +# push to main (prerelease) the test pipeline runs on the pushed commit; +# if it passes, a browser prerelease is built when the browser's +# sources changed, and -- when anything a library ships changed +# -- camoufox bN goes to PyPI (pip ignores it without +# --pre) and -beta.N to npm under the `next` dist-tag. +# tag vX.Y.Z (stable) on a main commit whose tests passed: the +# browser prerelease built from that commit's sources becomes +# the stable, latest release (no rebuild), and X.Y.Z goes to +# PyPI and to npm `latest`. +# +# Each package is stamped with the browser release built from the same sources +# (ci/release.py stamp -> pythonlib/camoufox/browser-pin.json), and both +# launchers fetch and launch exactly that build by default. A browser release is +# found by the source digest in its manifest.json asset (ci.browser_inputs). +# +# Packages are built once, in build-library, and the publish jobs upload exactly +# those files. PyPI and npm both use trusted publishing (OIDC): no token is +# stored, and each registry accepts uploads only from this file in this +# repository. A failed publish is retried with "Re-run failed jobs": every +# version and tag comes from `plan`, so a re-run publishes the same release. +# +# Branch protection makes main unwritable; nothing here commits to it. A browser +# release's number lives only in its tag, which points at the tested main commit +# (see `ci.release set-build`). Every published library version is tagged too +# (vX.Y.Z by the maintainer, vX.Y.ZbN by tag-library). + +on: + push: + branches: [main] + # vX.Y.Z only. Browser tags (v156.0.1-beta.33) and library prerelease tags + # (v0.5.8b2) are created by this workflow's token, which starts no run, and + # check-promotable refuses anything else a person pushes. + tags: ["v[0-9]+.[0-9]+.[0-9]+"] + +permissions: {} + +# One run at a time per channel. Browser release numbers are allocated in `plan`, +# so two prerelease runs must never overlap; GitHub keeps only the newest pending +# run in a group, so a burst of merges releases the last of them, which contains +# the others. A promotion never waits on a prerelease. +concurrency: + group: release-${{ github.ref_type }} + cancel-in-progress: false + +env: + # The version tests.yml builds and tests with, so a release is compiled by the + # same interpreter every pull request's build job already exercised. + PYTHON_VERSION: "3.12" + +jobs: + # --------------------------------------------------------------------------- + tests: + # The pushed commit, exactly -- not whatever main is by the time a job checks out. + if: github.ref_type == 'branch' + permissions: + contents: read + pull-requests: write # tests.yml's summary job declares it; it comments on pull requests only + uses: ./.github/workflows/tests.yml + with: + ref: ${{ github.sha }} + secrets: inherit + + # --------------------------------------------------------------------------- + plan: + needs: tests + # A tag skips `tests`: check-promotable requires that they passed on its commit. + if: >- + !cancelled() && + (github.ref_type == 'tag' || needs.tests.result == 'success') + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read # check-promotable: the test gate on the tagged commit + outputs: + channel: ${{ steps.channel.outputs.channel }} + build_browser: ${{ steps.browser.outputs.build || 'false' }} + browser_tag: ${{ steps.browser.outputs.tag || steps.paired.outputs.browser_tag }} + digest: ${{ steps.browser.outputs.digest }} + publish_library: ${{ steps.library.outputs.publish }} + py_version: ${{ steps.library.outputs.py_version }} + npm_version: ${{ steps.library.outputs.npm_version }} + dist_tag: ${{ steps.library.outputs.dist_tag }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.sha }} + fetch-depth: 0 # every tag: release numbers, the last library release, main + + - id: channel + run: echo "channel=${{ github.ref_type == 'tag' && 'stable' || 'prerelease' }}" >> "$GITHUB_OUTPUT" + + - name: Only a tested main commit is promoted + if: steps.channel.outputs.channel == 'stable' + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 -m ci.release check-promotable --tag "$GITHUB_REF_NAME" + + - id: paired + name: The browser release built from these sources + if: steps.channel.outputs.channel == 'stable' + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 -m ci.release paired + + - id: browser + name: Build a browser, or reuse the one built from these sources + if: steps.channel.outputs.channel == 'prerelease' + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 -m ci.release browser-plan + + - id: library + name: The library versions, and whether to publish them + run: | + python3 -m ci.release lib-plan --channel "${{ steps.channel.outputs.channel }}" \ + ${{ github.ref_type == 'tag' && format('--tag {0}', github.ref_name) || '' }} + + # --------------------------------------------------------------------------- + build-browser: + needs: plan + if: needs.plan.outputs.build_browser == 'true' + runs-on: ubuntu-24.04 + permissions: + contents: read + env: + # `make dir` (inside ci.run_prepare) writes the mozconfig and adds the Rust + # targets from BUILD_TARGET, defaulting to macos,arm64 when it is unset. + # multibuild.py sets the same value again before building. + BUILD_TARGET: ${{ matrix.target }},${{ matrix.arch }} + strategy: + matrix: + target: [linux, windows, macos] + arch: [x86_64, arm64, i686] + exclude: + # Fails (.mozbuild does not include clang++-cl) + - target: windows + arch: arm64 + # Unsupported + - target: macos + arch: i686 + - target: linux + arch: i686 + + steps: + - name: Maximize build space + uses: AdityaGarg8/remove-unwanted-software@8831c82abf29b34eb2caac48d5f999ecfc0d8eef # v4.1 + with: + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + remove-docker-images: "true" + remove-cached-tools: "true" + remove-swapfile: "true" + verbose: "false" + + - name: Remove unwanted tools + # Originally from here: https://github.com/AdityaGarg8/remove-unwanted-software/blob/master/action.yml + run: | + sudo apt-get remove -y '^aspnetcore-.*' > /dev/null + sudo apt-get remove -y '^dotnet-.*' > /dev/null + sudo apt-get remove -y '^llvm-.*' > /dev/null + sudo apt-get remove -y 'php.*' > /dev/null + sudo apt-get remove -y '^mongodb-.*' > /dev/null + sudo apt-get remove -y '^mysql-.*' > /dev/null + sudo apt-get remove -y azure-cli google-chrome-stable firefox ${POWERSHELL} mono-devel libgl1-mesa-dri --fix-missing > /dev/null + if [[ (${CODENAME} = focal) || (${CODENAME} = jammy) ]]; then + sudo apt-get remove -y google-cloud-sdk --fix-missing > /dev/null + sudo apt-get remove -y google-cloud-cli --fix-missing > /dev/null + fi + sudo apt-get autoremove -y > /dev/null + sudo apt-get clean > /dev/null + + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.sha }} + + - name: Name this release + # The tested commit's upstream.sh names the floor its number was taken + # from; the build takes the number itself from the tag. + run: python3 -m ci.release set-build --tag "${{ needs.plan.outputs.browser_tag }}" + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up LLVM + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo apt-get install -y lld-18 clang-18 + if [ "${{ matrix.arch }}" != "x86_64" ]; then + sudo apt-get install -y libc6-i386 lib32gcc-s1 lib32stdc++6 gcc-multilib g++-multilib + fi + sudo update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-18 100 + + - name: Check disk space + run: df -h + + - name: Install dependencies + run: | + sudo apt-get update + # ccache: the mozconfig enables --with-ccache, but the runner image no + # longer ships it, so configure fails with "Cannot find ccache". + # fontconfig: scripts/verify-fonts.py resolves every reportable family + # through fc-list/fc-match, so the bundle is checked with the same tool + # the browser uses rather than assumed correct. + sudo apt-get install -y msitools p7zip-full aria2 ccache fontconfig + + - name: Fetch the font bundle + # The bundle is a release asset, not repo content (~2.16 GB extracted, + # 843 MB as .tar.xz -- see scripts/fetch-fonts.py). Without this the + # package would ship with NO fonts while pythonlib/camoufox/fonts.json + # still reports hundreds of families, which is a reverse leak in the + # shipped browser. After the dependency install, so the download uses + # aria2c's parallel connections rather than the curl fallback. + run: make fonts-extract + + - name: Verify the font bundle matches the manifest + # Full run, not --quick: this is the release path, and the one invariant + # that cannot be recovered after publishing is a family fonts.json + # reports that the packaged fontconfig cannot resolve. + run: python3 scripts/verify-fonts.py + + - name: Prepare the source tree + # setup-minimal (which fetches the tarball) -> dir -> mozbootstrap, with + # the two network-bound steps retried on transient failures, exactly as + # the tests.yml build job does. multibuild.py then finds _READY and + # builds without re-patching. + run: python3 -m ci.run_prepare + + - name: Create swap space + run: | + sudo fallocate -l 24G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + free -h + df -h / + + - name: Build + env: + # Cap Rust parallelism to lower peak memory (avoids OOM/SIGTERM). + # Tune to taste: higher = faster but more RAM. + CARGO_BUILD_JOBS: "1" + run: python3 ./multibuild.py --target ${{ matrix.target }} --arch ${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: CamoufoxBuilds-${{ matrix.target }}-${{ matrix.arch }} + path: dist/* + + publish-browser: + needs: [plan, build-browser] + runs-on: ubuntu-24.04 + permissions: + contents: write # the release and its tag + id-token: write # build provenance + attestations: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.sha }} + + - name: Download the builds + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: CamoufoxBuilds-* + path: artifacts + + - name: Attest where the builds came from + # `gh attestation verify --repo daijro/camoufox` proves a download + # was built by this workflow from this commit. + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: artifacts/**/* + + - name: Publish the prerelease + # Not a draft: the library release that follows pairs with it through its + # manifest.json (ci.release paired). `--target` creates the tag on the + # tested commit. + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.plan.outputs.browser_tag }} + run: | + python3 -m ci.release manifest --tag "$TAG" \ + --digest "${{ needs.plan.outputs.digest }}" --commit "$GITHUB_SHA" > manifest.json + cat manifest.json + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --target "$GITHUB_SHA" \ + --prerelease --latest=false --title "$TAG" --generate-notes \ + $(find artifacts -type f) manifest.json + + # --------------------------------------------------------------------------- + build-library: + # Built and checked once, before anything is published or promoted; the + # publish jobs upload exactly these files. + needs: plan + if: needs.plan.outputs.publish_library == 'true' + runs-on: ubuntu-24.04 + permissions: + contents: read + env: + CAMOUFOX_FPGEN_DATA: ${{ github.workspace }}/.fpgen + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.sha }} + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install tools and pythonlib + # The npm build copies pythonlib's data files and the pinned fpgen model. + run: | + python3 -m venv .venv + .venv/bin/pip install vermin build twine -r ci/requirements.txt -e pythonlib + .venv/bin/python scripts/pin-fpgen-model.py + + - name: Check Python compatibility + # The package must run on the floor pyproject.toml declares (python = + # "^3.10"); `3.10-` means "needs at most 3.10". typing_extensions is a + # dependency, not a stdlib module for vermin to date. + working-directory: pythonlib + run: ../.venv/bin/vermin . --eval-annotations --backport typing_extensions --target=3.10- --violations camoufox/ + + - name: Stamp the version and the paired browser + run: | + python3 -m ci.release stamp \ + --browser-tag "${{ needs.plan.outputs.browser_tag }}" \ + --py-version "${{ needs.plan.outputs.py_version }}" \ + --npm-version "${{ needs.plan.outputs.npm_version }}" + cat pythonlib/camoufox/browser-pin.json + + - name: Build the Python package + working-directory: pythonlib + run: | + ../.venv/bin/python -m build + ../.venv/bin/twine check dist/* + + - name: The wheel carries its browser pin + working-directory: pythonlib + run: | + ../.venv/bin/python - <<'EOF' + import glob, json, zipfile + wheel = glob.glob("dist/*.whl")[0] + pin = json.loads(zipfile.ZipFile(wheel).read("camoufox/browser-pin.json")) + assert pin["tag"] == "${{ needs.plan.outputs.browser_tag }}", pin + print(f"{wheel} pins {pin['tag']}") + EOF + + - name: Set up pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + with: + package_json_file: typescript/package.json + + - name: Set up Node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: "24" + cache: pnpm + cache-dependency-path: typescript/pnpm-lock.yaml + + - name: Build the npm package + working-directory: typescript + run: | + pnpm install --frozen-lockfile + pnpm build + node scripts/check-pack.mjs + tag=$(node -p "require('./dist/data-files/browser-pin.json').tag") + test "$tag" = "${{ needs.plan.outputs.browser_tag }}" || { + echo "::error::dist pins '$tag', expected ${{ needs.plan.outputs.browser_tag }}"; exit 1; } + mkdir ../npm-dist + npm pack --ignore-scripts --pack-destination ../npm-dist + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: python-dist + path: pythonlib/dist/ + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: npm-dist + path: npm-dist/ + + # --------------------------------------------------------------------------- + promote-browser: + # After both packages are built and checked, so a broken package never + # leaves a promoted browser behind. + needs: [plan, build-library] + if: needs.plan.outputs.channel == 'stable' + runs-on: ubuntu-24.04 + permissions: + contents: write # release flags + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - env: + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + run: python3 -m ci.release promote + + publish-pypi: + # The browser a package pins is published (and, for stable, promoted) first: + # a failed browser build stops the release rather than pinning a tag that + # does not exist. + needs: [plan, build-library, publish-browser, promote-browser] + if: >- + !cancelled() && needs.build-library.result == 'success' && + (needs.plan.outputs.build_browser != 'true' || needs.publish-browser.result == 'success') && + (needs.plan.outputs.channel != 'stable' || needs.promote-browser.result == 'success') + runs-on: ubuntu-24.04 + permissions: + id-token: write # trusted publishing and attestations + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: python-dist + path: dist + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + + publish-npm: + # After PyPI, so npm never has a version PyPI does not. + needs: [plan, publish-pypi] + runs-on: ubuntu-24.04 + permissions: + id-token: write # trusted publishing and provenance + steps: + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + # Trusted publishing needs npm >= 11.5.1, which Node 24 ships. + node-version: "24" + registry-url: https://registry.npmjs.org + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: npm-dist + path: npm-dist + - run: npm publish npm-dist/*.tgz --access public --tag "${{ needs.plan.outputs.dist_tag }}" + + tag-library: + # Every published library version has a tag; the next merge compares against + # it to decide whether the library changed. A stable version's tag is the one + # that started this run. + needs: [plan, publish-npm] + if: needs.plan.outputs.channel == 'prerelease' + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/$GITHUB_REPOSITORY/git/refs" \ + -f ref="refs/tags/v${{ needs.plan.outputs.py_version }}" -f sha="$GITHUB_SHA" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9fc815a..c0dd3ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,10 @@ name: Tests -# The repository's test pipeline. Runs on every pull request, on pushes to main, -# on demand, and -- via workflow_call -- from any workflow that needs to test a -# specific browser version, -# so a contributor's pull request and an automated Firefox bump are judged by -# exactly the same checks. +# The repository's test pipeline. Runs on every pull request, on demand, and -- +# via workflow_call -- from release.yml on every push to main (a release is only +# built from a commit this pipeline passed) and from any workflow that needs to +# test a specific browser version, so a contributor's pull request, a release and +# an automated Firefox bump are judged by exactly the same checks. # # The browser version comes from upstream.sh unless a caller passes one in, # which is what lets one pipeline test both a pull request and a version bump. @@ -42,8 +42,7 @@ name: Tests on: pull_request: - push: - branches: [main] + # Pushes to main are tested by release.yml, which calls this workflow. schedule: # Keeps the ccache alive. GitHub evicts a cache after 7 days unused, and a # cold Firefox build is over an hour; twice a week keeps pull-request builds @@ -119,6 +118,7 @@ jobs: playwright_firefox: ${{ steps.versions.outputs.playwright_firefox }} version_note: ${{ steps.versions.outputs.note }} browser_changed: ${{ steps.scope.outputs.browser_changed }} + browser_tag: ${{ steps.scope.outputs.browser_tag }} has_sundial: ${{ steps.sundial.outputs.has_sundial }} shard_matrix: ${{ steps.shards.outputs.matrix }} steps: @@ -136,10 +136,10 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | # --check-upstream: only suite SELECTION follows browser_version. - # The build reads upstream.sh, and the fetch path downloads whatever - # pythonlib considers current, so a browser_version the branch does not - # pin would compile the OLD browser and judge it against the NEW - # suite -- silently. A real Firefox bump edits upstream.sh, and then + # The build reads upstream.sh, and the fetch path downloads the release + # built from this tree's sources, so a browser_version the branch does + # not pin would test the OLD browser against the NEW suite -- + # silently. A real Firefox bump edits upstream.sh, and then # resolution reads it by default and the two cannot disagree. python3 -m ci.versions --check-upstream \ ${{ inputs.browser_version && format('--browser-version {0}', inputs.browser_version) || '' }} \ @@ -147,6 +147,8 @@ jobs: - name: Does this change the browser? id: scope + env: + GITHUB_TOKEN: ${{ github.token }} # Only a change that can alter the binary justifies compiling one. A # pull request that touches pythonlib/ or ci/ is a driver change: it # still gets the full browser suite, but against the published build its @@ -167,31 +169,21 @@ jobs: echo "::notice::Browser sources changed -- rebuilding from source." exit 0 fi - # The pull request leaves the browser alone -- but the published - # release is only the right browser to test it on if it was built from - # the SAME browser sources as this tree. The patch guards and suites - # come from this checkout, so when the base branch has moved past the - # release (a merged browser change that is not published yet), testing - # the release pairs new guards with an old browser, and every guard for - # the unreleased change fails on a pull request that never touched it. - # Compare against the tag the release was cut from; if they differ, - # build -- which restores the base branch's cached browser when its - # compiled half matches, so it costs minutes, not the full build. - . ./upstream.sh - tag="v${version}-${release}" - if ! git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then - echo "browser_changed=true" >> "$GITHUB_OUTPUT" - echo "::notice::No release tag $tag -- the release this tree targets is not published; building." - exit 0 - fi - ahead=$(git diff --name-only "$tag" HEAD | grep -E "$sources" || true) - if [ -n "$ahead" ]; then - echo "browser sources that differ from $tag:"; echo "$ahead" | sed 's/^/ /' - echo "browser_changed=true" >> "$GITHUB_OUTPUT" - echo "::notice::The base branch's browser sources are ahead of the published $tag ($(echo "$ahead" | wc -l) files) -- building (a cache hit when the base branch already built it)." - else + # The pull request leaves the browser alone -- but a published release + # is only the right browser to test it on if it was built from the + # SAME browser sources as this tree. The patch guards and suites come + # from this checkout, so when the base branch has moved past every + # release (a merged browser change not built yet), testing a release + # pairs new guards with an old browser, and every guard for the + # unbuilt change fails on a pull request that never touched it. So + # test the release built from exactly these sources -- the one this + # tree's library would be paired with (ci/release.py) -- or build. + if python3 -m ci.release paired; then echo "browser_changed=false" >> "$GITHUB_OUTPUT" - echo "::notice::No browser sources differ from the published $tag -- testing against it." + echo "::notice::A published release was built from these browser sources -- testing against it." + else + echo "browser_changed=true" >> "$GITHUB_OUTPUT" + echo "::notice::No published release was built from these browser sources -- building (a cache hit when the base branch already built it)." fi - name: May the stealth check run? @@ -687,6 +679,11 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | set -euo pipefail + # The release built from this tree's sources, installed the way a + # released library installs its paired browser: stamp the pin, fetch. + python3 -m ci.release stamp --browser-tag "${{ needs.resolve.outputs.browser_tag }}" \ + --py-version "$(sed -n 's/^version = "\(.*\)"/\1/p' pythonlib/pyproject.toml)" \ + --npm-version "$(python3 -c "import json; print(json.load(open('typescript/package.json'))['version'])")" python -m camoufox fetch # The ACTIVE build's directory, resolved the way the launcher resolves # it. `camoufox path` prints the cache root, and multiversion installs @@ -694,13 +691,11 @@ jobs: # binary from the root failed every driver-only run since #772. install_dir="$(python -c 'from camoufox.pkgman import camoufox_path; print(camoufox_path(download_if_missing=False))')" echo "install dir: $install_dir" - # Which browser did we actually get? This path does not build, it - # downloads the current release -- correct for a driver change, since - # that is what users run. But the SUITE comes from upstream.sh, and in - # an upgrade window the two part company: upstream.sh moves to the new - # Firefox before any build of it is published, and this would then test - # the old browser against the new suite. Beta drift inside a generation - # is fine; a generation apart is not. + # Which browser did we actually get? This path does not build: it + # downloads the release built from exactly these sources, so its + # Firefox is upstream.sh's by construction (the source digest covers + # the version line). The suite comes from upstream.sh too; assert that + # the two agree rather than assume it. active="$(python -m camoufox active)" echo "fetched: $active" python3 -m ci.versions --check-fetched "$active" \ diff --git a/ci/README.md b/ci/README.md index 5810d97..64db3bc 100644 --- a/ci/README.md +++ b/ci/README.md @@ -555,13 +555,16 @@ Each tier gates the next, so a two-second lint failure never reaches the build: ``` **Driver-only pull requests test the published release, when it matches.** -There is nothing new to compile, so `fetch-browser` downloads the published -release and the browser suites run against the build users are actually on — a -minute instead of seventy. That is only right while the release was built from -this tree's browser sources: once a browser change has merged but not been -released, the guards in the checkout would judge an older browser. So the scope -step compares the browser sources against the release tag, and when they -differ it builds instead, which restores the base branch's cached browser. +There is nothing new to compile, so `fetch-browser` downloads a published +release and the browser suites run against a build users actually get, in a +minute instead of seventy. That is only right when the release was built from +this tree's browser sources. Once a browser change has merged but not been +built, the guards in the checkout would judge an older browser. So the scope +step asks `ci.release paired` for the release built from exactly this tree's +sources, which is the one this tree's library would be paired with (see +[Releases](#releases)). The job then installs it through the same pin a +released package carries. When no release matches, it builds instead, and the +build restores the base branch's cached browser. **Changing Juggler's JavaScript does not rebuild the browser.** Measured on a real build: ccache reported a **98.63%** hit rate, so almost none of those 24 @@ -623,13 +626,89 @@ whole pull request. `ci.run_prepare` runs `setup-minimal` → `dir` → text reads as transient. A failed patch hunk or a compile error still fails on the first attempt — retrying a broken tree only spends a runner to reach the same answer, and a retry loop that swallows a real breakage turns a red build -into a slow red build. The release workflow (`build.yml`) prepares its tree the same -way, so a tagged build gets the same hardening. +into a slow red build. The release workflow (`release.yml`) prepares its tree the same +way, so a release build gets the same hardening. > One consequence of `cancel-in-progress`: pushing to a branch cancels its > running build. That is right while iterating, but a 70-minute build will not > survive a push made 20 minutes in. +## Releases + +One workflow, [`release.yml`](../.github/workflows/release.yml), makes every +release. Every tested merge to `main` is released as a **prerelease**; a +maintainer promotes one to **stable** by pushing a tag. Each +library release is paired with exactly one browser build. + +``` +merge to main ─► tests ─► plan ─┬─► build-browser ─► publish-browser ─┐ + (tests.yml, │ (only if its v156.0.1-beta.N │ + this commit) │ sources changed) ├─► publish-pypi ─► publish-npm ─► tag-library + └─► build-library ─────────────────────┘ 0.5.8bN 0.5.8-beta.N v0.5.8bN + (only if what it ships changed) (pip: --pre) @next + +git tag v0.5.8 && git push origin v0.5.8 + ─► plan ─► build-library ─► promote-browser ─► publish-pypi ─► publish-npm + (checks) paired → stable 0.5.8 0.5.8 @latest +``` + +**Only a tested commit is released.** On a push to `main`, `release.yml` calls +`tests.yml` on the pushed commit and goes on only if it passes; `tests.yml` has no +push trigger of its own. Every later job checks out that same commit. + +**The browser** is built only when the merge changed its sources. +`ci.browser_inputs.source_digest()` hashes everything the browser is built +from, except the release number. Every browser release carries a +`manifest.json` asset with that digest and the commit it was built from; if a +published release already carries this digest, nothing is built. Otherwise +`plan` takes the next unused `beta.N`, never below `upstream.sh`'s `release` and +never reused. Nothing is committed: the tag points at the tested `main` commit, +and `ci.release set-build` writes the number from the tag into the build's +working tree, so rebuilding from the tag reproduces the release. The builds are +attested (`gh attestation verify --repo daijro/camoufox`) and published +as a GitHub prerelease, never a draft. + +Releases cut before manifests existed (up to `v156.0.1-beta.32`) are paired +the old way: the tag `upstream.sh` names, published (a prerelease counts, a +draft does not), with no browser source changed since. + +**The packages** are built once, in `build-library`, and `publish-pypi` and +`publish-npm` upload exactly those files: one version spelled for each +registry, `0.5.8b2` on PyPI and `0.5.8-beta.2` on npm. `pip install camoufox` +ignores prereleases unless `--pre` is passed, and `npm install +@camoufox/camoufox` takes `latest`, not `next`. A prerelease is of the version +in `pyproject.toml` while that version is unreleased, and of the next patch once +it has shipped. A merge that changes nothing a package ships (`pythonlib/`, +`typescript/` or the browser it pins) publishes no library prerelease; the last +published version is found by its tag, `vX.Y.Z` or `vX.Y.ZbN`. + +**The pairing.** `ci.release stamp` writes the browser release built from the +package's own sources into `pythonlib/camoufox/browser-pin.json`, which both +launchers read. By default, `camoufox fetch` installs exactly that build and a +launch uses exactly that build, prerelease or not, and whatever else is +installed or was marked active. A user who explicitly chooses another channel +or build keeps it, with a warning at launch. `camoufox set --release` goes back +to the paired build. On `main` the file is `{}`, so a development checkout +follows its channel as before. + +**Promotion** is a `vX.Y.Z` tag, and it is refused unless: + +- the tagged commit is on `main`; +- `All tests passed` succeeded on it; +- X.Y.Z is newer than every published release; +- a browser release was built from its sources. Wait for that commit's release + run to finish first. + +Both packages are then built and checked. The paired browser release becomes +the stable, latest release, with no rebuild, so users get the binaries that were +tested; the packages are published at X.Y.Z. + +**A failed publish** is retried with *Re-run failed jobs*. Every version and +tag comes from `plan`, so a re-run publishes the same release. + +**Credentials.** None are stored. PyPI and npm both use trusted publishing +(OIDC): each accepts uploads only from `release.yml` in this repository. + ## Running a piece by hand ```bash @@ -649,6 +728,9 @@ python3 -m ci.run_native --subset growth --binary path/to/camoufox-bin python3 -m ci.run_native --subset growth --binary path/to/camoufox-bin --shard 3/7 python3 -m ci.run_sundial --binary path/to/camoufox-bin python3 -m ci.summarize --results-dir .ci-work/results +python3 -m ci.release browser-plan # build a browser, or reuse one +python3 -m ci.release paired # the release built from this tree +python3 -m ci.release lib-plan --channel prerelease ``` Each suite runner writes one result file to `.ci-work/results/` (`run_prepare` diff --git a/ci/browser_inputs.py b/ci/browser_inputs.py index dc448fd..82415a3 100644 --- a/ci/browser_inputs.py +++ b/ci/browser_inputs.py @@ -175,6 +175,36 @@ def native_digest(root: Optional[Path] = None) -> str: return digest.hexdigest()[:32] +def source_inputs(root: Optional[Path] = None) -> List[str]: + """Every file that goes into the browser, compiled or packaged, sorted.""" + root = root or REPO_ROOT + return sorted(set(native_inputs(root)) | resource_sources(root)) + + +def source_digest(root: Optional[Path] = None) -> str: + """A hash of everything the browser is built from, except its release number. + + This is what pairs a library release with a browser release: two commits + with the same source digest produce the same browser, so a library built + from either may name that browser's release. The release number is left + out because the release workflow writes it into the build's working tree + from the release tag (ci/release.py set-build), and that must not make the + browser look different from the main commit it was built from. + """ + root = root or REPO_ROOT + digest = hashlib.sha256() + for rel in source_inputs(root): + data = (root / rel).read_bytes() + if rel == "upstream.sh": + data = b"\n".join( + line for line in data.splitlines() if not line.strip().startswith(b"release=") + ) + digest.update(rel.encode("utf-8")) + digest.update(b"\0") + digest.update(hashlib.sha256(data).digest()) + return digest.hexdigest()[:32] + + def overlay(dist_bin: Path, root: Optional[Path] = None) -> List[str]: """Lay the current resources over an already-built dist/bin. @@ -196,6 +226,7 @@ def overlay(dist_bin: Path, root: Optional[Path] = None) -> List[str]: def main(argv: Optional[List[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--digest", action="store_true") + parser.add_argument("--source-digest", action="store_true") parser.add_argument("--list", action="store_true") parser.add_argument("--resources", action="store_true") parser.add_argument("--overlay", type=Path, metavar="DIST_BIN") @@ -203,6 +234,8 @@ def main(argv: Optional[List[str]] = None) -> int: if args.digest: print(native_digest()) + if args.source_digest: + print(source_digest()) if args.list: for rel in native_inputs(): print(rel) diff --git a/ci/release.py b/ci/release.py new file mode 100644 index 0000000..ce25b7a --- /dev/null +++ b/ci/release.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +"""Release plumbing for .github/workflows/release.yml. + +Every merge to main that passes the test pipeline publishes a prerelease: a +browser build if the browser's sources changed (a GitHub prerelease with its own +release number), then -- if the library changed -- the Python package on PyPI and +the npm package under the `next` dist-tag. Pushing a `vX.Y.Z` tag on a tested main +commit promotes it: the browser release built from that commit's sources becomes +the stable, latest release, and X.Y.Z is published to PyPI and to npm `latest`. + +Each library release is paired with exactly one browser release -- the one built +from the same sources, found by `ci.browser_inputs.source_digest()` in the +release's manifest.json asset -- and `stamp` writes that pairing into the package +(pythonlib/camoufox/browser-pin.json, read by both launchers). A library therefore +never runs a browser it was not released with unless its user explicitly chooses +another. + +A browser release's number is not committed anywhere: its tag points at the +tested main commit, and `set-build` writes the number from the tag name into the +build's working tree. Rebuilding from the tag with `set-build` reproduces it. + + python3 -m ci.release browser-plan # build a new browser, or reuse one + python3 -m ci.release set-build --tag TAG # upstream.sh names TAG's number (working tree only) + python3 -m ci.release manifest --tag T --digest D --commit C # the release's manifest.json + python3 -m ci.release paired [--root DIR] [--releases JSON] # the browser release for this tree + python3 -m ci.release lib-plan --channel prerelease|stable [--tag vX.Y.Z] + python3 -m ci.release stamp --browser-tag T --py-version V --npm-version V + python3 -m ci.release promote # paired browser -> stable, latest + python3 -m ci.release check-promotable --tag vX.Y.Z +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Optional, Sequence, Tuple + +from ._util import REPO_ROOT, die, http_json, log, read_upstream_sh, run, set_output +from .browser_inputs import BROWSER_DIRS, BROWSER_FILES, NON_NATIVE_SCRIPTS, source_digest + +PYPI_PROJECT = "camoufox" +NPM_PACKAGE = "@camoufox/camoufox" +PIN_FILE = REPO_ROOT / "pythonlib" / "camoufox" / "browser-pin.json" +PYPROJECT = REPO_ROOT / "pythonlib" / "pyproject.toml" +PACKAGE_JSON = REPO_ROOT / "typescript" / "package.json" +TS_VERSION = REPO_ROOT / "typescript" / "src" / "__version__.ts" + +# Attached to every browser release; how a library finds its browser. +MANIFEST_ASSET = "manifest.json" + +# The one check the test pipeline reports. Run by release.yml it is named +# "tests / All tests passed"; run on a pull request, "All tests passed". +GATE_CHECK = "All tests passed" + +STABLE_TAG = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") +# Every published library version is tagged: vX.Y.Z, or vX.Y.ZbN for a +# prerelease (PEP 440 spelling, so it never reads as a browser tag). +LIBRARY_TAG = re.compile(r"^v(\d+)\.(\d+)\.(\d+)(?:b(\d+))?$") +# What a library release ships besides the browser it pairs with. +LIBRARY_DIRS = ("pythonlib", "typescript") +_BROWSER_TAG = re.compile(r"^v(?P[^-]+)-(?P[a-z]+)\.(?P\d+)$") + + +# --------------------------------------------------------------------------- +# versions +# --------------------------------------------------------------------------- + +def release_tuple(version: str) -> Tuple[int, int, int]: + m = re.match(r"^(\d+)\.(\d+)\.(\d+)$", version) + if not m: + raise ValueError(f"not a release version: {version!r}") + return int(m[1]), int(m[2]), int(m[3]) + + +def stable_versions(versions: Iterable[str]) -> List[Tuple[int, int, int]]: + out = [] + for v in versions: + try: + out.append(release_tuple(v)) + except ValueError: + continue + return out + + +def prerelease_numbers(base: str, pypi: Iterable[str], npm: Iterable[str]) -> List[int]: + """Prerelease counters already used for `base`, on either registry.""" + found = [] + py = re.compile(rf"^{re.escape(base)}b(\d+)$") + js = re.compile(rf"^{re.escape(base)}-beta\.(\d+)$") + for v in pypi: + if m := py.match(v): + found.append(int(m[1])) + for v in npm: + if m := js.match(v): + found.append(int(m[1])) + return found + + +@dataclass(frozen=True) +class LibVersion: + py: str + npm: str + dist_tag: str + + +def plan_prerelease(checked_in: str, pypi: Sequence[str], npm: Sequence[str]) -> LibVersion: + """The next prerelease: of the checked-in version if it is unreleased, else of the next patch.""" + released = stable_versions(pypi) + stable_versions(npm) + latest = max(released) if released else (0, 0, 0) + base_t = release_tuple(checked_in) + if base_t <= latest: + base_t = (latest[0], latest[1], latest[2] + 1) + base = ".".join(map(str, base_t)) + n = max(prerelease_numbers(base, pypi, npm), default=0) + 1 + return LibVersion(py=f"{base}b{n}", npm=f"{base}-beta.{n}", dist_tag="next") + + +def plan_stable(tag: str, pypi: Sequence[str], npm: Sequence[str]) -> LibVersion: + m = STABLE_TAG.match(tag) + if not m: + raise ValueError(f"{tag!r} is not a release tag (vX.Y.Z)") + version = f"{m[1]}.{m[2]}.{m[3]}" + if version in pypi or version in npm: + raise ValueError(f"{version} is already published") + released = stable_versions(pypi) + stable_versions(npm) + if released and release_tuple(version) <= max(released): + raise ValueError(f"{version} is not newer than the latest release " + f"{'.'.join(map(str, max(released)))}") + return LibVersion(py=version, npm=version, dist_tag="latest") + + +def next_browser_release(upstream_release: str, tags: Iterable[str]) -> str: + """The first unused release number: never below upstream.sh's, never reused. + + Numbers are global across Firefox versions, as they always have been + (beta.30 on 152.0.4, then beta.31 ...), so every tag counts. + """ + m = re.match(r"^([a-z]+)\.(\d+)$", upstream_release) + if not m: + raise ValueError(f"cannot number releases after {upstream_release!r}") + prefix, floor = m[1], int(m[2]) + used = [int(t["n"]) for t in (_BROWSER_TAG.match(x) for x in tags) + if t and t["prefix"] == prefix] + return f"{prefix}.{max([floor] + [u + 1 for u in used])}" + + +def manifest(tag: str, digest: str, commit: str) -> dict: + """What every browser release carries as its manifest.json asset.""" + return {"schema": 1, "tag": tag, "source_digest": digest, "commit": commit} + + +def release_manifest(rel: dict) -> Optional[dict]: + """A release's manifest.json, or None for a release published without one. + + A release dict may carry it pre-fetched under "manifest" (the tests do); + otherwise the asset is downloaded. The repository is public, so no token is + sent -- the download redirects to a host that must not receive it. + """ + if "manifest" in rel: + return rel["manifest"] + asset = next((a for a in rel.get("assets") or [] if a.get("name") == MANIFEST_ASSET), None) + if asset is None: + return None + return http_json(asset["browser_download_url"]) + + +def paired_release(releases: Sequence[dict], digest: str) -> Optional[dict]: + """The newest published browser release built from these sources.""" + for rel in releases: # the API lists newest first + if rel.get("draft"): + continue + found = release_manifest(rel) + if found and found.get("source_digest") == digest: + return rel + return None + + +def library_version_key(tag: str) -> Optional[Tuple[int, int, int, float]]: + """Version order for a library tag: 0.5.7b1 < 0.5.7b2 < 0.5.7 < 0.5.8b1.""" + m = LIBRARY_TAG.match(tag) + if not m: + return None + return int(m[1]), int(m[2]), int(m[3]), float(m[4]) if m[4] else float("inf") + + +def last_library_tag(tags: Iterable[str]) -> Optional[str]: + keyed = [(k, t) for t in tags if (k := library_version_key(t))] + return max(keyed)[1] if keyed else None + + +def is_library_source(rel: str) -> bool: + """Whether a change to `rel` changes what a library release would ship.""" + return rel.split("/", 1)[0] in LIBRARY_DIRS or is_browser_source(rel) + + +def is_browser_source(rel: str) -> bool: + """Whether a repo-relative path goes into the browser (as source_digest counts it).""" + if rel in NON_NATIVE_SCRIPTS: + return False + return rel in BROWSER_FILES or rel.split("/", 1)[0] in BROWSER_DIRS + + +@dataclass(frozen=True) +class Pairing: + release: Optional[dict] + why: str + ahead: Tuple[str, ...] = () + + +def find_paired(releases: Sequence[dict], root: Path = REPO_ROOT) -> Pairing: + """The published browser release built from the sources at `root`, and why. + + A release built by release.yml carries its source digest in its manifest.json, + and a match there is exact. Releases cut before the manifest existed (up to and + including v156.0.1-beta.32) are found the way tests.yml used to find them: + the tag upstream.sh names, published, with no browser source changed since. + Anything else -- a draft, a tag that was never released, or a base branch + whose browser sources moved past every release -- pairs with nothing, and + the caller builds. + """ + digest = source_digest(root) + found = paired_release(releases, digest) + if found: + return Pairing(found, f"{found['tag_name']} carries source digest {digest}") + + up = read_upstream_sh(root / "upstream.sh") + tag = f"v{up['version']}-{up['release']}" + rel = next((r for r in releases if r.get("tag_name") == tag), None) + if rel is None or rel.get("draft"): + return Pairing(None, f"no published release {tag}, and none carries source digest {digest}") + if not run(["git", "rev-parse", "-q", "--verify", f"refs/tags/{tag}"], cwd=root).ok: + return Pairing(None, f"release {tag} is published but its tag is not in this checkout") + diff = run(["git", "diff", "--name-only", tag, "HEAD"], cwd=root, check=True).stdout.split() + ahead = tuple(sorted(f for f in diff if is_browser_source(f))) + if ahead: + return Pairing(None, f"the browser sources differ from {tag} in {len(ahead)} files", ahead) + return Pairing(rel, f"no browser source differs from {tag}") + + +# --------------------------------------------------------------------------- +# I/O +# --------------------------------------------------------------------------- + +def _gh_headers() -> dict: + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + return {"Authorization": f"Bearer {token}"} if token else {} + + +def github_releases(repo: str) -> List[dict]: + out: List[dict] = [] + for page in range(1, 11): + batch = http_json(f"https://api.github.com/repos/{repo}/releases?per_page=100&page={page}", + headers=_gh_headers()) + out += batch + if len(batch) < 100: + break + return out + + +def registry_versions() -> Tuple[List[str], List[str]]: + import urllib.error + + def fetch(url: str, key: str) -> List[str]: + try: + return list(http_json(url).get(key, {})) + except urllib.error.HTTPError as err: + if err.code == 404: # never published + return [] + raise + + return (fetch(f"https://pypi.org/pypi/{PYPI_PROJECT}/json", "releases"), + fetch(f"https://registry.npmjs.org/{NPM_PACKAGE.replace('/', '%2f')}", "versions")) + + +def checked_in_version() -> str: + m = re.search(r'^version = "([^"]+)"', PYPROJECT.read_text(), re.M) + if not m: + die("pythonlib/pyproject.toml has no version") + return m[1] + + +def repo_name() -> str: + return os.environ.get("GITHUB_REPOSITORY") or "daijro/camoufox" + + +# --------------------------------------------------------------------------- +# commands +# --------------------------------------------------------------------------- + +def cmd_browser_plan(_: argparse.Namespace) -> int: + digest = source_digest() + pairing = find_paired(github_releases(repo_name())) + found = pairing.release + set_output("digest", digest) + if found: + log(f"{found['tag_name']} was built from these sources ({pairing.why}); no browser build needed") + set_output("build", "false") + set_output("tag", found["tag_name"]) + return 0 + up = read_upstream_sh() + tags = run(["git", "tag", "-l", "v*"], cwd=REPO_ROOT, check=True).stdout.split() + release = next_browser_release(up["release"], tags) + tag = f"v{up['version']}-{release}" + log(f"browser sources changed: building {tag}") + set_output("build", "true") + set_output("tag", tag) + set_output("release", release) + return 0 + + +def cmd_set_build(args: argparse.Namespace) -> int: + """upstream.sh names TAG's release number -- in the working tree only. + + The tag points at the tested main commit, whose upstream.sh names the floor + the number was allocated from; the build takes the number from the tag. + """ + m = _BROWSER_TAG.match(args.tag) + if not m: + die(f"{args.tag!r} is not a browser release tag") + up = read_upstream_sh() + if m["version"] != up["version"]: + die(f"{args.tag} is Firefox {m['version']}, but upstream.sh builds {up['version']}") + path = REPO_ROOT / "upstream.sh" + path.write_text(re.sub(r"^release=.*$", f"release={m['prefix']}.{m['n']}", path.read_text(), flags=re.M)) + log(f"upstream.sh: release={m['prefix']}.{m['n']}") + return 0 + + +def cmd_manifest(args: argparse.Namespace) -> int: + print(json.dumps(manifest(args.tag, args.digest, args.commit), indent=2)) + return 0 + + +def _releases(args: argparse.Namespace) -> List[dict]: + if getattr(args, "releases", None): + return json.loads(Path(args.releases).read_text(encoding="utf-8")) + return github_releases(repo_name()) + + +def cmd_paired(args: argparse.Namespace) -> int: + pairing = find_paired(_releases(args), Path(args.root).resolve()) + for rel in pairing.ahead: + print(f" {rel}") + found = pairing.release + if not found: + die(f"no browser release was built from these sources: {pairing.why}. " + "If this commit changed the browser, its build has failed or not finished; " + "see that commit's run of the Release workflow.") + log(f"paired with {found['tag_name']}: {pairing.why}") + set_output("browser_tag", found["tag_name"]) + set_output("browser_prerelease", "true" if found.get("prerelease") else "false") + return 0 + + +def cmd_lib_plan(args: argparse.Namespace) -> int: + pypi, npm = registry_versions() + try: + plan = (plan_prerelease(checked_in_version(), pypi, npm) if args.channel == "prerelease" + else plan_stable(args.tag, pypi, npm)) + except ValueError as err: + die(str(err)) + set_output("py_version", plan.py) + set_output("npm_version", plan.npm) + set_output("dist_tag", plan.dist_tag) + publish = True + if args.channel == "prerelease": + publish, why = library_changed(Path(args.root).resolve()) + log(why) + set_output("publish", "true" if publish else "false") + return 0 + + +def library_changed(root: Path = REPO_ROOT) -> Tuple[bool, str]: + """Whether HEAD ships anything the last library release did not. + + A docs- or CI-only merge publishes no library prerelease: it would be the + same package under a new number. + """ + tags = run(["git", "tag", "-l", "v*"], cwd=root, check=True).stdout.split() + last = last_library_tag(tags) + if last is None: + return True, "no library release is tagged yet" + diff = run(["git", "diff", "--name-only", last, "HEAD"], cwd=root, check=True).stdout.split() + changed = sorted(f for f in diff if is_library_source(f)) + if not changed: + return False, f"nothing a library release ships has changed since {last}" + return True, f"{len(changed)} shipped files changed since {last}, e.g. {', '.join(changed[:3])}" + + +def is_gate_check(name: str) -> bool: + """The test pipeline's gate, run directly or called by release.yml.""" + return name == GATE_CHECK or name.endswith(f" / {GATE_CHECK}") + + +def _require_tested(sha: str) -> None: + """The test pipeline's gate check passed on exactly this commit.""" + checks: List[dict] = [] + for page in range(1, 11): + batch = http_json( + f"https://api.github.com/repos/{repo_name()}/commits/{sha}/check-runs" + f"?filter=latest&per_page=100&page={page}", + headers=_gh_headers(), + ).get("check_runs", []) + checks += batch + if len(batch) < 100: + break + if not any(is_gate_check(c.get("name", "")) and c.get("conclusion") == "success" for c in checks): + die(f"'{GATE_CHECK}' has not passed on {sha[:10]}; nothing untested is released") + + +def _head() -> str: + return run(["git", "rev-parse", "HEAD"], cwd=REPO_ROOT, check=True).stdout.strip() + + +def cmd_check_promotable(args: argparse.Namespace) -> int: + """A tag promotes only a main commit on which the test pipeline passed.""" + if not STABLE_TAG.match(args.tag): + die(f"{args.tag!r} is not a release tag (vX.Y.Z); nothing to promote") + sha = _head() + if not run(["git", "merge-base", "--is-ancestor", sha, "origin/main"], cwd=REPO_ROOT).ok: + die(f"{args.tag} points at {sha[:10]}, which is not on main") + _require_tested(sha) + return 0 + + +def cmd_stamp(args: argparse.Namespace) -> int: + m = _BROWSER_TAG.match(args.browser_tag) + if not m: + die(f"{args.browser_tag!r} is not a browser release tag") + PIN_FILE.write_text(json.dumps({ + "tag": args.browser_tag, + "repo": repo_name(), + "repo_name": "Official", + "version": m["version"], + "build": f"{m['prefix']}.{m['n']}", + }, indent=2) + "\n") + PYPROJECT.write_text(re.sub(r'^version = "[^"]+"', f'version = "{args.py_version}"', + PYPROJECT.read_text(), count=1, flags=re.M)) + pkg = json.loads(PACKAGE_JSON.read_text()) + pkg["version"] = args.npm_version + PACKAGE_JSON.write_text(json.dumps(pkg, indent="\t") + "\n") + TS_VERSION.write_text(re.sub(r'(LIBRARY_VERSION\s*=\s*")[^"]+(")', rf"\g<1>{args.npm_version}\g<2>", + TS_VERSION.read_text(), count=1)) + log(f"stamped camoufox {args.py_version} / {NPM_PACKAGE} {args.npm_version} " + f"with browser {args.browser_tag}") + return 0 + + +def cmd_promote(_: argparse.Namespace) -> int: + pairing = find_paired(github_releases(repo_name())) + found = pairing.release + if not found: + die(f"no browser release was built from these sources: {pairing.why}") + tag = found["tag_name"] + if found.get("prerelease"): + run(["gh", "release", "edit", tag, "--repo", repo_name(), "--prerelease=false", "--latest"], + cwd=REPO_ROOT, check=True) + log(f"{tag} is now the stable, latest browser release") + else: + log(f"{tag} is already a stable release") + set_output("browser_tag", tag) + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("browser-plan").set_defaults(fn=cmd_browser_plan) + p = sub.add_parser("set-build"); p.add_argument("--tag", required=True); p.set_defaults(fn=cmd_set_build) + p = sub.add_parser("manifest"); p.add_argument("--tag", required=True); p.add_argument("--digest", required=True) + p.add_argument("--commit", required=True); p.set_defaults(fn=cmd_manifest) + p = sub.add_parser("paired") + p.add_argument("--root", default=str(REPO_ROOT), help="the checkout to pair (default: this one)") + p.add_argument("--releases", metavar="JSON", + help="the releases as the GitHub API lists them, instead of asking it") + p.set_defaults(fn=cmd_paired) + p = sub.add_parser("lib-plan"); p.add_argument("--channel", choices=["prerelease", "stable"], required=True) + p.add_argument("--tag") + p.add_argument("--root", default=str(REPO_ROOT), help="the checkout to compare (default: this one)") + p.set_defaults(fn=cmd_lib_plan) + p = sub.add_parser("check-promotable"); p.add_argument("--tag", required=True) + p.set_defaults(fn=cmd_check_promotable) + p = sub.add_parser("stamp"); p.add_argument("--browser-tag", required=True) + p.add_argument("--py-version", required=True); p.add_argument("--npm-version", required=True) + p.set_defaults(fn=cmd_stamp) + sub.add_parser("promote").set_defaults(fn=cmd_promote) + args = ap.parse_args(argv) + return args.fn(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/run_typescript.py b/ci/run_typescript.py index 8b7d0e9..a1a0dc8 100644 --- a/ci/run_typescript.py +++ b/ci/run_typescript.py @@ -125,7 +125,7 @@ def main(argv: Optional[List[str]] = None) -> int: result.record(f"pnpm {script}", evidence.PASS if proc.ok else evidence.FAIL) # The tarball a user would install: builds, ships every data file, installs - # and imports in an empty project, and its CLI starts. publish-npm.yml runs + # and imports in an empty project, and its CLI starts. release.yml runs # the same check before uploading; running it here means a packaging mistake # is caught on the pull request that makes it, not on release day. if not args.browser: diff --git a/ci/tests/test_ci.py b/ci/tests/test_ci.py index a8304c6..ae467b4 100644 --- a/ci/tests/test_ci.py +++ b/ci/tests/test_ci.py @@ -2632,8 +2632,16 @@ def test_build_tester_accepts_every_core_count_pythonlib_presents(): # --------------------------------------------------------------------------- -def _scope(repo: pathlib.Path, base: str) -> tuple[str, str]: - """Run the workflow's own "Does this change the browser?" step in `repo`.""" +# What the GitHub API says is published, for the fixture below. +PUBLISHED = [{"tag_name": "v1.0-beta.1", "draft": False, "prerelease": True, "body": ""}] + + +def _scope(repo: pathlib.Path, base: str, releases=None) -> tuple[str, str]: + """Run the workflow's own "Does this change the browser?" step in `repo`. + + The step asks `ci.release paired` for the release built from the tree's + sources; here it pairs `repo` against `releases` instead of the live API. + """ import subprocess text = WORKFLOW.read_text(encoding="utf-8") @@ -2643,13 +2651,20 @@ def _scope(repo: pathlib.Path, base: str) -> tuple[str, str]: block = "\n".join(line[10:] if line.startswith(" " * 10) else line.strip() for line in lines) block = block.replace("${{ github.event_name }}", "pull_request") block = block.replace("${{ github.event.pull_request.base.sha }}", base) - out = repo / "out.txt" + listed = repo.parent / "releases.json" + listed.write_text(json.dumps(PUBLISHED if releases is None else releases)) + assert "python3 -m ci.release paired" in block, "the step no longer asks ci.release for the pairing" + block = block.replace( + "python3 -m ci.release paired", + f"python3 -m ci.release paired --root {repo} --releases {listed}", + ) + out = repo.parent / "out.txt" proc = subprocess.run( ["bash", "-e", "-c", block], cwd=repo, capture_output=True, text=True, - env={**os.environ, "GITHUB_OUTPUT": str(out)}, + env={**os.environ, "GITHUB_OUTPUT": str(out), "PYTHONPATH": str(CI_ROOT.parent)}, ) assert proc.returncode == 0, proc.stderr - return out.read_text().strip(), proc.stdout + return out.read_text().strip(), proc.stdout + proc.stderr def _git(repo: pathlib.Path, *args: str) -> str: @@ -2670,6 +2685,8 @@ def release_repo(tmp_path): (repo / "upstream.sh").write_text("version=1.0\nrelease=beta.1\n") (repo / "patches").mkdir() (repo / "patches" / "a.patch").write_text("a\n") + (repo / "additions" / "juggler").mkdir(parents=True) + (repo / "additions" / "juggler" / "jar.mn").write_text("% content juggler %content/\n") (repo / "typescript").mkdir() (repo / "typescript" / "x.ts").write_text("x\n") _git(repo, "add", "-A") @@ -2682,7 +2699,7 @@ def test_driver_pr_on_the_released_sources_tests_the_release(release_repo): base = _git(release_repo, "rev-parse", "HEAD") (release_repo / "typescript" / "x.ts").write_text("y\n") _git(release_repo, "commit", "-qam", "driver change") - assert _scope(release_repo, base)[0] == "browser_changed=false" + assert "browser_changed=false" in _scope(release_repo, base)[0].splitlines() def test_driver_pr_on_unreleased_browser_sources_builds(release_repo): @@ -2713,6 +2730,47 @@ def test_unpublished_release_tag_builds(release_repo): assert _scope(release_repo, base)[0] == "browser_changed=true" +@pytest.mark.parametrize("releases", [[], [{**PUBLISHED[0], "draft": True}]], ids=["missing", "draft"]) +def test_driver_pr_whose_release_is_not_published_builds(release_repo, releases): + """A tag alone is not a release: #811 must not pair with a draft or a missing one.""" + base = _git(release_repo, "rev-parse", "HEAD") + (release_repo / "typescript" / "x.ts").write_text("y\n") + _git(release_repo, "commit", "-qam", "driver change") + assert _scope(release_repo, base, releases)[0] == "browser_changed=true" + + +def test_driver_pr_tests_the_prerelease_of_its_sources_not_the_latest_release(release_repo): + """#811: v156.0.1-beta.32 was a prerelease and the fetch took the latest stable + release, 152.0.4. The pairing names the prerelease, and the fetch job uses it.""" + base = _git(release_repo, "rev-parse", "HEAD") + (release_repo / "typescript" / "x.ts").write_text("y\n") + _git(release_repo, "commit", "-qam", "driver change") + stable = {"tag_name": "v0.9-beta.0", "draft": False, "prerelease": False, "body": ""} + result, _ = _scope(release_repo, base, PUBLISHED + [stable]) + assert "browser_changed=false" in result.splitlines() + assert "browser_tag=v1.0-beta.1" in (release_repo.parent / "out.txt").read_text() + fetch = _jobs()["fetch-browser"]["steps"] + assert any("--browser-tag \"${{ needs.resolve.outputs.browser_tag }}\"" in (s.get("run") or "") + for s in fetch), "the fetch job no longer installs the paired release" + + +def test_driver_pr_pairs_by_the_source_digest_in_the_release_manifest(release_repo): + """A release built by release.yml takes its number from its tag; main's + upstream.sh never names it, so the pairing comes from the digest in the + release's manifest.json.""" + from ci.browser_inputs import source_digest + from ci.release import manifest + + base = _git(release_repo, "rev-parse", "HEAD") + (release_repo / "typescript" / "x.ts").write_text("y\n") + _git(release_repo, "commit", "-qam", "driver change") + cut = {"tag_name": "v1.0-beta.2", "draft": False, "prerelease": True, + "manifest": manifest("v1.0-beta.2", source_digest(release_repo), "0" * 40)} + result, _ = _scope(release_repo, base, [cut]) + assert "browser_changed=false" in result.splitlines() + assert "browser_tag=v1.0-beta.2" in (release_repo.parent / "out.txt").read_text() + + # --------------------------------------------------------------------------- # patch guards: every guard runs in exactly one CI leg # --------------------------------------------------------------------------- diff --git a/ci/tests/test_release.py b/ci/tests/test_release.py new file mode 100644 index 0000000..5d0b258 --- /dev/null +++ b/ci/tests/test_release.py @@ -0,0 +1,210 @@ +"""The release plan: which versions a merge or a tag publishes, and which browser they pair with.""" + +import argparse +import json +import re +import shutil + +import pytest + +from ci import browser_inputs, release +from ci._util import read_upstream_sh +from ci.release import ( + is_gate_check, + last_library_tag, + library_changed, + next_browser_release, + paired_release, + plan_prerelease, + plan_stable, +) + +PYPI = ["0.5.5", "0.5.6", "0.5.6b1"] +NPM = ["0.5.6"] + + +def test_a_merge_prereleases_the_checked_in_version_when_it_is_unreleased(): + plan = plan_prerelease("0.5.7", PYPI, NPM) + assert (plan.py, plan.npm, plan.dist_tag) == ("0.5.7b1", "0.5.7-beta.1", "next") + + +def test_prerelease_numbers_count_up_across_both_registries(): + assert plan_prerelease("0.5.7", PYPI + ["0.5.7b1", "0.5.7b2"], NPM).py == "0.5.7b3" + assert plan_prerelease("0.5.7", PYPI, NPM + ["0.5.7-beta.4"]).npm == "0.5.7-beta.5" + + +def test_after_a_release_the_next_prerelease_is_of_the_next_patch(): + """Never 0.5.7b3 after 0.5.7 shipped: pip would rank it below the release.""" + plan = plan_prerelease("0.5.7", PYPI + ["0.5.7"], NPM + ["0.5.7"]) + assert plan.py == "0.5.8b1" + + +def test_a_tag_publishes_its_own_version_as_latest(): + plan = plan_stable("v0.5.7", PYPI, NPM) + assert (plan.py, plan.npm, plan.dist_tag) == ("0.5.7", "0.5.7", "latest") + + +@pytest.mark.parametrize("tag,why", [ + ("v0.5.6", "already published"), + ("v0.5.4", "not newer"), + ("v156.0.1-beta.33", "not a release tag"), + ("0.5.7", "not a release tag"), +]) +def test_a_tag_that_cannot_be_a_release_is_refused(tag, why): + with pytest.raises(ValueError, match=why): + plan_stable(tag, PYPI, NPM) + + +def test_browser_release_numbers_are_global_and_never_reused(): + tags = ["v152.0.4-beta.30", "v152.0.4-beta.31", "v156.0.1-beta.33", "v152.0.2-alpha.1", "font-bundle-v1"] + assert next_browser_release("beta.32", tags) == "beta.34" + assert next_browser_release("beta.40", tags) == "beta.40" # upstream.sh is the floor + assert next_browser_release("beta.32", []) == "beta.32" + + +def _released(tag, digest, **extra): + return {"tag_name": tag, "manifest": release.manifest(tag, digest, "0" * 40), **extra} + + +def test_a_library_pairs_with_the_newest_non_draft_release_from_its_sources(): + releases = [ + _released("v156.0.1-beta.35", "aaa", draft=True), + _released("v156.0.1-beta.34", "bbb"), + _released("v156.0.1-beta.33", "aaa"), + {"tag_name": "v152.0.4-beta.31", "assets": []}, # before manifests + ] + assert paired_release(releases, "aaa")["tag_name"] == "v156.0.1-beta.33" + assert paired_release(releases, "ccc") is None + + +def test_a_release_without_a_manifest_asset_is_not_downloaded(monkeypatch): + monkeypatch.setattr(release, "http_json", lambda url: pytest.fail(f"fetched {url}")) + assert release.release_manifest({"tag_name": "v152.0.4-beta.30", "assets": [{"name": "a.zip"}]}) is None + + +def test_library_tags_order_as_versions_and_never_as_browser_tags(): + tags = ["v0.5.7b1", "v0.5.7b10", "v0.5.7b2", "v0.5.6", "v156.0.1-beta.33", "font-bundle-v1"] + assert last_library_tag(tags) == "v0.5.7b10" + assert last_library_tag(tags + ["v0.5.7"]) == "v0.5.7" + assert last_library_tag(["v156.0.1-beta.33"]) is None + # a library prerelease tag must not take a browser release number + assert next_browser_release("beta.32", ["v0.5.7b40"]) == "beta.32" + + +def test_the_gate_check_is_found_when_release_yml_runs_the_tests(): + assert is_gate_check("All tests passed") + assert is_gate_check("tests / All tests passed") + assert not is_gate_check("Not All tests passed") + + +def _git(repo, *args): + import subprocess + + return subprocess.run(["git", "-c", "user.email=ci@test", "-c", "user.name=ci", *args], + cwd=repo, check=True, capture_output=True, text=True).stdout.strip() + + +@pytest.fixture +def library_repo(tmp_path): + """A repo whose last library release, v0.5.7b1, is tagged on HEAD.""" + for rel in ("pythonlib/camoufox/a.py", "docs/a.md", "patches/a.patch", "upstream.sh"): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text("a\n") + _git(tmp_path, "init", "-q", "-b", "main") + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-qm", "released") + _git(tmp_path, "tag", "v0.5.7b1") + return tmp_path + + +@pytest.mark.parametrize("changed, publish", [ + ("docs/a.md", False), + ("pythonlib/camoufox/a.py", True), + ("patches/a.patch", True), # a new browser means a new pin +]) +def test_a_library_prerelease_is_published_only_when_what_it_ships_changed(library_repo, changed, publish): + (library_repo / changed).write_text("b\n") + _git(library_repo, "commit", "-qam", "change") + assert library_changed(library_repo)[0] is publish + + +def test_the_first_library_prerelease_is_always_published(library_repo): + _git(library_repo, "tag", "-d", "v0.5.7b1") + assert library_changed(library_repo)[0] is True + + +def test_set_build_names_the_tags_release_in_the_working_tree_only(tmp_path, monkeypatch): + shutil.copy(release.REPO_ROOT / "upstream.sh", tmp_path / "upstream.sh") + monkeypatch.setattr(release, "REPO_ROOT", tmp_path) + monkeypatch.setattr(release, "read_upstream_sh", + lambda: read_upstream_sh(tmp_path / "upstream.sh")) + version = read_upstream_sh(tmp_path / "upstream.sh")["version"] + + release.cmd_set_build(argparse.Namespace(tag=f"v{version}-beta.77")) + assert re.search(r"^release=beta\.77$", (tmp_path / "upstream.sh").read_text(), re.M) + with pytest.raises(SystemExit): + release.cmd_set_build(argparse.Namespace(tag="v1.0-beta.78")) # another Firefox + + +def _copy_browser_inputs(tmp_path): + root = tmp_path / "repo" + for name in browser_inputs.BROWSER_DIRS: + if (browser_inputs.REPO_ROOT / name).is_dir(): + shutil.copytree(browser_inputs.REPO_ROOT / name, root / name) + for name in browser_inputs.BROWSER_FILES: + shutil.copy(browser_inputs.REPO_ROOT / name, root / name) + return root + + +def test_the_source_digest_ignores_the_release_number_and_nothing_else(tmp_path): + root = _copy_browser_inputs(tmp_path) + before = browser_inputs.source_digest(root) + up = root / "upstream.sh" + up.write_text(re.sub(r"^release=.*$", "release=beta.999", up.read_text(), flags=re.M)) + assert browser_inputs.source_digest(root) == before + + juggler = next((root / "additions" / "juggler").rglob("*.js")) + juggler.write_text(juggler.read_text() + "\n") + assert browser_inputs.source_digest(root) != before, "a packaged resource is a browser source" + + up.write_text(up.read_text().replace("version=", "version=999", 1)) + assert browser_inputs.source_digest(root) != before + + +def test_stamp_writes_the_pin_and_every_version(tmp_path, monkeypatch): + for name, src in [("PIN_FILE", release.PIN_FILE), ("PYPROJECT", release.PYPROJECT), + ("PACKAGE_JSON", release.PACKAGE_JSON), ("TS_VERSION", release.TS_VERSION)]: + dst = tmp_path / src.name + shutil.copy(src, dst) + monkeypatch.setattr(release, name, dst) + monkeypatch.setenv("GITHUB_REPOSITORY", "daijro/camoufox") + + release.cmd_stamp(argparse.Namespace(browser_tag="v156.0.1-beta.33", + py_version="0.5.8b2", npm_version="0.5.8-beta.2")) + + assert json.loads(release.PIN_FILE.read_text()) == { + "tag": "v156.0.1-beta.33", "repo": "daijro/camoufox", "repo_name": "Official", + "version": "156.0.1", "build": "beta.33", + } + assert 'version = "0.5.8b2"' in release.PYPROJECT.read_text() + assert json.loads(release.PACKAGE_JSON.read_text())["version"] == "0.5.8-beta.2" + assert 'LIBRARY_VERSION = "0.5.8-beta.2"' in release.TS_VERSION.read_text() + + +def test_the_pin_and_the_launchers_agree_on_its_format(tmp_path, monkeypatch): + """What stamp writes is what pythonlib's loader reads.""" + import sys + + sys.path.insert(0, str(release.REPO_ROOT / "pythonlib")) + from camoufox.browser_pin import load_pin + + monkeypatch.setattr(release, "PIN_FILE", tmp_path / "browser-pin.json") + for name in ("PYPROJECT", "PACKAGE_JSON", "TS_VERSION"): + dst = tmp_path / getattr(release, name).name + shutil.copy(getattr(release, name), dst) + monkeypatch.setattr(release, name, dst) + release.cmd_stamp(argparse.Namespace(browser_tag="v156.0.1-beta.33", + py_version="0.5.8", npm_version="0.5.8")) + pin = load_pin(release.PIN_FILE) + assert (pin.repo_name, pin.version, pin.build, pin.tag) == \ + ("official", "156.0.1", "beta.33", "v156.0.1-beta.33") diff --git a/docs/FONTS.md b/docs/FONTS.md index 1a30ac0..1f5894b 100644 --- a/docs/FONTS.md +++ b/docs/FONTS.md @@ -39,8 +39,8 @@ repo or deletes the release. Publishing a new bundle: rebuild the archive, `python3 scripts/fetch-fonts.py --write-spec --tag font-bundle-vN`, then upload it under that tag. -Font-bundle tags are excluded from `build.yml`, so they do not trigger a browser -build or appear among the browser downloads. +Font-bundle tags do not match `release.yml`'s `vX.Y.Z` trigger, so they do not +start a release, and they carry no `manifest.json`, so no library pairs with them. ## Layout: each face stored once diff --git a/pythonlib/README.md b/pythonlib/README.md index 8871e2c..6684281 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -125,9 +125,19 @@ Synced 26 versions from 2 repos.
+### Which browser build is used + +Each camoufox release is paired with the one browser build it was built and tested with. By default, `camoufox fetch` installs exactly that build and every launch uses it. That holds even when other builds are installed, and even when the paired build is a prerelease (a prerelease of this package pairs with a prerelease browser). Upgrading the package therefore never runs a browser it was not tested with. Run `camoufox fetch` after upgrading to install the new pairing. + +Choosing a channel or a build with `camoufox set` overrides the pairing. The choice is kept, and a launch warns that the build differs from the paired one. `camoufox set --release` goes back to the paired build. + +A development checkout (installed from the repository, not from PyPI) is paired with nothing and follows its channel, `official/stable` by default. + +
+ ### `set` -Choose a version channel or pin a specific version. Can also be called with a specifier to activate directly. +Choose a version channel or pin a specific version, overriding the paired build. Can also be called with a specifier to activate directly. Interactive selector: @@ -135,10 +145,10 @@ Interactive selector: > camoufox set ``` -You can also pass a specifier to pin a specific version or choose a channel to follow directly. This will pull the latest stable version from the official repo on `camoufox fetch`. +You can also pass a specifier to pin a specific version or choose a channel to follow directly. Following `official/stable` pulls the latest stable version from the official repo on `camoufox fetch`: ```bash -> camoufox set official/stable # Default setting +> camoufox set official/stable ``` Follow latest prerelease version from the official repo, if applicable: @@ -153,6 +163,12 @@ Pin a specific version: > camoufox set official/stable/134.0.2-beta.20 ``` +Go back to the build this release is paired with: + +```bash +> camoufox set --release +``` +
### `active` @@ -160,8 +176,8 @@ Pin a specific version: Prints the current active version string: ```bash -> camoufox active # Default channel is active -official/stable +> camoufox active # A released package uses its paired build by default +official/prerelease/156.0.1-beta.33 (1a2b3c4d) (paired with this release) ``` ```bash @@ -177,10 +193,10 @@ coryking/stable/142.0.1-fork.26 (not installed) ### `fetch` -Install the latest version from the active channel. By default, this is official/stable. This will also automatically sync repository assets. +Install the browser build this release is paired with, or, after `camoufox set`, the latest version from the chosen channel. This will also automatically sync repository assets. ```bash -> camoufox fetch # Install the latest in the channel +> camoufox fetch # Install the paired build (or the latest in the chosen channel) ``` To download the latest from a different channel, or pin a version: diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index 81e44ff..9b47059 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -38,6 +38,7 @@ from .multiversion import ( save_config, save_repo_cache, ) +from .browser_pin import effective_pin, load_pin from .pkgman import ( INSTALL_DIR, AvailableVersion, @@ -268,6 +269,7 @@ def fetch(version): repo_name = None repo_data = None ver_data = None + paired = False if version: parts = version.lower().split("/") if len(parts) == 1: @@ -286,6 +288,17 @@ def fetch(version): repo_data = _repo_data(cache, repo_name) if repo_data is not None: ver_data = _pin_target(repo_data, config["pinned"], config.get("pinned_sha")) + elif pin := effective_pin(config): + # This library release pairs with exactly one browser build. + repo_name = pin.repo_name + repo_data = _repo_data(cache, repo_name) + if repo_data is not None: + ver_data = _pin_target(repo_data, pin.spec, None) + if ver_data is None: + rprint(f"{pin.tag}, the browser this camoufox release pairs with, is not in " + f"{repo_name}'s releases. Run 'camoufox sync' and try again.", fg="red") + return + paired = True else: channel = config.get("channel") or get_default_channel() repo_name, ctype = (channel.split("/", 1) + ["stable"])[:2] @@ -312,7 +325,11 @@ def fetch(version): ) repo_config = RepoConfig.find_by_name(repo_data["name"]) try: - CamoufoxUpdate(repo_config=repo_config, selected_version=selected).update() + # The paired build is what this release was tested with; installing it + # needs no "this is a prerelease" confirmation even when it is one. + CamoufoxUpdate(repo_config=repo_config, selected_version=selected).update( + i_know_what_im_doing=paired + ) except Exception as e: msg = str(e) if "404" in msg or "Not Found" in msg: @@ -426,7 +443,11 @@ def _set_pinned(repo_name: str, channel_type: str, ver_data: dict, inst, sha: Op @cli.command(name="set") @click.argument("specifier", required=False) @click.option("--geoip", is_flag=True, help="Select GeoIP source instead") -def set_cmd(specifier, geoip): +@click.option( + "--release", "paired", is_flag=True, + help="Forget any explicit choice and use the browser this camoufox release pairs with", +) +def set_cmd(specifier, geoip, paired): """ \b Set the active Camoufox version to use & fetch. @@ -441,6 +462,19 @@ def set_cmd(specifier, geoip): _select_geoip_source() return + if paired: + pin = load_pin() + if pin is None: + rprint("This camoufox is a development copy; it pairs with no particular browser.", fg="yellow") + return + config = load_config() + for key in ("channel", "pinned", "pinned_sha", "active_version"): + config.pop(key, None) + save_config(config) + click.secho(f"Using the paired browser: {pin.repo_name} {pin.spec} ({pin.tag})", fg="green") + click.secho("Run 'camoufox fetch' if it is not installed.", fg="yellow") + return + if specifier: parts = specifier.lower().split("/") @@ -1020,6 +1054,16 @@ def active_cmd(): sha8 = (v.sha256 or "")[:8] return f"{v.channel_path} ({sha8})" if sha8 else v.channel_path + pin = effective_pin(config) + if pin: + target = _find_installed(f"{pin.repo_name}/{pin.spec}") + if target: + click.echo(f"{_label(target)} (paired with this release)") + else: + click.echo(f"{pin.repo_name}/{pin.spec} (paired with this release) ", nl=False) + rprint("(not fetched)", fg="yellow") + return + if pinned: pinned_sha = config.get("pinned_sha") if pinned_sha: diff --git a/pythonlib/camoufox/async_api.py b/pythonlib/camoufox/async_api.py index dcb35a2..69a6368 100644 --- a/pythonlib/camoufox/async_api.py +++ b/pythonlib/camoufox/async_api.py @@ -180,7 +180,9 @@ async def _launch( async def _resolve_proxy_geo(proxy: Dict[str, str]) -> Tuple[str, str]: """The proxy's exit IP and timezone, looked up off the event loop.""" - return await asyncio.to_thread(proxy_exit_geo, Proxy(**proxy).as_string()) + # run_in_executor rather than asyncio.to_thread, which needs Python 3.9. + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, proxy_exit_geo, Proxy(**proxy).as_string()) async def AsyncNewContext( diff --git a/pythonlib/camoufox/browser-pin.json b/pythonlib/camoufox/browser-pin.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/pythonlib/camoufox/browser-pin.json @@ -0,0 +1 @@ +{} diff --git a/pythonlib/camoufox/browser_pin.py b/pythonlib/camoufox/browser_pin.py new file mode 100644 index 0000000..14be6be --- /dev/null +++ b/pythonlib/camoufox/browser_pin.py @@ -0,0 +1,99 @@ +""" +The browser build this copy of the library was released with. + +Every published camoufox package (PyPI and npm) is stamped at release time with +the one browser release built from the same sources: `browser-pin.json` names +it. By default the library fetches and launches exactly that build, so a +library upgrade can never silently run against a browser whose Juggler, prefs +or patches it was not tested with -- and a new browser release can never be +picked up by an older library. + +A user who explicitly chooses something else (`camoufox set official/stable`, +`camoufox set 156.0.1-beta.33`, ...) keeps that choice; launching then warns +that the build differs from the one this library was released with. + +A development checkout carries an empty pin (`{}`): nothing is pinned and the +channel logic applies unchanged, because there is no release to pair with yet. +The release workflow writes the real file (ci/release.py stamp). +""" + +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Optional + +import orjson + +PIN_FILE = Path(__file__).parent / "browser-pin.json" + + +@dataclass(frozen=True) +class BrowserPin: + tag: str + repo: str + repo_name: str + version: str + build: str + + @property + def spec(self) -> str: + """`-`, the form `camoufox set` and the cache use.""" + return f"{self.version}-{self.build}" + + +def load_pin(path: Optional[Path] = None) -> Optional[BrowserPin]: + """The stamped pin, or None for an unpinned (development) copy.""" + try: + data: Dict[str, Any] = orjson.loads((path or PIN_FILE).read_bytes()) + except (FileNotFoundError, orjson.JSONDecodeError): + return None + if not data or not data.get("tag"): + return None + return BrowserPin( + tag=data["tag"], + repo=data["repo"], + repo_name=data["repo_name"].lower(), + version=data["version"], + build=data["build"], + ) + + +def is_explicit_choice(config: Dict[str, Any]) -> bool: + """Whether the user chose a channel or a build themselves.""" + return bool(config.get("channel") or config.get("pinned")) + + +def effective_pin(config: Optional[Dict[str, Any]] = None) -> Optional[BrowserPin]: + """The package pin, unless the user explicitly chose otherwise.""" + pin = load_pin() + if pin is None: + return None + if config is None: + from .multiversion import load_config + + config = load_config() + return None if is_explicit_choice(config) else pin + + +def matches(pin: BrowserPin, repo_name: str, version: str, build: str) -> bool: + return (repo_name.lower(), version, build) == (pin.repo_name, pin.version, pin.build) + + +_warned = False + + +def warn_if_unpaired(repo_name: str, version: str, build: str) -> None: + """Warn once when an explicitly chosen build is not the one this library pairs with.""" + global _warned + pin = load_pin() + if _warned or pin is None or matches(pin, repo_name, version, build): + return + _warned = True + warnings.warn( + f"Launching {repo_name.lower()} {version}-{build}, which you selected explicitly. " + f"This camoufox release was built and tested with {pin.repo_name} {pin.spec} " + f"({pin.tag}); other builds may not be compatible. " + "Run `camoufox set --release` to go back to the paired build.", + RuntimeWarning, + stacklevel=3, + ) diff --git a/pythonlib/camoufox/multiversion.py b/pythonlib/camoufox/multiversion.py index 08e53e1..4ed539c 100644 --- a/pythonlib/camoufox/multiversion.py +++ b/pythonlib/camoufox/multiversion.py @@ -323,6 +323,20 @@ def get_active_path() -> Optional[Path]: config = load_config() active = config.get('active_version') + # A released library launches the build it was released with, whatever + # happens to be marked active, unless the user explicitly chose otherwise. + from .browser_pin import effective_pin, matches + + pin = effective_pin(config) + if pin: + for inst in list_installed(): + if matches(pin, inst.repo_name, inst.version.version or '', inst.version.build): + if active != inst.relative_path: + config['active_version'] = inst.relative_path + save_config(config) + return inst.path + return None + if active: path = INSTALL_DIR / active if path.exists() and (path / 'version.json').exists(): diff --git a/pythonlib/camoufox/pkgman.py b/pythonlib/camoufox/pkgman.py index bcbefb2..e7aef25 100644 --- a/pythonlib/camoufox/pkgman.py +++ b/pythonlib/camoufox/pkgman.py @@ -128,6 +128,10 @@ def _parse_semver(version: str) -> Tuple[int, ...]: Parse a semver string into a comparable tuple """ version = version.lstrip('^~') + # A prerelease compares as its release: 0.5.8b1 (PEP 440) and 0.5.8-beta.1 + # (semver) are both 0.5.8 -- not 0.5.0, which is what splitting on '.' + # alone made of them. + version = re.split(r'(?<=\d)(?:[-+]|(?:a|b|rc|dev|\.dev|\.post)\d*)', version, maxsplit=1)[0] parts = [] for part in version.split('.'): try: @@ -580,8 +584,15 @@ class CamoufoxFetcher(GitHubDownloader): return None version = Version(build=match['build'], version=match['version']) + # A released library installs exactly the browser it was released with + # (browser_pin.py), unless the user explicitly chose something else. + from .browser_pin import effective_pin, matches + + pin = effective_pin() + if pin and not matches(pin, self.repo_config.name, match['version'], match['build']): + return None is_prerelease = bool(release and release.get('prerelease')) or version.is_alpha - if not self.repo_config.is_version_supported(version, is_prerelease): + if not pin and not self.repo_config.is_version_supported(version, is_prerelease): return None digest = asset.get('digest') or '' @@ -732,6 +743,22 @@ def list_available_versions( return versions +def _not_installed_message() -> str: + """What is missing -- the paired build, a pin, or a channel -- and how to get it.""" + from .browser_pin import effective_pin + from .multiversion import get_default_channel, load_config + + config = load_config() + pin = effective_pin(config) + pinned = config.get("pinned") + channel = config.get("channel") or get_default_channel() + if pin: + missing = f"{pin.repo_name} {pin.spec}, the browser this camoufox release pairs with," + else: + missing = f"{channel}/{pinned}" if pinned else channel + return f"{missing} is not installed. Please run `camoufox fetch` to install." + + def installed_verstr() -> str: """ Get the full version string of the active install @@ -740,16 +767,13 @@ def installed_verstr() -> str: active = get_active_path() if active is None: - from .multiversion import get_default_channel, load_config + raise CamoufoxNotInstalled(_not_installed_message()) + version = Version.from_path(active) + if active.parent.parent.name == "browsers": + from .browser_pin import warn_if_unpaired - config = load_config() - pinned = config.get("pinned") - channel = config.get("channel") or get_default_channel() - active_display = f"{channel}/{pinned}" if pinned else channel - raise CamoufoxNotInstalled( - f"{active_display} is not installed. " f"Please run `camoufox fetch` to install." - ) - return Version.from_path(active).full_string + warn_if_unpaired(active.parent.name, version.version or "", version.build) + return version.full_string def _root_install_supported() -> bool: @@ -787,24 +811,18 @@ def camoufox_path(download_if_missing: bool = True) -> Path: if not os.path.exists(INSTALL_DIR) or not os.listdir(INSTALL_DIR): if not download_if_missing: - from .multiversion import load_config, get_default_channel - - config = load_config() - pinned = config.get("pinned") - channel = config.get("channel") or get_default_channel() - if pinned: - active_display = f"{channel}/{pinned}" - else: - active_display = channel - raise CamoufoxNotInstalled( - f"{active_display} is not installed. " f"Please run `camoufox fetch` to install." - ) + raise CamoufoxNotInstalled(_not_installed_message()) elif os.path.exists(INSTALL_DIR) and _root_install_supported(): return INSTALL_DIR else: if not download_if_missing: + from .browser_pin import effective_pin + + # Other builds are installed, but not the one this release pairs with. + if effective_pin(): + raise CamoufoxNotInstalled(_not_installed_message()) raise UnsupportedVersion("Camoufox executable is outdated.") CamoufoxFetcher().install() diff --git a/pythonlib/tests/test_browser_pin.py b/pythonlib/tests/test_browser_pin.py new file mode 100644 index 0000000..3db53ec --- /dev/null +++ b/pythonlib/tests/test_browser_pin.py @@ -0,0 +1,146 @@ +"""A released library runs the browser it was released with, and nothing else by default. + +Each published package is stamped with `browser-pin.json` naming the browser +release built from the same sources. These tests hold the three places that +decide which build is used -- the active install at launch, the asset the +fetcher accepts, and the explicit-choice escape hatch -- to that pairing. +""" + +import json +import warnings + +import pytest + +from camoufox import browser_pin, multiversion, pkgman +from camoufox.exceptions import CamoufoxNotInstalled + +PIN = { + "tag": "v156.0.1-beta.33", + "repo": "daijro/camoufox", + "repo_name": "Official", + "version": "156.0.1", + "build": "beta.33", +} + + +def _setup(tmp_path, monkeypatch, *, pin, installed, config): + root = tmp_path / "cache" + root.mkdir() + (root / ".0.5_FLAG").write_text("") + (root / "repo_cache.json").write_text("{}") + (root / "config.json").write_text(json.dumps(config)) + for spec in installed: + version, build = spec.split("-", 1) + d = root / "browsers" / "official" / spec + d.mkdir(parents=True) + (d / "version.json").write_text(json.dumps({"version": version, "build": build})) + pin_file = tmp_path / "browser-pin.json" + pin_file.write_text(json.dumps(pin)) + monkeypatch.setattr(browser_pin, "PIN_FILE", pin_file) + monkeypatch.setattr(browser_pin, "_warned", False) + for module in (pkgman, multiversion): + monkeypatch.setattr(module, "INSTALL_DIR", root) + monkeypatch.setattr(multiversion, "BROWSERS_DIR", root / "browsers") + monkeypatch.setattr(multiversion, "CONFIG_FILE", root / "config.json") + monkeypatch.setattr(multiversion, "COMPAT_FLAG", root / ".0.5_FLAG") + return root + + +@pytest.mark.parametrize("content", ["{}", '{"tag": null}', "", "not json"]) +def test_an_empty_or_missing_pin_pins_nothing(tmp_path, content): + f = tmp_path / "browser-pin.json" + f.write_text(content) + assert browser_pin.load_pin(f) is None + assert browser_pin.load_pin(tmp_path / "absent.json") is None + + +def test_the_checked_in_pin_is_empty(): + """main pins nothing; only the release workflow writes a real pin.""" + assert json.loads(browser_pin.PIN_FILE.read_text()) == {} + + +def test_launch_uses_the_paired_build_even_when_another_is_marked_active(tmp_path, monkeypatch): + root = _setup( + tmp_path, monkeypatch, pin=PIN, + installed=["156.0.1-beta.33", "157.0-beta.34"], + config={"active_version": "browsers/official/157.0-beta.34"}, + ) + assert multiversion.get_active_path() == root / "browsers/official/156.0.1-beta.33" + assert pkgman.installed_verstr() == "156.0.1-beta.33" + + +def test_a_newer_build_is_not_used_when_the_paired_one_is_missing(tmp_path, monkeypatch): + _setup(tmp_path, monkeypatch, pin=PIN, installed=["157.0-beta.34"], config={}) + assert multiversion.get_active_path() is None + with pytest.raises(CamoufoxNotInstalled, match="156.0.1-beta.33"): + pkgman.installed_verstr() + + +def test_an_explicit_choice_is_kept_and_warned_about(tmp_path, monkeypatch): + root = _setup( + tmp_path, monkeypatch, pin=PIN, + installed=["156.0.1-beta.33", "157.0-beta.34"], + config={"channel": "official/stable", "active_version": "browsers/official/157.0-beta.34"}, + ) + assert multiversion.get_active_path() == root / "browsers/official/157.0-beta.34" + with pytest.warns(RuntimeWarning, match="v156.0.1-beta.33"): + assert pkgman.installed_verstr() == "157.0-beta.34" + with warnings.catch_warnings(): + warnings.simplefilter("error") + pkgman.installed_verstr() # once per process, not per launch + + +def test_an_explicit_choice_of_the_paired_build_does_not_warn(tmp_path, monkeypatch): + _setup( + tmp_path, monkeypatch, pin=PIN, installed=["156.0.1-beta.33"], + config={"pinned": "156.0.1-beta.33", "channel": "official/prerelease", + "active_version": "browsers/official/156.0.1-beta.33"}, + ) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert pkgman.installed_verstr() == "156.0.1-beta.33" + + +def test_without_a_pin_the_channel_logic_is_unchanged(tmp_path, monkeypatch): + root = _setup( + tmp_path, monkeypatch, pin={}, installed=["156.0.1-beta.33", "157.0-beta.34"], + config={"active_version": "browsers/official/157.0-beta.34"}, + ) + assert multiversion.get_active_path() == root / "browsers/official/157.0-beta.34" + + +@pytest.mark.parametrize("config,accepts_newer", [({}, False), ({"channel": "official/stable"}, True)]) +def test_the_fetcher_accepts_only_the_paired_asset_by_default(tmp_path, monkeypatch, config, accepts_newer): + _setup(tmp_path, monkeypatch, pin=PIN, installed=[], config=config) + fetcher = pkgman.CamoufoxFetcher.__new__(pkgman.CamoufoxFetcher) + fetcher.repo_config = pkgman.RepoConfig.get_default() + fetcher.pattern = fetcher.repo_config.build_pattern(spoof_os="lin", spoof_arch="x86_64") + fetcher.installed_sha256 = None + fetcher.installed_created_at = None + + def asset(version, build): + return {"name": f"camoufox-{version}-{build}-lin.x86_64.zip", + "browser_download_url": f"https://example.invalid/{version}-{build}.zip"} + + assert fetcher.check_asset(asset("156.0.1", "beta.33"), {"prerelease": True}) is not None + newer = fetcher.check_asset(asset("157.0", "beta.34"), {"prerelease": False}) + assert (newer is not None) is accepts_newer + + +@pytest.mark.parametrize("raw,expected", [ + ("0.5.8", (0, 5, 8)), + ("0.5.8b1", (0, 5, 8)), + ("0.5.8rc2", (0, 5, 8)), + ("0.5.8.dev3", (0, 5, 8)), + ("0.5.8-beta.1", (0, 5, 8)), + ("^0.5.0", (0, 5, 0)), + ("1", (1, 0, 0)), +]) +def test_prerelease_library_versions_parse_as_their_release(raw, expected): + assert pkgman._parse_semver(raw) == expected + + +def test_a_missing_paired_build_is_reported_as_not_installed_not_outdated(tmp_path, monkeypatch): + _setup(tmp_path, monkeypatch, pin=PIN, installed=["157.0-beta.34"], config={}) + with pytest.raises(CamoufoxNotInstalled, match="pairs with"): + pkgman.camoufox_path(download_if_missing=False) diff --git a/typescript/README.md b/typescript/README.md index df907f7..d32972d 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -111,8 +111,9 @@ Python, `headless: "virtual"` is handled by `Camoufox()`, `NewBrowser()` and ``` camoufox sync # refresh the version catalogue -camoufox fetch [version] # install the active or a specific version +camoufox fetch [version] # install the paired (or chosen) build, or a specific version camoufox set [specifier] # pin a version or channel; no specifier opens a picker +camoufox set --release # go back to the build this release is paired with camoufox set --geoip # pick a GeoIP source camoufox list [installed|all] # list versions camoufox remove [version] # remove one version, or everything (--select to pick) @@ -123,6 +124,13 @@ camoufox test [url] # open the Playwright inspector camoufox server # launch a Playwright server ``` +Each release of this package is paired with the one browser build it was built +and tested with, the same build as the `camoufox` Python release of the same +version. The first launch installs that build, and every launch uses it, until +you choose another with `camoufox set`. A launch then warns that the build +differs from the paired one. See the Python package's README, under "Which +browser build is used". + The commands and pickers match the Python CLI. The one exception is `gui`, a PySide6 desktop app that only the Python package provides. @@ -166,15 +174,15 @@ CAMOUFOX_E2E=1 CAMOUFOX_EXECUTABLE=/path/to/camoufox-bin pnpm test tests/e2e.tes ## Releasing -The npm package and pythonlib are released together, at one version, from one -place: **Actions → Publish to pypi**. That run first calls -`.github/workflows/publish-npm.yml` as a dry run -- type check, lint, tests, -build, and `scripts/check-pack.mjs` (the version must equal pythonlib's; every -data file must be in the tarball; the tarball must install and import in an -empty project) -- then uploads to PyPI, and its success triggers -`publish-npm.yml` to publish the same commit to npm with trusted publishing (no -token is stored). Starting `publish-npm.yml` by hand only retries the npm half, -and it refuses unless PyPI already has the version. +The npm package and pythonlib are released together by +[`.github/workflows/release.yml`](../.github/workflows/release.yml): a +prerelease under the `next` dist-tag for every tested merge to `main` that +changes the package, and a stable release under `latest` for a `vX.Y.Z` tag (see +[`ci/README.md`](../ci/README.md#releases)). One job builds both packages and +runs `scripts/check-pack.mjs` (the version must equal pythonlib's; every data +file must be in the tarball; the tarball must install and import in an empty +project); PyPI gets its upload first, then npm gets that same tarball, published +with trusted publishing (no token is stored). ## Licence diff --git a/typescript/scripts/check-pack.mjs b/typescript/scripts/check-pack.mjs index 99b824a..d3c53ed 100644 --- a/typescript/scripts/check-pack.mjs +++ b/typescript/scripts/check-pack.mjs @@ -26,7 +26,9 @@ const problems = []; // 1. version lockstep with pythonlib const pyproject = readFileSync(join(root, "..", "pythonlib", "pyproject.toml"), "utf8"); const pyVersion = pyproject.match(/^version\s*=\s*"([^"]+)"/m)?.[1]; -if (pyVersion !== pkg.version) { +// One release, spelled for each registry: npm's 0.5.8-beta.2 is PyPI's 0.5.8b2. +const asPep440 = (v) => v.replace(/-beta\.(\d+)$/, "b$1"); +if (pyVersion !== asPep440(pkg.version)) { problems.push(`package.json version ${pkg.version} != pythonlib ${pyVersion}`); } if (pkg.private) problems.push("package.json is private: npm will refuse to publish it"); diff --git a/typescript/src/__main__.ts b/typescript/src/__main__.ts index 80e8340..38d4e6f 100644 --- a/typescript/src/__main__.ts +++ b/typescript/src/__main__.ts @@ -15,6 +15,7 @@ import * as readline from "node:readline"; import { Argument, Command, Option } from "commander"; import { LIBRARY_VERSION } from "./__version__.js"; import { DefaultAddons, maybeDownloadAddons } from "./addons.js"; +import { effectivePin, loadPin, pinSpec } from "./browser-pin.js"; import { FileNotFoundError } from "./exceptions.js"; import { allowGeoip, @@ -522,6 +523,8 @@ program let repoName: string | null = null; let repo: RepoBlock | null = null; let verData: CachedVersion | null = null; + let paired = false; + const pin = effectivePin(config); if (version) { const parts = version.toLowerCase().split("/"); @@ -546,6 +549,20 @@ program repoName = channel.includes("/") ? channel.split("/")[0] : channel; repo = repoData(cache, repoName); if (repo) verData = pinTarget(repo, config.pinned, config.pinned_sha); + } else if (pin) { + // This library release pairs with exactly one browser build. + repoName = pin.repoName; + repo = repoData(cache, repoName); + if (repo) verData = pinTarget(repo, pinSpec(pin), undefined); + if (!verData) { + rprint( + `${pin.tag}, the browser this camoufox release pairs with, is not in ` + + `${repoName}'s releases. Run 'camoufox sync' and try again.`, + "red", + ); + return; + } + paired = true; } else { const channel = config.channel || getDefaultChannel(); const slash = channel.indexOf("/"); @@ -580,7 +597,9 @@ program const repoConfig = RepoConfig.findByName(repo.name); try { const updater = await new CamoufoxUpdate(repoConfig, selected).init(); - await updater.update(); + // The paired build is what this release was tested with; installing + // it needs no "this is a prerelease" confirmation even when it is one. + await updater.update(false, paired); } catch (e) { const msg = errorMessage(e); if (msg.includes("404") || msg.includes("Not Found")) { @@ -615,12 +634,39 @@ program ) .argument("[specifier]") .option("--geoip", "Select GeoIP source instead") - .action(async (specifier: string | undefined, { geoip }) => { + .option( + "--release", + "Forget any explicit choice and use the browser this camoufox release pairs with", + ) + .action(async (specifier: string | undefined, { geoip, release }) => { if (geoip) { await selectGeoIPSource(); return; } + if (release) { + const releasePin = loadPin(); + if (!releasePin) { + rprint( + "This camoufox is a development copy; it pairs with no particular browser.", + "yellow", + ); + return; + } + const config = loadConfig(); + delete config.channel; + delete config.pinned; + delete config.pinned_sha; + delete config.active_version; + saveConfig(config); + rprint( + `Using the paired browser: ${releasePin.repoName} ${pinSpec(releasePin)} (${releasePin.tag})`, + "green", + ); + rprint("Run 'camoufox fetch' if it is not installed.", "yellow"); + return; + } + if (specifier) { const parts = specifier.toLowerCase().split("/"); @@ -1305,6 +1351,21 @@ program return sha8 ? `${v.channelPath} (${sha8})` : v.channelPath; }; + const pin = effectivePin(config); + if (pin) { + const target = findInstalled(`${pin.repoName}/${pinSpec(pin)}`); + if (target) { + echo(`${label(target)} (paired with this release)`); + } else { + echo( + `${pin.repoName}/${pinSpec(pin)} (paired with this release) `, + false, + ); + rprint("(not fetched)", "yellow"); + } + return; + } + if (pinned) { const pinnedSha = config.pinned_sha; const target = pinnedSha diff --git a/typescript/src/browser-pin.ts b/typescript/src/browser-pin.ts new file mode 100644 index 0000000..ba21c0b --- /dev/null +++ b/typescript/src/browser-pin.ts @@ -0,0 +1,107 @@ +/** + * The browser build this copy of the library was released with. + * + * Twin of pythonlib/camoufox/browser_pin.py, reading the same + * `browser-pin.json` (a DATA_FILES entry). Every published package is stamped + * with the one browser release built from the same sources; by default the + * library fetches and launches exactly that build. An explicit user choice + * (`camoufox set ...`) is kept, with a warning at launch. A development + * checkout carries `{}` and pins nothing. + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { LOCAL_DATA } from "./paths.js"; +import { warn } from "./warnings.js"; + +/** Test seam, like pkgmanDeps: where the pin is read from. */ +export const browserPinDeps = { + file: path.join(LOCAL_DATA, "browser-pin.json"), +}; + +export interface BrowserPin { + tag: string; + repo: string; + repoName: string; + version: string; + build: string; +} + +/** `-`, the form `camoufox set` and the cache use. */ +export function pinSpec(pin: BrowserPin): string { + return `${pin.version}-${pin.build}`; +} + +/** The stamped pin, or null for an unpinned (development) copy. */ +export function loadPin(file: string = browserPinDeps.file): BrowserPin | null { + let data: Record; + try { + data = JSON.parse(fs.readFileSync(file, "utf-8")); + } catch { + return null; + } + if (!data?.tag) return null; + return { + tag: data.tag, + repo: String(data.repo), + repoName: String(data.repo_name).toLowerCase(), + version: String(data.version), + build: String(data.build), + }; +} + +/** Whether the user chose a channel or a build themselves. */ +export function isExplicitChoice(config: { + channel?: string; + pinned?: string; +}): boolean { + return Boolean(config.channel || config.pinned); +} + +/** The package pin, unless the user explicitly chose otherwise. */ +export function effectivePin(config: { + channel?: string; + pinned?: string; +}): BrowserPin | null { + const pin = loadPin(); + return pin && !isExplicitChoice(config) ? pin : null; +} + +export function pinMatches( + pin: BrowserPin, + repoName: string, + version: string, + build: string, +): boolean { + return ( + repoName.toLowerCase() === pin.repoName && + version === pin.version && + build === pin.build + ); +} + +let warned = false; + +/** Warn once when an explicitly chosen build is not the one this library pairs with. */ +export function warnIfUnpaired( + repoName: string, + version: string, + build: string, +): void { + const pin = loadPin(); + if (warned || pin === null || pinMatches(pin, repoName, version, build)) { + return; + } + warned = true; + warn( + `Launching ${repoName.toLowerCase()} ${version}-${build}, which you selected explicitly. ` + + `This camoufox release was built and tested with ${pin.repoName} ${pinSpec(pin)} ` + + `(${pin.tag}); other builds may not be compatible. ` + + "Run `camoufox set --release` to go back to the paired build.", + ); +} + +/** Test hook: forget that the warning was already issued. */ +export function resetUnpairedWarning(): void { + warned = false; +} diff --git a/typescript/src/multiversion.ts b/typescript/src/multiversion.ts index f76354a..48af302 100644 --- a/typescript/src/multiversion.ts +++ b/typescript/src/multiversion.ts @@ -7,6 +7,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { finished } from "node:stream/promises"; +import { effectivePin, pinMatches } from "./browser-pin.js"; import { INSTALL_DIR, OS_NAME, rprint } from "./paths.js"; // pkgman and multiversion are mutually dependent, exactly as the Python twin's // function-local imports are. Every use of these sits inside a function body, @@ -383,6 +384,29 @@ export function getActivePath(): string | null { const config = loadConfig(); const active = config.active_version; + // A released library launches the build it was released with, whatever + // happens to be marked active, unless the user explicitly chose otherwise. + const pin = effectivePin(config); + if (pin) { + for (const inst of listInstalled()) { + if ( + pinMatches( + pin, + inst.repoName, + inst.version.version ?? "", + inst.version.build, + ) + ) { + if (active !== inst.relativePath) { + config.active_version = inst.relativePath; + saveConfig(config); + } + return inst.path; + } + } + return null; + } + if (active) { const activePath = path.join(INSTALL_DIR, active); if ( diff --git a/typescript/src/paths.ts b/typescript/src/paths.ts index 113a039..87cbc07 100644 --- a/typescript/src/paths.ts +++ b/typescript/src/paths.ts @@ -72,6 +72,7 @@ export const INSTALL_DIR: string = userCacheDir("camoufox"); * tarball, so one seed draws one identity in either launcher. */ export const DATA_FILES: readonly string[] = [ + "browser-pin.json", "essential-fonts.json", "fingerprint-presets.json", "fingerprint-presets-v150.json", diff --git a/typescript/src/pkgman.ts b/typescript/src/pkgman.ts index de34b18..fcbf202 100644 --- a/typescript/src/pkgman.ts +++ b/typescript/src/pkgman.ts @@ -17,6 +17,12 @@ import cliProgress, { type Options as BarOptions } from "cli-progress"; import prettyBytes from "pretty-bytes"; import { parse as parseYaml } from "yaml"; import { CONSTRAINTS, LIBRARY_VERSION } from "./__version__.js"; +import { + effectivePin, + pinMatches, + pinSpec, + warnIfUnpaired, +} from "./browser-pin.js"; import { CamoufoxNotInstalled, CorruptedDownload, @@ -136,8 +142,13 @@ function expandUser(p: string): string { /** * Parse a semver string into a comparable tuple. */ -function parseSemver(version: string): number[] { - const parts = version.replace(/^[\^~]+/, "").split("."); +export function parseSemver(version: string): number[] { + // A prerelease compares as its release: 0.5.8b1 (PEP 440) and 0.5.8-beta.1 + // (semver) are both 0.5.8, exactly as in the Python twin. + const release = version + .replace(/^[\^~]+/, "") + .split(/(?<=\d)(?:[-+]|(?:a|b|rc|dev|\.dev|\.post)\d*)/)[0]; + const parts = release.split("."); // int() semantics: the whole part must be an integer ("1a" -> 0). const out = parts.map((part) => /^\s*[+-]?\d+\s*$/.test(part) ? Number.parseInt(part, 10) : 0, @@ -731,8 +742,22 @@ export class CamoufoxFetcher extends GitHubDownloader { if (!match?.groups) return null; const version = new Version(match.groups.build, match.groups.version); + // A released library installs exactly the browser it was released with + // (browser-pin.ts), unless the user explicitly chose something else. + const pin = effectivePin(loadConfig()); + if ( + pin && + !pinMatches( + pin, + this.repoConfig.name, + match.groups.version, + match.groups.build, + ) + ) { + return null; + } const isPrerelease = Boolean(release?.prerelease) || version.isAlpha; - if (!this.repoConfig.isVersionSupported(version, isPrerelease)) { + if (!pin && !this.repoConfig.isVersionSupported(version, isPrerelease)) { return null; } @@ -922,16 +947,16 @@ export function installedVerStr(fromDir?: string): string { } const active = getActivePath(); - if (active === null) { - const config = loadConfig(); - const pinned = config.pinned; - const channel = config.channel || getDefaultChannel(); - const activeDisplay = pinned ? `${channel}/${pinned}` : channel; - throw new CamoufoxNotInstalled( - `${activeDisplay} is not installed. Please run \`camoufox fetch\` to install.`, + if (active === null) throw notInstalledError(); + const version = Version.fromPath(active); + if (path.basename(path.dirname(path.dirname(active))) === "browsers") { + warnIfUnpaired( + path.basename(path.dirname(active)), + version.version ?? "", + version.build, ); } - return Version.fromPath(active).fullString; + return version.fullString; } /** @@ -952,13 +977,19 @@ export function rootInstallSupported(): boolean { } } +/** What is missing -- the paired build, a pin, or a channel -- and how to get it. */ function notInstalledError(): CamoufoxNotInstalled { const config = loadConfig(); + const pin = effectivePin(config); const pinned = config.pinned; const channel = config.channel || getDefaultChannel(); - const activeDisplay = pinned ? `${channel}/${pinned}` : channel; + const missing = pin + ? `${pin.repoName} ${pinSpec(pin)}, the browser this camoufox release pairs with,` + : pinned + ? `${channel}/${pinned}` + : channel; return new CamoufoxNotInstalled( - `${activeDisplay} is not installed. Please run \`camoufox fetch\` to install.`, + `${missing} is not installed. Please run \`camoufox fetch\` to install.`, ); } @@ -988,6 +1019,8 @@ function resolveInstalledPath(downloadIfMissing: boolean): string | null { } else if (rootInstallSupported()) { return INSTALL_DIR; } else if (!downloadIfMissing) { + // Other builds are installed, but not the one this release pairs with. + if (effectivePin(loadConfig())) throw notInstalledError(); throw new UnsupportedVersion("Camoufox executable is outdated."); } return null; diff --git a/typescript/tests/browser-pin.test.ts b/typescript/tests/browser-pin.test.ts new file mode 100644 index 0000000..3fdd08e --- /dev/null +++ b/typescript/tests/browser-pin.test.ts @@ -0,0 +1,195 @@ +/** + * Mirrors pythonlib/tests/test_browser_pin.py: a released library runs the + * browser it was released with, and nothing else by default. + * + * INSTALL_DIR is computed at import from XDG_CACHE_HOME, so every test points + * that at a fresh directory and re-imports the modules. + */ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const PIN = { + tag: "v156.0.1-beta.33", + repo: "daijro/camoufox", + repo_name: "Official", + version: "156.0.1", + build: "beta.33", +}; + +let tmp: string; +let savedXdg: string | undefined; + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "camoufox-pin-")); + savedXdg = process.env.XDG_CACHE_HOME; + process.env.XDG_CACHE_HOME = tmp; + vi.resetModules(); +}); + +afterEach(() => { + if (savedXdg === undefined) delete process.env.XDG_CACHE_HOME; + else process.env.XDG_CACHE_HOME = savedXdg; + fs.rmSync(tmp, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +async function setup(opts: { + pin: object; + installed: string[]; + config: Record; +}) { + const pinMod = await import("../src/browser-pin.js"); + const pkgman = await import("../src/pkgman.js"); + const multiversion = await import("../src/multiversion.js"); + const exc = await import("../src/exceptions.js"); + const root = pkgman.INSTALL_DIR; + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync(path.join(root, ".0.5_FLAG"), ""); + fs.writeFileSync(path.join(root, "repo_cache.json"), "{}"); + fs.writeFileSync(path.join(root, "config.json"), JSON.stringify(opts.config)); + for (const spec of opts.installed) { + const dash = spec.indexOf("-"); + const dir = path.join(root, "browsers", "official", spec); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, "version.json"), + JSON.stringify({ + version: spec.slice(0, dash), + build: spec.slice(dash + 1), + }), + ); + } + const pinFile = path.join(tmp, "browser-pin.json"); + fs.writeFileSync(pinFile, JSON.stringify(opts.pin)); + pinMod.browserPinDeps.file = pinFile; + pinMod.resetUnpairedWarning(); + return { pinMod, pkgman, multiversion, exc, root }; +} + +describe("the pin file", () => { + it.each([ + "{}", + '{"tag": null}', + "", + "not json", + ])("an empty or unreadable pin (%j) pins nothing", async (content) => { + const { loadPin } = await import("../src/browser-pin.js"); + const f = path.join(tmp, "p.json"); + fs.writeFileSync(f, content); + expect(loadPin(f)).toBeNull(); + expect(loadPin(path.join(tmp, "absent.json"))).toBeNull(); + }); + + it("the checked-in pin is empty: only the release workflow writes one", () => { + const file = path.resolve( + __dirname, + "../../pythonlib/camoufox/browser-pin.json", + ); + expect(JSON.parse(fs.readFileSync(file, "utf-8"))).toEqual({}); + }); +}); + +describe("launch", () => { + it("uses the paired build even when another is marked active", async () => { + const { pkgman, multiversion, root } = await setup({ + pin: PIN, + installed: ["156.0.1-beta.33", "157.0-beta.34"], + config: { active_version: "browsers/official/157.0-beta.34" }, + }); + expect(multiversion.getActivePath()).toBe( + path.join(root, "browsers/official/156.0.1-beta.33"), + ); + expect(pkgman.installedVerStr()).toBe("156.0.1-beta.33"); + }); + + it("does not fall back to a newer build when the paired one is missing", async () => { + const { pkgman, multiversion, exc } = await setup({ + pin: PIN, + installed: ["157.0-beta.34"], + config: {}, + }); + expect(multiversion.getActivePath()).toBeNull(); + expect(() => pkgman.installedVerStr()).toThrow(exc.CamoufoxNotInstalled); + expect(() => pkgman.installedVerStr()).toThrow(/156\.0\.1-beta\.33/); + }); + + it("reports a missing paired build as not installed, not outdated", async () => { + const { pkgman, exc } = await setup({ + pin: PIN, + installed: ["157.0-beta.34"], + config: {}, + }); + expect(() => pkgman.camoufoxPath()).toThrow(exc.CamoufoxNotInstalled); + expect(() => pkgman.camoufoxPath()).toThrow(/pairs with/); + }); + + it("keeps an explicit choice and warns about it once", async () => { + const { pkgman, multiversion, root } = await setup({ + pin: PIN, + installed: ["156.0.1-beta.33", "157.0-beta.34"], + config: { + channel: "official/stable", + active_version: "browsers/official/157.0-beta.34", + }, + }); + const emit = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + expect(multiversion.getActivePath()).toBe( + path.join(root, "browsers/official/157.0-beta.34"), + ); + expect(pkgman.installedVerStr()).toBe("157.0-beta.34"); + pkgman.installedVerStr(); + expect(emit).toHaveBeenCalledTimes(1); + expect(String(emit.mock.calls[0][0])).toContain("v156.0.1-beta.33"); + }); + + it("does not warn when the explicit choice is the paired build", async () => { + const { pkgman } = await setup({ + pin: PIN, + installed: ["156.0.1-beta.33"], + config: { + pinned: "156.0.1-beta.33", + channel: "official/prerelease", + active_version: "browsers/official/156.0.1-beta.33", + }, + }); + const emit = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + expect(pkgman.installedVerStr()).toBe("156.0.1-beta.33"); + expect(emit).not.toHaveBeenCalled(); + }); + + it("leaves the channel logic unchanged without a pin", async () => { + const { multiversion, root } = await setup({ + pin: {}, + installed: ["156.0.1-beta.33", "157.0-beta.34"], + config: { active_version: "browsers/official/157.0-beta.34" }, + }); + expect(multiversion.getActivePath()).toBe( + path.join(root, "browsers/official/157.0-beta.34"), + ); + }); +}); + +describe("the fetcher", () => { + it.each([ + [{}, false], + [{ channel: "official/stable" }, true], + ])("with config %j accepts only the paired asset unless chosen otherwise", async (config, acceptsNewer) => { + const { pkgman } = await setup({ pin: PIN, installed: [], config }); + const fetcher = Object.create(pkgman.CamoufoxFetcher.prototype); + fetcher.repoConfig = pkgman.RepoConfig.getDefault(); + fetcher.pattern = fetcher.repoConfig.buildPattern("lin", "x86_64"); + const asset = (version: string, build: string) => ({ + name: `camoufox-${version}-${build}-lin.x86_64.zip`, + browser_download_url: `https://example.invalid/${version}-${build}.zip`, + }); + expect( + fetcher.checkAsset(asset("156.0.1", "beta.33"), { prerelease: true }), + ).not.toBeNull(); + const newer = fetcher.checkAsset(asset("157.0", "beta.34"), { + prerelease: false, + }); + expect(newer !== null).toBe(acceptsNewer); + }); +}); diff --git a/typescript/tests/pkgman.test.ts b/typescript/tests/pkgman.test.ts index b095b0a..e5f1d14 100644 --- a/typescript/tests/pkgman.test.ts +++ b/typescript/tests/pkgman.test.ts @@ -176,3 +176,19 @@ describe("AvailableVersion.toMetadata", () => { ); }); }); + +describe("parseSemver", () => { + // Mirrors test_prerelease_library_versions_parse_as_their_release. + it.each([ + ["0.5.8", [0, 5, 8]], + ["0.5.8b1", [0, 5, 8]], + ["0.5.8rc2", [0, 5, 8]], + ["0.5.8.dev3", [0, 5, 8]], + ["0.5.8-beta.1", [0, 5, 8]], + ["^0.5.0", [0, 5, 0]], + ["1", [1, 0, 0]], + ])("%s parses as its release", async (raw, expected) => { + const { parseSemver } = await import("../src/pkgman.js"); + expect(parseSemver(raw)).toEqual(expected); + }); +});