From b20cdc4f93fded018d0ee7d71ef80a3654992a83 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 27 May 2026 13:43:42 -0700 Subject: [PATCH] ci: fix pypi publish on mac/windows/arm (#3449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The python-v0.32.0 publish run failed on every build matrix entry. Three independent issues: 1. **Mac and Windows**: `pypa/gh-action-pypi-publish` only runs on Linux, but was being called inline from each build job. 2. **Linux (all arches)**: `pypa/gh-action-pypi-publish` derives its docker image name from `github.action_repository`, which is empty when the action is invoked from inside a composite action (actions/runner#2473 — pypa's own `action.yml` references this bug). It falls back to `github.repository`, generating `docker://ghcr.io/lancedb/lancedb:`, which doesn't exist → `denied`. Only the ARM matrix entry surfaced this because it failed first and cancel-cascaded the rest. 3. **Windows**: `upload-artifact` in `build_windows_wheel` pointed at `python\target\wheels`, but maturin writes to the workspace-root `target/wheels`. The artifact was always empty. Also, `pypi-publish.yml` passed a `vcpkg_token` input that the composite doesn't declare. ## Changes - Build jobs (linux/mac/windows) now upload their wheels as `actions/upload-artifact` artifacts. - New Linux `publish` job downloads all wheel artifacts and runs the Fury or PyPA publish step directly (not via a composite), so `github.action_repository` resolves correctly. - Delete the unused `upload_wheel` composite action. - Drop the broken upload-artifact step inside `build_windows_wheel`. - Remove the bogus `vcpkg_token` input. - Fury upload now loops over all wheels instead of just the first. - Bump `actions/checkout`, `actions/upload-artifact`, `actions/download-artifact` to current major versions (Node 24) to clear deprecation warnings. - Bump Windows job timeout 60 → 90 minutes; previous run was cancel-timing-out on a 60m cap. - Use `rust-lld` as the Windows MSVC linker via `CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER`. `link.exe` is single-threaded and the long pole on Windows builds. Fixes #3445 ## Test plan - [x] Open this PR — `paths` filter triggers a dry-run build on all three platforms. - [x] Verify all three builds produce wheels. - [x] Confirm the `pypa/gh-action-pypi-publish` container actually starts (the actions/runner#2473 bug) via the `publish-dry-run` job pointed at TestPyPI. - [x] **REMOVE BEFORE MERGE**: drop the `publish-dry-run` job and the now-redundant `actions/upload-artifact` runs on PRs (currently always-on so the dry-run has wheels to publish). - [ ] After merge, cherry-pick onto `python-v0.32.0` and force-push the tag to re-trigger the publish. --- .../workflows/build_windows_wheel/action.yml | 4 - .github/workflows/pypi-publish.yml | 100 ++++++++++++++---- .github/workflows/upload_wheel/action.yml | 34 ------ 3 files changed, 77 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/upload_wheel/action.yml diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml index 742b187b2..18f39818f 100644 --- a/.github/workflows/build_windows_wheel/action.yml +++ b/.github/workflows/build_windows_wheel/action.yml @@ -29,7 +29,3 @@ runs: args: ${{ inputs.args }} docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'" working-directory: python - - uses: actions/upload-artifact@v4 - with: - name: windows-wheels - path: python\target\wheels diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index ca6e3219b..00289b383 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -8,6 +8,9 @@ on: # This should trigger a dry run (we skip the final publish step) paths: - .github/workflows/pypi-publish.yml + - .github/workflows/build_linux_wheel/action.yml + - .github/workflows/build_mac_wheel/action.yml + - .github/workflows/build_windows_wheel/action.yml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock @@ -21,9 +24,6 @@ jobs: linux: name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }} timeout-minutes: 60 - permissions: - id-token: write - contents: read strategy: matrix: config: @@ -46,7 +46,7 @@ jobs: runner: ubuntu-2404-8x-arm64 runs-on: ${{ matrix.config.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -60,15 +60,14 @@ jobs: args: "--release --strip ${{ matrix.config.extra_args }}" arm-build: ${{ matrix.config.platform == 'aarch64' }} manylinux: ${{ matrix.config.manylinux }} - - uses: ./.github/workflows/upload_wheel + - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - fury_token: ${{ secrets.FURY_TOKEN }} + name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }} + path: target/wheels/lancedb-*.whl + if-no-files-found: error mac: timeout-minutes: 90 - permissions: - id-token: write - contents: read runs-on: ${{ matrix.config.runner }} strategy: matrix: @@ -78,7 +77,7 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: 10.15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -90,18 +89,21 @@ jobs: with: python-minor-version: 10 args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels" - - uses: ./.github/workflows/upload_wheel + - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - fury_token: ${{ secrets.FURY_TOKEN }} + name: wheels-mac-${{ matrix.config.target }} + path: target/wheels/lancedb-*.whl + if-no-files-found: error windows: - timeout-minutes: 60 - permissions: - id-token: write - contents: read + timeout-minutes: 90 runs-on: windows-latest + env: + # link.exe is single-threaded and the long pole on Windows builds. Use + # rustc's bundled lld-link instead. + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -113,18 +115,70 @@ jobs: with: python-minor-version: 10 args: "--release --strip" - vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }} - - uses: ./.github/workflows/upload_wheel + - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - fury_token: ${{ secrets.FURY_TOKEN }} + name: wheels-windows + path: target/wheels/lancedb-*.whl + if-no-files-found: error + publish: + name: Publish wheels + if: startsWith(github.ref, 'refs/tags/python-v') + needs: [linux, mac, windows] + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - name: Download wheel artifacts + uses: actions/download-artifact@v8 + with: + pattern: wheels-* + path: target/wheels + merge-multiple: true + - name: List wheels + run: ls -la target/wheels + - name: Choose repo + id: choose_repo + run: | + if [[ ${{ github.ref }} == *beta* ]]; then + echo "repo=fury" >> $GITHUB_OUTPUT + else + echo "repo=pypi" >> $GITHUB_OUTPUT + fi + - name: Publish to Fury + if: steps.choose_repo.outputs.repo == 'fury' + env: + FURY_TOKEN: ${{ secrets.FURY_TOKEN }} + run: | + shopt -s nullglob + WHEELS=(target/wheels/lancedb-*.whl) + if [[ ${#WHEELS[@]} -eq 0 ]]; then + echo "No wheels found in target/wheels/" >&2 + exit 1 + fi + for WHEEL in "${WHEELS[@]}"; do + echo "Uploading $WHEEL to Fury" + curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/" + done + # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a + # workflow file, not from inside a composite action. When called from a + # composite, `github.action_repository` is empty (actions/runner#2473) + # and the action falls back to `github.repository`, producing a bogus + # `docker://ghcr.io/:` image reference that GHA tries to pull. + - name: Publish to PyPI + if: steps.choose_repo.outputs.repo == 'pypi' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels/ gh-release: if: startsWith(github.ref, 'refs/tags/python-v') runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -187,13 +241,13 @@ jobs: report-failure: name: Report Workflow Failure runs-on: ubuntu-latest - needs: [linux, mac, windows] + needs: [linux, mac, windows, publish] permissions: contents: read issues: write if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v') steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: ./.github/actions/create-failure-issue with: job-results: ${{ toJSON(needs) }} diff --git a/.github/workflows/upload_wheel/action.yml b/.github/workflows/upload_wheel/action.yml deleted file mode 100644 index 8bcdb7a88..000000000 --- a/.github/workflows/upload_wheel/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: upload-wheel - -description: "Upload wheels to Pypi" -inputs: - fury_token: - required: true - description: "release token for the fury repo" - -runs: - using: "composite" - steps: - - name: Choose repo - shell: bash - id: choose_repo - run: | - if [[ ${{ github.ref }} == *beta* ]]; then - echo "repo=fury" >> $GITHUB_OUTPUT - else - echo "repo=pypi" >> $GITHUB_OUTPUT - fi - - name: Publish to Fury - if: steps.choose_repo.outputs.repo == 'fury' - shell: bash - env: - FURY_TOKEN: ${{ inputs.fury_token }} - run: | - WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1) - echo "Uploading $WHEEL to Fury" - curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/ - - name: Publish to PyPI - if: steps.choose_repo.outputs.repo == 'pypi' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels/