diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml index 742b187b2..18f39818f 100644 --- a/.github/workflows/build_windows_wheel/action.yml +++ b/.github/workflows/build_windows_wheel/action.yml @@ -29,7 +29,3 @@ runs: args: ${{ inputs.args }} docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'" working-directory: python - - uses: actions/upload-artifact@v4 - with: - name: windows-wheels - path: python\target\wheels diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index ca6e3219b..00289b383 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -8,6 +8,9 @@ on: # This should trigger a dry run (we skip the final publish step) paths: - .github/workflows/pypi-publish.yml + - .github/workflows/build_linux_wheel/action.yml + - .github/workflows/build_mac_wheel/action.yml + - .github/workflows/build_windows_wheel/action.yml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock @@ -21,9 +24,6 @@ jobs: linux: name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }} timeout-minutes: 60 - permissions: - id-token: write - contents: read strategy: matrix: config: @@ -46,7 +46,7 @@ jobs: runner: ubuntu-2404-8x-arm64 runs-on: ${{ matrix.config.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -60,15 +60,14 @@ jobs: args: "--release --strip ${{ matrix.config.extra_args }}" arm-build: ${{ matrix.config.platform == 'aarch64' }} manylinux: ${{ matrix.config.manylinux }} - - uses: ./.github/workflows/upload_wheel + - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - fury_token: ${{ secrets.FURY_TOKEN }} + name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }} + path: target/wheels/lancedb-*.whl + if-no-files-found: error mac: timeout-minutes: 90 - permissions: - id-token: write - contents: read runs-on: ${{ matrix.config.runner }} strategy: matrix: @@ -78,7 +77,7 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: 10.15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -90,18 +89,21 @@ jobs: with: python-minor-version: 10 args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels" - - uses: ./.github/workflows/upload_wheel + - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - fury_token: ${{ secrets.FURY_TOKEN }} + name: wheels-mac-${{ matrix.config.target }} + path: target/wheels/lancedb-*.whl + if-no-files-found: error windows: - timeout-minutes: 60 - permissions: - id-token: write - contents: read + timeout-minutes: 90 runs-on: windows-latest + env: + # link.exe is single-threaded and the long pole on Windows builds. Use + # rustc's bundled lld-link instead. + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -113,18 +115,70 @@ jobs: with: python-minor-version: 10 args: "--release --strip" - vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }} - - uses: ./.github/workflows/upload_wheel + - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - fury_token: ${{ secrets.FURY_TOKEN }} + name: wheels-windows + path: target/wheels/lancedb-*.whl + if-no-files-found: error + publish: + name: Publish wheels + if: startsWith(github.ref, 'refs/tags/python-v') + needs: [linux, mac, windows] + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - name: Download wheel artifacts + uses: actions/download-artifact@v8 + with: + pattern: wheels-* + path: target/wheels + merge-multiple: true + - name: List wheels + run: ls -la target/wheels + - name: Choose repo + id: choose_repo + run: | + if [[ ${{ github.ref }} == *beta* ]]; then + echo "repo=fury" >> $GITHUB_OUTPUT + else + echo "repo=pypi" >> $GITHUB_OUTPUT + fi + - name: Publish to Fury + if: steps.choose_repo.outputs.repo == 'fury' + env: + FURY_TOKEN: ${{ secrets.FURY_TOKEN }} + run: | + shopt -s nullglob + WHEELS=(target/wheels/lancedb-*.whl) + if [[ ${#WHEELS[@]} -eq 0 ]]; then + echo "No wheels found in target/wheels/" >&2 + exit 1 + fi + for WHEEL in "${WHEELS[@]}"; do + echo "Uploading $WHEEL to Fury" + curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/" + done + # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a + # workflow file, not from inside a composite action. When called from a + # composite, `github.action_repository` is empty (actions/runner#2473) + # and the action falls back to `github.repository`, producing a bogus + # `docker://ghcr.io/:` image reference that GHA tries to pull. + - name: Publish to PyPI + if: steps.choose_repo.outputs.repo == 'pypi' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels/ gh-release: if: startsWith(github.ref, 'refs/tags/python-v') runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 lfs: true @@ -187,13 +241,13 @@ jobs: report-failure: name: Report Workflow Failure runs-on: ubuntu-latest - needs: [linux, mac, windows] + needs: [linux, mac, windows, publish] permissions: contents: read issues: write if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v') steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: ./.github/actions/create-failure-issue with: job-results: ${{ toJSON(needs) }} diff --git a/.github/workflows/upload_wheel/action.yml b/.github/workflows/upload_wheel/action.yml deleted file mode 100644 index 8bcdb7a88..000000000 --- a/.github/workflows/upload_wheel/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: upload-wheel - -description: "Upload wheels to Pypi" -inputs: - fury_token: - required: true - description: "release token for the fury repo" - -runs: - using: "composite" - steps: - - name: Choose repo - shell: bash - id: choose_repo - run: | - if [[ ${{ github.ref }} == *beta* ]]; then - echo "repo=fury" >> $GITHUB_OUTPUT - else - echo "repo=pypi" >> $GITHUB_OUTPUT - fi - - name: Publish to Fury - if: steps.choose_repo.outputs.repo == 'fury' - shell: bash - env: - FURY_TOKEN: ${{ inputs.fury_token }} - run: | - WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1) - echo "Uploading $WHEEL to Fury" - curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/ - - name: Publish to PyPI - if: steps.choose_repo.outputs.repo == 'pypi' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: target/wheels/