lancedb/.github/workflows/pypi-publish.yml

name: PyPI Publish

on:
  push:
    tags:
      - 'v*'
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/pypi-publish.yml
      - .github/workflows/build_linux_wheel/action.yml
      - .github/workflows/build_mac_wheel/action.yml
      - .github/workflows/build_windows_wheel/action.yml
      - Cargo.toml # Change in dependency frequently breaks builds
      - Cargo.lock

env:
  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"

permissions:
  contents: read

# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  linux:
    name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
    timeout-minutes: 60
    strategy:
      matrix:
        config:
          - platform: x86_64
            manylinux: "2_28"
            extra_args: "--features fp16kernels"
            runner: ubuntu-22.04
            package_name: "lancedb"
            rustflags: ""
          # For successful fat LTO builds, we need a large runner to avoid OOM errors.
          - platform: aarch64
            manylinux: "2_28"
            extra_args: "--features fp16kernels"
            runner: ubuntu-2404-8x-arm64
            package_name: "lancedb"
            rustflags: ""
          # `lancedb-compat`: pre-Haswell-friendly variant for x86_64 hosts
          # without AVX2 (Sandy Bridge / Ivy Bridge / Westmere on Intel,
          # Bulldozer / Piledriver / Steamroller on AMD). Compiled at the
          # `x86-64-v2` baseline; runtime SIMD dispatch in lance-linalg
          # picks the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA
          # / AVX-512) at load time. Same import as `lancedb` -- conflicts
          # at install time, so users pick one.
          - platform: x86_64
            manylinux: "2_28"
            extra_args: ""
            runner: ubuntu-22.04
            package_name: "lancedb-compat"
            rustflags: "-Ctarget-cpu=x86-64-v2"
    runs-on: ${{ matrix.config.runner }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
      - name: Add swap for Arm fat LTO
        if: matrix.config.platform == 'aarch64'
        shell: bash
        run: |
          swap_file="$RUNNER_TEMP/lancedb-swap"
          sudo fallocate --length 16G "$swap_file"
          sudo chmod 600 "$swap_file"
          sudo mkswap "$swap_file"
          sudo swapon "$swap_file"
          free -h
      - uses: ./.github/workflows/build_linux_wheel
        with:
          python-minor-version: 10
          args: "--release --strip ${{ matrix.config.extra_args }}"
          arm-build: ${{ matrix.config.platform == 'aarch64' }}
          manylinux: ${{ matrix.config.manylinux }}
          package-name: ${{ matrix.config.package_name }}
          rustflags: ${{ matrix.config.rustflags }}
      - uses: actions/upload-artifact@v7
        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
          path: target/wheels/*.whl
          if-no-files-found: error
  mac:
    timeout-minutes: 90
    runs-on: ${{ matrix.config.runner }}
    strategy:
      matrix:
        config:
          - target: aarch64-apple-darwin
            runner: warp-macos-14-arm64-6x
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.15
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - uses: ./.github/workflows/build_mac_wheel
        with:
          python-minor-version: 10
          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
      - uses: actions/upload-artifact@v7
        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: wheels-mac-${{ matrix.config.target }}
          path: target/wheels/lancedb-*.whl
          if-no-files-found: error
  windows:
    timeout-minutes: 90
    runs-on: windows-latest
    env:
      # link.exe is single-threaded and the long pole on Windows builds. Use
      # rustc's bundled lld-link instead.
      CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
      # Keep the repository's fat LTO / 1 codegen unit release profile to leave
      # headroom below PyPI's file-size limit for Windows wheels.
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      # NOTE: caching cargo here would be a no-op. This workflow only runs on
      # tags and PRs, and GitHub only lets a run restore caches from its own ref
      # or the default branch -- so with no run on main there is nothing that
      # can populate an entry the release build would be allowed to read. Fixing
      # this needs a main/nightly trigger (which would also catch wheel-build
      # breakage before a release); the ~74 minutes here is otherwise dominated
      # by the fat-LTO link, which no cache avoids.
      - uses: ./.github/workflows/build_windows_wheel
        with:
          python-minor-version: 10
          args: "--release --strip"
      - uses: actions/upload-artifact@v7
        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: wheels-windows
          path: target/wheels/lancedb-*.whl
          if-no-files-found: error
  publish:
    name: Publish wheels
    if: startsWith(github.ref, 'refs/tags/v')
    needs: [linux, mac, windows]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@v6
      - name: Download wheel artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: wheels-*
          path: target/wheels
          merge-multiple: true
      - name: List wheels
        run: ls -la target/wheels
      - name: Choose repo
        id: choose_repo
        run: |
          if [[ ${{ github.ref }} == *beta* ]]; then
            echo "repo=fury" >> $GITHUB_OUTPUT
          else
            echo "repo=pypi" >> $GITHUB_OUTPUT
          fi
      - name: Publish to Fury
        if: steps.choose_repo.outputs.repo == 'fury'
        env:
          FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
        run: |
          shopt -s nullglob
          WHEELS=(target/wheels/*.whl)
          if [[ ${#WHEELS[@]} -eq 0 ]]; then
            echo "No wheels found in target/wheels/" >&2
            exit 1
          fi
          for WHEEL in "${WHEELS[@]}"; do
            echo "Uploading $WHEEL to Fury"
            curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
          done
      # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
      # workflow file, not from inside a composite action. When called from a
      # composite, `github.action_repository` is empty (actions/runner#2473)
      # and the action falls back to `github.repository`, producing a bogus
      # `docker://ghcr.io/<repo>:<ref>` image reference that GHA tries to pull.
      - name: Publish to PyPI
        if: steps.choose_repo.outputs.repo == 'pypi'
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: target/wheels/
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [linux, mac, windows, publish]
    permissions:
      contents: read
      issues: write
    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}