build(python): publish an sdist alongside the wheels (#4522)

Closes #4246.

lancedb publishes no source distribution, so a platform without a
prebuilt wheel (#4104, #4239) has nothing to install. This adds an sdist
to the Python release.

## The sdist job

The release workflow builds an sdist with the maturin version of the
wheel jobs. On a tag, the publish job uploads it with the wheels: to
PyPI, and to Fury for beta tags. The job also runs on the pull request
dry run, which now starts on a change to `python/pyproject.toml`.

## Checks

- The job builds a wheel from the sdist with `--locked` and smoke-tests
it in a fresh venv.
- `ci/check_sdist.py` fails if a tracked file of the Python package is
missing from the tarball.
- `twine check --strict` checks that the metadata renders. `pydistcheck`
enforces PyPI's 100 MiB size limit.
- The unit tests of both check scripts run in the CI scripts workflow.

## Cargo.lock

maturin ships the repository lockfile, which pins crates that only the
dropped `nodejs` member uses, so `cargo --locked` fails
(PyO3/maturin#2609). The job first runs `cargo metadata --locked` on the
repository workspace, so a lockfile that has drifted from `Cargo.toml`
fails there. It then removes the nodejs-only pins and repacks the sdist.
`ci/check_cargo_lock.py` fails if the prune changes any package it
keeps, or any dependency of one. The published file is the repacked one.

## Behavior change

After the first release with an sdist, pip on a platform with no current
wheel builds the newest version from source, which needs a Rust
toolchain and `protoc`. Before, pip installed the last version with a
wheel for that platform.

.github/workflows/ci-scripts.yml

@@ -5,11 +5,15 @@ on:
     branches:
       - main
     paths:
+      - ci/check_cargo_lock.py
+      - ci/check_sdist.py
       - ci/set_lance_version.py
       - ci/tests/**
       - .github/workflows/ci-scripts.yml
   pull_request:
     paths:
+      - ci/check_cargo_lock.py
+      - ci/check_sdist.py
       - ci/set_lance_version.py
       - ci/tests/**
       - .github/workflows/ci-scripts.yml

.github/workflows/pypi-publish.yml

@@ -12,6 +12,9 @@ on:
       - .github/workflows/build_mac_wheel/action.yml
       - .github/workflows/build_windows_wheel/action.yml
       - .github/actions/setup-protoc/action.yml
+      - ci/check_cargo_lock.py
+      - ci/check_sdist.py
+      - python/pyproject.toml
       - Cargo.toml # Change in dependency frequently breaks builds
       - Cargo.lock
 
@@ -161,10 +164,91 @@ jobs:
           name: wheels-windows
           path: target/wheels/lancedb-*.whl
           if-no-files-found: error
+  sdist:
+    name: Python sdist
+    timeout-minutes: 60
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          maturin-version: "1.12.4"
+          command: sdist
+          working-directory: python
+          args: --out dist
+      # maturin ships the nodejs-only pins in Cargo.lock. Remove this step when
+      # the pinned maturin fixes https://github.com/PyO3/maturin/issues/2609.
+      - name: Sync Cargo.lock to the packaged workspace
+        shell: bash
+        run: |
+          # A repo lock in sync with every manifest means the prune only drops pins.
+          cargo metadata --locked --format-version 1 > /dev/null
+          mkdir "$RUNNER_TEMP/sdist-src"
+          tar -xzf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src"
+          lock=$(echo "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.lock)
+          cp "$lock" "$RUNNER_TEMP/Cargo.lock.before"
+          cargo metadata --format-version 1 \
+            --manifest-path "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.toml > /dev/null
+          python ci/check_cargo_lock.py "$RUNNER_TEMP/Cargo.lock.before" "$lock"
+          tar -czf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src" \
+            "$(basename python/dist/lancedb-*.tar.gz .tar.gz)"
+      - name: Install sdist check tools
+        shell: bash
+        run: python -m pip install twine==7.0.0 pydistcheck==0.11.3
+      - name: Check sdist size
+        shell: bash
+        run: |
+          pydistcheck --select distro-too-large-compressed \
+            --max-allowed-size-compressed 100M python/dist/lancedb-*.tar.gz
+      - name: Check sdist contents
+        shell: bash
+        run: python ci/check_sdist.py python/dist/lancedb-*.tar.gz
+      - name: Check sdist with twine
+        shell: bash
+        run: python -m twine check --strict python/dist/lancedb-*.tar.gz
+      - uses: ./.github/actions/setup-protoc
+      # Build the way `pip install lancedb` does when no wheel matches. The dev
+      # profile keeps the build within the runner's disk.
+      - name: Build wheel from sdist
+        shell: bash
+        env:
+          MATURIN_PEP517_ARGS: "--profile dev --locked"
+          CARGO_PROFILE_DEV_DEBUG: "0"
+        run: |
+          python -m pip wheel --no-deps --wheel-dir "$RUNNER_TEMP/sdist-wheel" \
+            python/dist/lancedb-*.tar.gz
+      - name: Smoke test wheel built from sdist
+        shell: bash
+        working-directory: ${{ runner.temp }}
+        run: |
+          python -m venv sdist-venv
+          sdist-venv/bin/pip install sdist-wheel/lancedb-*.whl
+          sdist-venv/bin/python -c '
+          import tempfile
+          import lancedb
+          db = lancedb.connect(tempfile.mkdtemp())
+          table = db.create_table("t", [{"vector": [1.0, 2.0], "id": 1}, {"vector": [3.0, 4.0], "id": 2}])
+          assert table.search([1.0, 2.0]).limit(1).to_list()[0]["id"] == 1
+          print("lancedb", lancedb.__version__, "built from sdist OK")
+          '
+      - uses: actions/upload-artifact@v7
+        if: startsWith(github.ref, 'refs/tags/v')
+        with:
+          name: sdist
+          path: python/dist/lancedb-*.tar.gz
+          if-no-files-found: error
   publish:
-    name: Publish wheels
+    name: Publish wheels and sdist
     if: startsWith(github.ref, 'refs/tags/v')
-    needs: [linux, mac, windows]
+    needs: [linux, mac, windows, sdist]
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -177,7 +261,12 @@ jobs:
           pattern: wheels-*
           path: target/wheels
           merge-multiple: true
-      - name: List wheels
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v8
+        with:
+          name: sdist
+          path: target/wheels
+      - name: List packages
         run: ls -la target/wheels
       - name: Choose repo
         id: choose_repo
@@ -194,13 +283,18 @@ jobs:
         run: |
           shopt -s nullglob
           WHEELS=(target/wheels/*.whl)
+          SDISTS=(target/wheels/*.tar.gz)
           if [[ ${#WHEELS[@]} -eq 0 ]]; then
             echo "No wheels found in target/wheels/" >&2
             exit 1
           fi
-          for WHEEL in "${WHEELS[@]}"; do
-            echo "Uploading $WHEEL to Fury"
-            curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
+          if [[ ${#SDISTS[@]} -ne 1 ]]; then
+            echo "Expected one sdist in target/wheels/, found ${#SDISTS[@]}" >&2
+            exit 1
+          fi
+          for PACKAGE in "${WHEELS[@]}" "${SDISTS[@]}"; do
+            echo "Uploading $PACKAGE to Fury"
+            curl -f -F package=@"$PACKAGE" "https://$FURY_TOKEN@push.fury.io/lancedb/"
           done
       # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
       # workflow file, not from inside a composite action. When called from a
@@ -215,7 +309,7 @@ jobs:
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
-    needs: [linux, mac, windows, publish]
+    needs: [linux, mac, windows, sdist, publish]
     permissions:
       contents: read
       issues: write