mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-28 22:46:14 +00:00
Jack YeandLance Release
197403e891
Consumers that provide a local Lance checkout cannot resolve the dependency graph because the SDK directly selects a vendored `lance-arrow`, producing two path packages with the same name and version. The Rust core now declares its upstream Git dependency separately so the consuming workspace can override that source consistently. Python continues inheriting the vendored workspace dependency, keeping the sliced-struct validity fix in Maturin source distributions. Standalone SDK builds still select that same fix through the existing root patch. Adds a downstream consumer CI regression that resolves a local Lance override and checks that exactly one canonical `lance-arrow` package is selected. The Lance version updater also updates the core declaration in stable, preview and local modes. Public APIs and query behavior are unchanged. The workspace declaration lint excludes path dependencies because the Python binding intentionally uses the bundled source. Duplicate and unused external declarations remain errors; advisory, license, banned-package and source checks continue covering the full dependency graph. --------- Co-authored-by: Lance Release <lance-dev@lancedb.com>
326 lines
12 KiB
YAML
326 lines
12 KiB
YAML
name: PyPI Publish
|
|
|
|
on:
|
|
push:
|
|
tags:
|
|
- 'v*'
|
|
pull_request:
|
|
# This should trigger a dry run (we skip the final publish step)
|
|
paths:
|
|
- .github/workflows/pypi-publish.yml
|
|
- .github/workflows/build_linux_wheel/action.yml
|
|
- .github/workflows/build_mac_wheel/action.yml
|
|
- .github/workflows/build_windows_wheel/action.yml
|
|
- .github/actions/setup-protoc/action.yml
|
|
- ci/check_cargo_lock.py
|
|
- ci/check_sdist.py
|
|
- python/pyproject.toml
|
|
- python/Cargo.toml
|
|
- python/vendor/**
|
|
- rust/lancedb/Cargo.toml
|
|
- Cargo.toml # Change in dependency frequently breaks builds
|
|
- Cargo.lock
|
|
|
|
env:
|
|
PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Without this, a force-push to a PR leaves the previous run going -- including
|
|
# a long fat-LTO Windows job and a billed arm64 wheel build.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
linux:
|
|
name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
|
timeout-minutes: 60
|
|
strategy:
|
|
matrix:
|
|
config:
|
|
- platform: x86_64
|
|
manylinux: "2_28"
|
|
extra_args: "--features fp16kernels"
|
|
runner: ubuntu-22.04
|
|
package_name: "lancedb"
|
|
rustflags: ""
|
|
# For successful fat LTO builds, we need a large runner to avoid OOM errors.
|
|
- platform: aarch64
|
|
manylinux: "2_28"
|
|
extra_args: "--features fp16kernels"
|
|
runner: ubuntu-2404-8x-arm64
|
|
package_name: "lancedb"
|
|
rustflags: ""
|
|
# `lancedb-compat`: pre-Haswell-friendly variant for x86_64 hosts
|
|
# without AVX2 (Sandy Bridge / Ivy Bridge / Westmere on Intel,
|
|
# Bulldozer / Piledriver / Steamroller on AMD). Compiled at the
|
|
# `x86-64-v2` baseline; runtime SIMD dispatch in lance-linalg
|
|
# picks the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA
|
|
# / AVX-512) at load time. Same import as `lancedb` -- conflicts
|
|
# at install time, so users pick one.
|
|
- platform: x86_64
|
|
manylinux: "2_28"
|
|
extra_args: ""
|
|
runner: ubuntu-22.04
|
|
package_name: "lancedb-compat"
|
|
rustflags: "-Ctarget-cpu=x86-64-v2"
|
|
runs-on: ${{ matrix.config.runner }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.10"
|
|
- name: Add swap for Arm fat LTO
|
|
if: matrix.config.platform == 'aarch64'
|
|
shell: bash
|
|
run: |
|
|
swap_file="$RUNNER_TEMP/lancedb-swap"
|
|
sudo fallocate --length 16G "$swap_file"
|
|
sudo chmod 600 "$swap_file"
|
|
sudo mkswap "$swap_file"
|
|
sudo swapon "$swap_file"
|
|
free -h
|
|
- uses: ./.github/workflows/build_linux_wheel
|
|
with:
|
|
python-minor-version: 10
|
|
args: "--release --strip ${{ matrix.config.extra_args }}"
|
|
arm-build: ${{ matrix.config.platform == 'aarch64' }}
|
|
manylinux: ${{ matrix.config.manylinux }}
|
|
package-name: ${{ matrix.config.package_name }}
|
|
rustflags: ${{ matrix.config.rustflags }}
|
|
- uses: actions/upload-artifact@v7
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
with:
|
|
name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
|
|
path: target/wheels/*.whl
|
|
if-no-files-found: error
|
|
mac:
|
|
timeout-minutes: 90
|
|
runs-on: ${{ matrix.config.runner }}
|
|
strategy:
|
|
matrix:
|
|
config:
|
|
- target: aarch64-apple-darwin
|
|
runner: warp-macos-14-arm64-6x
|
|
env:
|
|
MACOSX_DEPLOYMENT_TARGET: 10.15
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.13"
|
|
- uses: ./.github/workflows/build_mac_wheel
|
|
with:
|
|
python-minor-version: 10
|
|
args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
|
|
- uses: actions/upload-artifact@v7
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
with:
|
|
name: wheels-mac-${{ matrix.config.target }}
|
|
path: target/wheels/lancedb-*.whl
|
|
if-no-files-found: error
|
|
windows:
|
|
# The fat-LTO build of the `lancedb` crate does not finish within 90 minutes
|
|
# on the 4-core standard runner, so use the 8-core larger runner with extra
|
|
# headroom.
|
|
timeout-minutes: 150
|
|
runs-on: windows-2025-8x-x64
|
|
env:
|
|
# link.exe is single-threaded and the long pole on Windows builds. Use
|
|
# rustc's bundled lld-link instead.
|
|
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
|
|
# Keep the repository's fat LTO / 1 codegen unit release profile to leave
|
|
# headroom below PyPI's file-size limit for Windows wheels.
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.13"
|
|
# NOTE: caching cargo here would be a no-op. This workflow only runs on
|
|
# tags and PRs, and GitHub only lets a run restore caches from its own ref
|
|
# or the default branch -- so with no run on main there is nothing that
|
|
# can populate an entry the release build would be allowed to read. Fixing
|
|
# this needs a main/nightly trigger (which would also catch wheel-build
|
|
# breakage before a release); the build time here is otherwise dominated
|
|
# by the fat-LTO link, which no cache avoids.
|
|
- uses: ./.github/workflows/build_windows_wheel
|
|
with:
|
|
python-minor-version: 10
|
|
args: "--release --strip"
|
|
- uses: actions/upload-artifact@v7
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
with:
|
|
name: wheels-windows
|
|
path: target/wheels/lancedb-*.whl
|
|
if-no-files-found: error
|
|
sdist:
|
|
name: Python sdist
|
|
timeout-minutes: 60
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.12"
|
|
- name: Build sdist
|
|
uses: PyO3/maturin-action@v1
|
|
with:
|
|
maturin-version: "1.12.4"
|
|
command: sdist
|
|
working-directory: python
|
|
args: --out dist
|
|
# maturin ships the nodejs-only pins in Cargo.lock. Remove this step when
|
|
# the pinned maturin fixes https://github.com/PyO3/maturin/issues/2609.
|
|
- name: Sync Cargo.lock to the packaged workspace
|
|
shell: bash
|
|
run: |
|
|
# A repo lock in sync with every manifest means the prune only drops pins.
|
|
cargo metadata --locked --format-version 1 > /dev/null
|
|
mkdir "$RUNNER_TEMP/sdist-src"
|
|
tar -xzf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src"
|
|
lock=$(echo "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.lock)
|
|
cp "$lock" "$RUNNER_TEMP/Cargo.lock.before"
|
|
cargo metadata --format-version 1 \
|
|
--manifest-path "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.toml > /dev/null
|
|
python ci/check_cargo_lock.py "$RUNNER_TEMP/Cargo.lock.before" "$lock"
|
|
tar -czf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src" \
|
|
"$(basename python/dist/lancedb-*.tar.gz .tar.gz)"
|
|
- name: Install sdist check tools
|
|
shell: bash
|
|
run: python -m pip install twine==7.0.0 pydistcheck==0.11.3
|
|
- name: Check sdist size
|
|
shell: bash
|
|
run: |
|
|
pydistcheck --select distro-too-large-compressed \
|
|
--max-allowed-size-compressed 100M python/dist/lancedb-*.tar.gz
|
|
- name: Check sdist contents
|
|
shell: bash
|
|
run: python ci/check_sdist.py python/dist/lancedb-*.tar.gz
|
|
- name: Check sdist with twine
|
|
shell: bash
|
|
run: python -m twine check --strict python/dist/lancedb-*.tar.gz
|
|
- uses: ./.github/actions/setup-protoc
|
|
# Build the way `pip install lancedb` does when no wheel matches. The dev
|
|
# profile keeps the build within the runner's disk.
|
|
- name: Build wheel from sdist
|
|
shell: bash
|
|
env:
|
|
MATURIN_PEP517_ARGS: "--profile dev --locked"
|
|
CARGO_PROFILE_DEV_DEBUG: "0"
|
|
run: |
|
|
python -m pip wheel --no-deps --wheel-dir "$RUNNER_TEMP/sdist-wheel" \
|
|
python/dist/lancedb-*.tar.gz
|
|
- name: Smoke test wheel built from sdist
|
|
shell: bash
|
|
working-directory: ${{ runner.temp }}
|
|
run: |
|
|
python -m venv sdist-venv
|
|
sdist-venv/bin/pip install sdist-wheel/lancedb-*.whl
|
|
sdist-venv/bin/python -c '
|
|
import tempfile
|
|
import lancedb
|
|
db = lancedb.connect(tempfile.mkdtemp())
|
|
table = db.create_table("t", [{"vector": [1.0, 2.0], "id": 1}, {"vector": [3.0, 4.0], "id": 2}])
|
|
assert table.search([1.0, 2.0]).limit(1).to_list()[0]["id"] == 1
|
|
print("lancedb", lancedb.__version__, "built from sdist OK")
|
|
'
|
|
- uses: actions/upload-artifact@v7
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
with:
|
|
name: sdist
|
|
path: python/dist/lancedb-*.tar.gz
|
|
if-no-files-found: error
|
|
publish:
|
|
name: Publish wheels and sdist
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
needs: [linux, mac, windows, sdist]
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Download wheel artifacts
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: wheels-*
|
|
path: target/wheels
|
|
merge-multiple: true
|
|
- name: Download sdist artifact
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: sdist
|
|
path: target/wheels
|
|
- name: List packages
|
|
run: ls -la target/wheels
|
|
- name: Choose repo
|
|
id: choose_repo
|
|
run: |
|
|
if [[ ${{ github.ref }} == *beta* ]]; then
|
|
echo "repo=fury" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "repo=pypi" >> $GITHUB_OUTPUT
|
|
fi
|
|
- name: Publish to Fury
|
|
if: steps.choose_repo.outputs.repo == 'fury'
|
|
env:
|
|
FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
|
|
run: |
|
|
shopt -s nullglob
|
|
WHEELS=(target/wheels/*.whl)
|
|
SDISTS=(target/wheels/*.tar.gz)
|
|
if [[ ${#WHEELS[@]} -eq 0 ]]; then
|
|
echo "No wheels found in target/wheels/" >&2
|
|
exit 1
|
|
fi
|
|
if [[ ${#SDISTS[@]} -ne 1 ]]; then
|
|
echo "Expected one sdist in target/wheels/, found ${#SDISTS[@]}" >&2
|
|
exit 1
|
|
fi
|
|
for PACKAGE in "${WHEELS[@]}" "${SDISTS[@]}"; do
|
|
echo "Uploading $PACKAGE to Fury"
|
|
curl -f -F package=@"$PACKAGE" "https://$FURY_TOKEN@push.fury.io/lancedb/"
|
|
done
|
|
# NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
|
|
# workflow file, not from inside a composite action. When called from a
|
|
# composite, `github.action_repository` is empty (actions/runner#2473)
|
|
# and the action falls back to `github.repository`, producing a bogus
|
|
# `docker://ghcr.io/<repo>:<ref>` image reference that GHA tries to pull.
|
|
- name: Publish to PyPI
|
|
if: steps.choose_repo.outputs.repo == 'pypi'
|
|
uses: pypa/gh-action-pypi-publish@release/v1
|
|
with:
|
|
packages-dir: target/wheels/
|
|
report-failure:
|
|
name: Report Workflow Failure
|
|
runs-on: ubuntu-latest
|
|
needs: [linux, mac, windows, sdist, publish]
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: ./.github/actions/create-failure-issue
|
|
with:
|
|
job-results: ${{ toJSON(needs) }}
|
|
workflow-name: ${{ github.workflow }}
|