mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-27 14:06:14 +00:00
Jack YeandLance Release
197403e891
Consumers that provide a local Lance checkout cannot resolve the dependency graph because the SDK directly selects a vendored `lance-arrow`, producing two path packages with the same name and version. The Rust core now declares its upstream Git dependency separately so the consuming workspace can override that source consistently. Python continues inheriting the vendored workspace dependency, keeping the sliced-struct validity fix in Maturin source distributions. Standalone SDK builds still select that same fix through the existing root patch. Adds a downstream consumer CI regression that resolves a local Lance override and checks that exactly one canonical `lance-arrow` package is selected. The Lance version updater also updates the core declaration in stable, preview and local modes. Public APIs and query behavior are unchanged. The workspace declaration lint excludes path dependencies because the Python binding intentionally uses the bundled source. Duplicate and unused external declarations remain errors; advisory, license, banned-package and source checks continue covering the full dependency graph. --------- Co-authored-by: Lance Release <lance-dev@lancedb.com>
406 lines
14 KiB
YAML
406 lines
14 KiB
YAML
name: Rust
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
paths:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- rust-toolchain.toml
|
|
- deny.toml
|
|
- rust/**
|
|
- nodejs/Cargo.toml
|
|
- python/Cargo.toml
|
|
- .github/workflows/rust.yml
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# This env var is used by Swatinem/rust-cache@v2 for the cache
|
|
# key, so we set it to make sure it is always consistent.
|
|
CARGO_TERM_COLOR: always
|
|
RUST_BACKTRACE: "1"
|
|
|
|
jobs:
|
|
dependency-overrides:
|
|
name: Downstream Lance override
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 15
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
|
- name: Read the pinned Lance tag
|
|
id: lance
|
|
run: |
|
|
python3 - <<'PYTHON'
|
|
import os
|
|
import tomllib
|
|
from pathlib import Path
|
|
|
|
manifest = tomllib.loads(Path("Cargo.toml").read_text())
|
|
tag = manifest["workspace"]["dependencies"]["lance"]["tag"]
|
|
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
|
output.write(f"tag={tag}\n")
|
|
PYTHON
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
repository: lance-format/lance
|
|
ref: ${{ steps.lance.outputs.tag }}
|
|
path: local-lance
|
|
- name: Resolve a downstream workspace with a local Lance override
|
|
run: |
|
|
mkdir -p downstream/src
|
|
touch downstream/src/lib.rs
|
|
cat > downstream/Cargo.toml <<'TOML'
|
|
[package]
|
|
name = "lancedb-local-lance-consumer"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
|
|
[workspace]
|
|
|
|
[dependencies]
|
|
lancedb = { path = "../rust/lancedb", default-features = false }
|
|
lance-arrow = { path = "../local-lance/rust/lance-arrow" }
|
|
|
|
[patch."https://github.com/lance-format/lance.git"]
|
|
lance-arrow = { path = "../local-lance/rust/lance-arrow" }
|
|
TOML
|
|
cargo metadata --manifest-path downstream/Cargo.toml --format-version 1 > downstream/graph.json
|
|
python3 - <<'PYTHON'
|
|
import json
|
|
from pathlib import Path
|
|
|
|
graph = json.loads(Path("downstream/graph.json").read_text())
|
|
packages = [p for p in graph["packages"] if p["name"] == "lance-arrow"]
|
|
assert len(packages) == 1, [p["id"] for p in packages]
|
|
assert Path(packages[0]["manifest_path"]).resolve() == Path(
|
|
"local-lance/rust/lance-arrow/Cargo.toml"
|
|
).resolve()
|
|
PYTHON
|
|
|
|
lint:
|
|
timeout-minutes: 30
|
|
runs-on: ubuntu-24.04
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
env:
|
|
# Need up-to-date compilers for kernels
|
|
CC: clang-18
|
|
CXX: clang++-18
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
|
with:
|
|
components: rustfmt, clippy
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y protobuf-compiler libssl-dev
|
|
- name: Run format
|
|
run: cargo fmt --all -- --check
|
|
- name: Run clippy
|
|
run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
|
|
- name: Run clippy (without remote feature)
|
|
run: cargo clippy --profile ci --workspace --tests -- -D warnings
|
|
|
|
deny:
|
|
# Supply-chain checks: advisories, licenses, banned crates, and source
|
|
# restrictions. Configuration lives in `deny.toml` at the workspace root.
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: EmbarkStudios/cargo-deny-action@v2
|
|
with:
|
|
command: check advisories bans licenses sources
|
|
|
|
build-no-lock:
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
env:
|
|
# Need up-to-date compilers for kernels
|
|
CC: clang
|
|
CXX: clang++
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
# Building without a lock file often requires the latest Rust version since downstream
|
|
# dependencies may have updated their minimum Rust version.
|
|
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
|
with:
|
|
toolchain: "stable"
|
|
# Remove cargo.lock to force a fresh build
|
|
- name: Remove Cargo.lock
|
|
run: rm -f Cargo.lock
|
|
- uses: rui314/setup-mold@v1
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y protobuf-compiler libssl-dev
|
|
- name: Build all
|
|
run: |
|
|
cargo build --profile ci --benches --all-features --tests
|
|
|
|
linux:
|
|
timeout-minutes: 60
|
|
# To build all features, we need more disk space than is available
|
|
# on the free OSS github runner. This is mostly due to the the
|
|
# sentence-transformers feature.
|
|
runs-on: ubuntu-2404-4x-x64
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: rust
|
|
env:
|
|
# Need up-to-date compilers for kernels
|
|
CC: clang-18
|
|
CXX: clang++-18
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y protobuf-compiler libssl-dev
|
|
- uses: rui314/setup-mold@v1
|
|
- name: Make Swap
|
|
run: |
|
|
swapfile=/swapfile
|
|
min_swap_bytes=$((15 * 1024 * 1024 * 1024))
|
|
active_swap_bytes="$(sudo swapon --show=NAME,SIZE --bytes --noheadings | awk '$1 == "/swapfile" { print $2 }')"
|
|
if [ -n "$active_swap_bytes" ]; then
|
|
if [ "$active_swap_bytes" -ge "$min_swap_bytes" ]; then
|
|
echo "/swapfile is already active with enough space; skipping swap creation"
|
|
exit 0
|
|
fi
|
|
echo "/swapfile is already active but smaller than 16G; using /mnt/lancedb-swapfile"
|
|
swapfile=/mnt/lancedb-swapfile
|
|
fi
|
|
if sudo swapon --show=NAME --noheadings | grep -Fxq "$swapfile"; then
|
|
echo "$swapfile is already active; skipping swap creation"
|
|
exit 0
|
|
fi
|
|
sudo rm -f "$swapfile"
|
|
sudo fallocate -l 16G "$swapfile"
|
|
sudo chmod 600 "$swapfile"
|
|
sudo mkswap "$swapfile"
|
|
sudo swapon "$swapfile"
|
|
- name: Build
|
|
run: cargo build --profile ci --all-features --tests --locked --examples
|
|
- name: Run feature tests
|
|
run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
|
|
- name: Run examples
|
|
run: cargo run --profile ci --all-features --example simple --locked
|
|
|
|
remote:
|
|
timeout-minutes: 30
|
|
# Running this requires access to secrets, so skip if this is a PR from a
|
|
# fork. Keep it separate from the all-features build so Cargo does not
|
|
# retain both dependency graphs in one target directory.
|
|
if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
|
|
runs-on: ubuntu-2404-4x-x64
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: rust
|
|
env:
|
|
CC: clang-18
|
|
CXX: clang++-18
|
|
GH_TOKEN: ${{ github.token }}
|
|
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Remote tests use a different feature graph from the main Linux
|
|
# job. Cache downloads, but build into a fresh target directory.
|
|
cache-targets: false
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y protobuf-compiler libssl-dev
|
|
- uses: rui314/setup-mold@v1
|
|
- name: Run remote tests
|
|
run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
|
|
|
|
macos:
|
|
timeout-minutes: 60
|
|
strategy:
|
|
matrix:
|
|
mac-runner: ["macos-14", "macos-15"]
|
|
runs-on: "${{ matrix.mac-runner }}"
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: rust
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: CPU features
|
|
run: sysctl -a | grep cpu
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install dependencies
|
|
run: brew install protobuf
|
|
- name: Run tests
|
|
run: |
|
|
# Don't run the s3 integration tests since docker isn't available
|
|
# on this image.
|
|
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
|
|
| jq -r '.packages[] | .features | keys | .[]' \
|
|
| grep -v s3-test | sort | uniq | paste -s -d "," -`
|
|
# Run doctests before test binaries fill the runner disk. Examples are
|
|
# already built by the Linux job, so avoid retaining them here.
|
|
cargo test --profile ci --features $ALL_FEATURES --locked --doc
|
|
cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
|
|
|
|
windows:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- target: x86_64-pc-windows-msvc
|
|
runner: windows-2022
|
|
runs-on: ${{ matrix.runner }}
|
|
defaults:
|
|
run:
|
|
working-directory: rust/lancedb
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Set target
|
|
run: rustup target add ${{ matrix.target }}
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install Protoc v21.12
|
|
run: choco install --no-progress protoc
|
|
- name: Build
|
|
run: |
|
|
$env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
|
|
cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }}
|
|
- name: Run tests
|
|
run: |
|
|
$env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
|
|
# `--target` has to match the build step above. Without it cargo uses
|
|
# target/ci/ rather than target/<triple>/ci/ and rebuilds the entire
|
|
# dependency graph a second time.
|
|
cargo test --profile ci --features aws,remote --locked --target ${{ matrix.target }}
|
|
|
|
# Run the aarch64 Windows tests only on main so PRs wait for a single
|
|
# Windows platform. windows-11-arm is a standard runner, so it is free on
|
|
# public repos. Running natively lets the aarch64 tests actually execute --
|
|
# this job used to cross-compile them and then skip the test step, paying
|
|
# full codegen and link cost for a compile check.
|
|
windows-arm:
|
|
# Keep the historical name so the check shows up as before on main.
|
|
name: windows (aarch64-pc-windows-msvc, windows-11-arm)
|
|
if: github.ref == 'refs/heads/main'
|
|
runs-on: windows-11-arm
|
|
defaults:
|
|
run:
|
|
working-directory: rust/lancedb
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Set target
|
|
run: rustup target add aarch64-pc-windows-msvc
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Install Protoc v21.12
|
|
run: choco install --no-progress protoc
|
|
- name: Build
|
|
run: |
|
|
$env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
|
|
cargo build --profile ci --features aws,remote --tests --locked --target aarch64-pc-windows-msvc
|
|
- name: Run tests
|
|
run: |
|
|
$env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
|
|
# `--target` has to match the build step above. Without it cargo uses
|
|
# target/ci/ rather than target/<triple>/ci/ and rebuilds the entire
|
|
# dependency graph a second time.
|
|
cargo test --profile ci --features aws,remote --locked --target aarch64-pc-windows-msvc
|
|
|
|
msrv:
|
|
# Check the minimum supported Rust version
|
|
name: MSRV Check - Rust v${{ matrix.msrv }}
|
|
runs-on: ubuntu-24.04
|
|
strategy:
|
|
matrix:
|
|
msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
|
|
env:
|
|
# Need up-to-date compilers for kernels
|
|
CC: clang-18
|
|
CXX: clang++-18
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: true
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y protobuf-compiler libssl-dev
|
|
- name: Install ${{ matrix.msrv }}
|
|
uses: dtolnay/rust-toolchain@master
|
|
with:
|
|
toolchain: ${{ matrix.msrv }}
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
# Restore everywhere, but only save from main. Per-PR saves are
|
|
# unreadable outside their own branch anyway, since GitHub scopes
|
|
# caches to the creating ref.
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
- name: Downgrade dependencies
|
|
# aws-smithy-types 1.6.1+ requires rustc 1.94.1
|
|
run: cargo update -p aws-smithy-types --precise 1.5.0
|
|
- name: cargo +${{ matrix.msrv }} check
|
|
env:
|
|
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
|
|
run: cargo check --profile ci --workspace --tests --benches --all-features
|