diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml index b43f27354..8fa5b1f79 100644 --- a/.github/workflows/build_linux_wheel/action.yml +++ b/.github/workflows/build_linux_wheel/action.yml @@ -18,6 +18,14 @@ inputs: description: "The manylinux version to build for" required: false default: "2_17" + package-name: + description: "Override [project] name in python/pyproject.toml (e.g. 'lancedb-compat'). Default keeps 'lancedb'." + required: false + default: "lancedb" + rustflags: + description: "RUSTFLAGS for the build container, as a single whitespace-free token (e.g. '-Ctarget-cpu=x86-64-v2'). Empty leaves RUSTFLAGS unset, keeping the defaults from .cargo/config.toml." + required: false + default: "" runs: using: "composite" steps: @@ -27,6 +35,18 @@ runs: ARM_BUILD: ${{ inputs.arm-build }} run: | echo "ARM BUILD: $ARM_BUILD" + - name: Patch package name for variant build + if: ${{ inputs.package-name != 'lancedb' }} + shell: bash + env: + PACKAGE_NAME: ${{ inputs.package-name }} + run: | + # Swap the [project] name so this build produces e.g. lancedb-compat + # wheels. The package still installs files under the lancedb/ + # namespace -- import lancedb still works after pip install. + sed -i.bak 's/^name = "lancedb"$/name = "'"$PACKAGE_NAME"'"/' python/pyproject.toml + rm -f python/pyproject.toml.bak + grep '^name = ' python/pyproject.toml - name: Build x86_64 Manylinux wheel if: ${{ inputs.arm-build == 'false' }} uses: PyO3/maturin-action@v1 @@ -34,7 +54,7 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: x86_64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} @@ -51,7 +71,7 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: aarch64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index b15f2e6b2..720079292 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -22,7 +22,7 @@ permissions: jobs: linux: - name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }} + name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }} timeout-minutes: 60 strategy: matrix: @@ -31,11 +31,28 @@ jobs: manylinux: "2_28" extra_args: "--features fp16kernels" runner: ubuntu-22.04 + package_name: "lancedb" + rustflags: "" # For successful fat LTO builds, we need a large runner to avoid OOM errors. - platform: aarch64 manylinux: "2_28" extra_args: "--features fp16kernels" runner: ubuntu-2404-8x-arm64 + package_name: "lancedb" + rustflags: "" + # `lancedb-compat`: pre-Haswell-friendly variant for x86_64 hosts + # without AVX2 (Sandy Bridge / Ivy Bridge / Westmere on Intel, + # Bulldozer / Piledriver / Steamroller on AMD). Compiled at the + # `x86-64-v2` baseline; runtime SIMD dispatch in lance-linalg + # picks the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA + # / AVX-512) at load time. Same import as `lancedb` -- conflicts + # at install time, so users pick one. + - platform: x86_64 + manylinux: "2_28" + extra_args: "" + runner: ubuntu-22.04 + package_name: "lancedb-compat" + rustflags: "-Ctarget-cpu=x86-64-v2" runs-on: ${{ matrix.config.runner }} steps: - uses: actions/checkout@v6 @@ -52,11 +69,13 @@ jobs: args: "--release --strip ${{ matrix.config.extra_args }}" arm-build: ${{ matrix.config.platform == 'aarch64' }} manylinux: ${{ matrix.config.manylinux }} + package-name: ${{ matrix.config.package_name }} + rustflags: ${{ matrix.config.rustflags }} - uses: actions/upload-artifact@v7 if: startsWith(github.ref, 'refs/tags/python-v') with: - name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }} - path: target/wheels/lancedb-*.whl + name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }} + path: target/wheels/*.whl if-no-files-found: error mac: timeout-minutes: 90 @@ -145,7 +164,7 @@ jobs: FURY_TOKEN: ${{ secrets.FURY_TOKEN }} run: | shopt -s nullglob - WHEELS=(target/wheels/lancedb-*.whl) + WHEELS=(target/wheels/*.whl) if [[ ${#WHEELS[@]} -eq 0 ]]; then echo "No wheels found in target/wheels/" >&2 exit 1 diff --git a/python/README.md b/python/README.md index 81d567f04..550698500 100644 --- a/python/README.md +++ b/python/README.md @@ -8,6 +8,27 @@ A Python library for [LanceDB](https://github.com/lancedb/lancedb). pip install lancedb ``` +### Pre-Haswell x86_64 hosts: `lancedb-compat` + +The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C) for full performance on modern hardware. Pre-Haswell hosts — Intel Sandy Bridge / Ivy Bridge / Westmere; AMD Bulldozer / Piledriver / Steamroller — don't have AVX2 and crash with `Illegal instruction` at `import lancedb`. + +For those hosts, install the `lancedb-compat` package instead: + +```bash +pip install lancedb-compat +``` + +Same Python API (`import lancedb` works as usual). The compat wheel is compiled at the `x86-64-v2` baseline (Nehalem-class) and uses runtime SIMD dispatch in the embedded lance crate to pick the right kernel tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) at load time, so it still goes fast on modern hardware while running cleanly on the pre-Haswell silicon. Use `lance.simd_info()` from Python to verify which tier was selected. + +`lancedb` and `lancedb-compat` install to the same `lancedb/` namespace and conflict at install time. Pick one. To switch, `pip uninstall lancedb` first, then `pip install lancedb-compat` (or vice-versa). + +If you need a custom baseline (or `lancedb-compat` isn't yet published for your platform), build from source with the override: + +```bash +RUSTFLAGS="-C target-cpu=x86-64-v2" maturin build --release +pip install ./target/wheels/lancedb-*.whl +``` + ### Preview Releases Stable releases are created about every 2 weeks. For the latest features and bug fixes, you can install the preview release. These releases receive the same level of testing as stable releases, but are not guaranteed to be available for more than 6 months after they are released. Once your application is stable, we recommend switching to stable releases.