lancedb/.github/workflows/rust.yml

name: Rust

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - Cargo.toml
      - Cargo.lock
      - rust-toolchain.toml
      - deny.toml
      - rust/**
      - docs/web-tests/blob-v2/rs/**
      - nodejs/Cargo.toml
      - python/Cargo.toml
      - .github/workflows/rust.yml

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: "1"

jobs:
  dependency-overrides:
    name: Downstream Lance override
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v6
      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Read the pinned Lance tag
        id: lance
        run: |
          python3 - <<'PYTHON'
          import os
          import tomllib
          from pathlib import Path

          manifest = tomllib.loads(Path("Cargo.toml").read_text())
          tag = manifest["workspace"]["dependencies"]["lance"]["tag"]
          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
              output.write(f"tag={tag}\n")
          PYTHON
      - uses: actions/checkout@v6
        with:
          repository: lance-format/lance
          ref: ${{ steps.lance.outputs.tag }}
          path: local-lance
      - name: Resolve a downstream workspace with a local Lance override
        run: |
          mkdir -p downstream/src
          touch downstream/src/lib.rs
          cat > downstream/Cargo.toml <<'TOML'
          [package]
          name = "lancedb-local-lance-consumer"
          version = "0.1.0"
          edition = "2024"

          [workspace]

          [dependencies]
          lancedb = { path = "../rust/lancedb", default-features = false }
          lance-arrow = { path = "../local-lance/rust/lance-arrow" }

          [patch."https://github.com/lance-format/lance.git"]
          lance-arrow = { path = "../local-lance/rust/lance-arrow" }
          TOML
          cargo metadata --manifest-path downstream/Cargo.toml --format-version 1 > downstream/graph.json
          python3 - <<'PYTHON'
          import json
          from pathlib import Path

          graph = json.loads(Path("downstream/graph.json").read_text())
          packages = [p for p in graph["packages"] if p["name"] == "lance-arrow"]
          assert len(packages) == 1, [p["id"] for p in packages]
          assert Path(packages[0]["manifest_path"]).resolve() == Path(
              "local-lance/rust/lance-arrow/Cargo.toml"
          ).resolve()
          PYTHON

  lint:
    timeout-minutes: 30
    runs-on: ubuntu-24.04
    defaults:
      run:
        shell: bash
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Run format
        run: cargo fmt --all -- --check
      - name: Run clippy
        run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
      - name: Run clippy (without remote feature)
        run: cargo clippy --profile ci --workspace --tests -- -D warnings

  deny:
    # Supply-chain checks: advisories, licenses, banned crates, and source
    # restrictions. Configuration lives in `deny.toml` at the workspace root.
    timeout-minutes: 10
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
      - uses: EmbarkStudios/cargo-deny-action@v2
        with:
          command: check advisories bans licenses sources

  build-no-lock:
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    env:
      # Need up-to-date compilers for kernels
      CC: clang
      CXX: clang++
    steps:
      - uses: actions/checkout@v6
      # Building without a lock file often requires the latest Rust version since downstream
      # dependencies may have updated their minimum Rust version.
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: "stable"
      # Remove cargo.lock to force a fresh build
      - name: Remove Cargo.lock
        run: rm -f Cargo.lock
      - uses: rui314/setup-mold@v1
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Build all
        run: |
          cargo build --profile ci --benches --all-features --tests

  linux:
    timeout-minutes: 60
    # To build all features, we need more disk space than is available
    # on the free OSS github runner. This is mostly due to the the
    # sentence-transformers feature.
    runs-on: ubuntu-2404-4x-x64
    defaults:
      run:
        shell: bash
        working-directory: rust
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - uses: rui314/setup-mold@v1
      - name: Make Swap
        run: |
          swapfile=/swapfile
          min_swap_bytes=$((15 * 1024 * 1024 * 1024))
          active_swap_bytes="$(sudo swapon --show=NAME,SIZE --bytes --noheadings | awk '$1 == "/swapfile" { print $2 }')"
          if [ -n "$active_swap_bytes" ]; then
            if [ "$active_swap_bytes" -ge "$min_swap_bytes" ]; then
              echo "/swapfile is already active with enough space; skipping swap creation"
              exit 0
            fi
            echo "/swapfile is already active but smaller than 16G; using /mnt/lancedb-swapfile"
            swapfile=/mnt/lancedb-swapfile
          fi
          if sudo swapon --show=NAME --noheadings | grep -Fxq "$swapfile"; then
            echo "$swapfile is already active; skipping swap creation"
            exit 0
          fi
          sudo rm -f "$swapfile"
          sudo fallocate -l 16G "$swapfile"
          sudo chmod 600 "$swapfile"
          sudo mkswap "$swapfile"
          sudo swapon "$swapfile"
      - name: Build
        run: cargo build --profile ci --all-features --tests --locked --examples
      - name: Run feature tests
        run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
      - name: Run examples
        run: cargo run --profile ci --all-features --example simple --locked
      - name: Run blob v2 docs example
        # Python and TypeScript blob docs tests install preview packages.
        # Those builds are not guaranteed to remain available for more than
        # 6 months, so they stay a manual run. This example uses the workspace crate.
        run: cargo run --profile ci --all-features -p lancedb-blob-v2-docs --example multimodal --locked

  remote:
    timeout-minutes: 30
    # Running this requires access to secrets, so skip if this is a PR from a
    # fork. Keep it separate from the all-features build so Cargo does not
    # retain both dependency graphs in one target directory.
    if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
    runs-on: ubuntu-2404-4x-x64
    defaults:
      run:
        shell: bash
        working-directory: rust
    env:
      CC: clang-18
      CXX: clang++-18
      GH_TOKEN: ${{ github.token }}
      SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - uses: Swatinem/rust-cache@v2
        with:
          # Remote tests use a different feature graph from the main Linux
          # job. Cache downloads, but build into a fresh target directory.
          cache-targets: false
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - uses: rui314/setup-mold@v1
      - name: Run remote tests
        run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests

  macos:
    timeout-minutes: 60
    strategy:
      matrix:
        mac-runner: ["macos-14", "macos-15"]
    runs-on: "${{ matrix.mac-runner }}"
    defaults:
      run:
        shell: bash
        working-directory: rust
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: CPU features
        run: sysctl -a | grep cpu
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install dependencies
        run: brew install protobuf
      - name: Run tests
        run: |
          # Don't run the s3 integration tests since docker isn't available
          # on this image.
          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
            | jq -r '.packages[] | .features | keys | .[]' \
            | grep -v s3-test | sort | uniq | paste -s -d "," -`
          # Run doctests before test binaries fill the runner disk. Examples are
          # already built by the Linux job, so avoid retaining them here.
          cargo test --profile ci --features $ALL_FEATURES --locked --doc
          cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests

  windows:
    strategy:
      fail-fast: false
      matrix:
        include:
          - target: x86_64-pc-windows-msvc
            runner: windows-2022
    runs-on: ${{ matrix.runner }}
    defaults:
      run:
        working-directory: rust/lancedb
    steps:
      - uses: actions/checkout@v6
      - name: Set target
        run: rustup target add ${{ matrix.target }}
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install Protoc v21.12
        run: choco install --no-progress protoc
      - name: Build
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }}
      - name: Run tests
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          # `--target` has to match the build step above. Without it cargo uses
          # target/ci/ rather than target/<triple>/ci/ and rebuilds the entire
          # dependency graph a second time.
          cargo test --profile ci --features aws,remote --locked --target ${{ matrix.target }}

  # Run the aarch64 Windows tests only on main so PRs wait for a single
  # Windows platform. windows-11-arm is a standard runner, so it is free on
  # public repos. Running natively lets the aarch64 tests actually execute --
  # this job used to cross-compile them and then skip the test step, paying
  # full codegen and link cost for a compile check.
  windows-arm:
    # Keep the historical name so the check shows up as before on main.
    name: windows (aarch64-pc-windows-msvc, windows-11-arm)
    if: github.ref == 'refs/heads/main'
    runs-on: windows-11-arm
    defaults:
      run:
        working-directory: rust/lancedb
    steps:
      - uses: actions/checkout@v6
      - name: Set target
        run: rustup target add aarch64-pc-windows-msvc
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install Protoc v21.12
        run: choco install --no-progress protoc
      - name: Build
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo build --profile ci --features aws,remote --tests --locked --target aarch64-pc-windows-msvc
      - name: Run tests
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          # `--target` has to match the build step above. Without it cargo uses
          # target/ci/ rather than target/<triple>/ci/ and rebuilds the entire
          # dependency graph a second time.
          cargo test --profile ci --features aws,remote --locked --target aarch64-pc-windows-msvc

  msrv:
    # Check the minimum supported Rust version
    name: MSRV Check - Rust v${{ matrix.msrv }}
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: true
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Install ${{ matrix.msrv }}
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.msrv }}
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Downgrade dependencies
        # aws-smithy-types 1.6.1+ requires rustc 1.94.1
        run: cargo update -p aws-smithy-types --precise 1.5.0
      - name: cargo +${{ matrix.msrv }} check
        env:
          RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
        run: cargo check --profile ci --workspace --tests --benches --all-features