lancedb/.github/workflows/python.yml

name: Python

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - Cargo.toml
      - Cargo.lock
      - rust-toolchain.toml
      - python/**
      - rust/**
      - .github/workflows/python.yml
      - .github/workflows/build_linux_wheel/**
      - .github/workflows/build_mac_wheel/**
      - .github/workflows/build_windows_wheel/**
      - .github/actions/setup-protoc/**
      - .github/workflows/run_tests/**

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Color output for pytest is off by default.
  PYTEST_ADDOPTS: "--color=yes"
  FORCE_COLOR: "1"
  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
  RUST_BACKTRACE: "1"

jobs:
  lint:
    name: "Lint"
    timeout-minutes: 30
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - name: Install ruff
        run: |
          pip install ruff==0.9.9
      - name: Format check
        run: ruff format --check .
      - name: Lint
        run: ruff check .

  type-check:
    name: "Type Check"
    timeout-minutes: 60
    runs-on: ubuntu-2404-8x-x64
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - name: Install protobuf compiler
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
          pip install toml
      - name: Install dependencies
        run: |
          python ../ci/parse_requirements.py pyproject.toml --extras dev,tests,embeddings > requirements.txt
          pip install -r requirements.txt
      - name: Run pyright
        run: pyright

  doctest:
    name: "Doctest"
    timeout-minutes: 60
    runs-on: ubuntu-2404-8x-x64
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
          cache: "pip"
      - name: Install protobuf
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      # `pip install -e .` builds the extension with maturin, which is most of
      # this job's ~33 minutes. It had no Rust cache, so every dependency was
      # recompiled from scratch on every run.
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install
        run: |
          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
          pip install mlx
      - name: Test remote torch data loader
        run: |
          python -c "import torch"
          pytest python/tests/test_torch.py::test_remote_permutation_dataloader_multiprocessing -q -rs
      - name: Doctest
        run: pytest --doctest-modules python/lancedb
  linux:
    name: "Linux: python-3.${{ matrix.python-minor-version }}"
    # A cold native build plus both test passes can exhaust 30 minutes.
    timeout-minutes: 60
    strategy:
      matrix:
        python-minor-version: ["10", "13"]
    runs-on: "ubuntu-24.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Install protobuf
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: 3.${{ matrix.python-minor-version }}
      - uses: ./.github/workflows/build_linux_wheel
        with:
          args: --profile ci
      - uses: ./.github/workflows/run_tests
        with:
          integration: true
      - name: Test without pylance or pandas
        run: |
          pip uninstall -y pylance pandas
          # These modules still import optional packages during collection or
          # exercise them throughout many tests; the full run above covers them.
          pytest -m "not slow and not s3_test" -vv --durations=30 python/tests \
            --ignore=python/tests/docs \
            --ignore=python/tests/test_context.py \
            --ignore=python/tests/test_db.py \
            --ignore=python/tests/test_embeddings.py \
            --ignore=python/tests/test_embeddings_slow.py \
            --ignore=python/tests/test_namespace_integration.py \
            --ignore=python/tests/test_query.py
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  platform:
    name: "Mac: ${{ matrix.features }}"
    strategy:
      matrix:
        features: ["default", "experimental-functions"]
    # A cold native build plus experimental-functions tests can exceed 30 minutes.
    timeout-minutes: 60
    runs-on: macos-15
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      # maturin runs cargo natively on macOS (docker is Linux-only), so the host
      # target dir is cacheable. This job had no Rust cache.
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - uses: ./.github/workflows/build_mac_wheel
        with:
          args: --profile ci --features ${{ matrix.features }}
      - uses: ./.github/workflows/run_tests
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  windows:
    name: "Windows: ${{ matrix.config.name }}"
    timeout-minutes: 60
    strategy:
      matrix:
        config:
          - name: x86
            runner: windows-latest
    runs-on: "${{ matrix.config.runner }}"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      # maturin runs cargo natively on Windows (docker is Linux-only), so the
      # host target dir is cacheable. This job had no Rust cache at all and so
      # rebuilt every dependency from scratch on every run.
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. The repo sits at
          # GitHub's cache cap, so per-PR saves just evict main's entries.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - uses: ./.github/workflows/build_windows_wheel
        with:
          args: --profile ci
      - uses: ./.github/workflows/run_tests
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  min-deps:
    name: "Minimum dependencies"
    timeout-minutes: 60
    runs-on: "ubuntu-24.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          lfs: true
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
      # As with Doctest, `pip install -e .` compiles the extension and this job
      # had no Rust cache, which is most of its ~37 minutes.
      - uses: Swatinem/rust-cache@v2
        with:
          # Restore everywhere, but only save from main. Per-PR saves are
          # unreadable outside their own branch anyway, since GitHub scopes
          # caches to the creating ref.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install lancedb
        run: |
          pip install "pydantic==2.7.4" "pyarrow==16"
          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
      - name: Run tests
        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests