Bump version: 0.25.4-beta.3 → 0.26.0-beta.0

## Summary
- bump Lance workspace dependencies to v1.1.0-beta.1 via
ci/set_lance_version.py
- verified `cargo clippy --workspace --tests --all-features -- -D
warnings` and `cargo fmt --all` after the bump

Link: refs/tags/v1.1.0-beta.1

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.9.0"
+current_version = "0.22.4-beta.3"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -24,34 +24,51 @@ commit = true
 message = "Bump version: {current_version} → {new_version}"
 commit_args = ""
 
-[tool.bumpversion.parts.pre_l]
-values = ["beta", "final"]
-optional_value = "final"
+# Java maven files
+pre_commit_hooks = [
+  """
+    NEW_VERSION="${BVHOOK_NEW_MAJOR}.${BVHOOK_NEW_MINOR}.${BVHOOK_NEW_PATCH}"
+    if [ ! -z "$BVHOOK_NEW_PRE_L" ] && [ ! -z "$BVHOOK_NEW_PRE_N" ]; then
+        NEW_VERSION="${NEW_VERSION}-${BVHOOK_NEW_PRE_L}.${BVHOOK_NEW_PRE_N}"
+    fi
+    echo "Constructed new version: $NEW_VERSION"
+    cd java && mvn versions:set -DnewVersion=$NEW_VERSION && mvn versions:commit
 
-[[tool.bumpversion.files]]
-filename = "node/package.json"
-search = "\"version\": \"{current_version}\","
-replace = "\"version\": \"{new_version}\","
+    # Check for any modified but unstaged pom.xml files
+    MODIFIED_POMS=$(git ls-files -m | grep pom.xml)
+    if [ ! -z "$MODIFIED_POMS" ]; then
+        echo "The following pom.xml files were modified but not staged. Adding them now:"
+        echo "$MODIFIED_POMS" | while read -r file; do
+            git add "$file"
+            echo "Added: $file"
+        done
+    fi
+    """,
+]
+
+[tool.bumpversion.parts.pre_l]
+optional_value = "final"
+values = ["beta", "final"]
 
 [[tool.bumpversion.files]]
 filename = "nodejs/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
 
 # nodejs binary packages
 [[tool.bumpversion.files]]
 glob = "nodejs/npm/*/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
 
 # Cargo files
 # ------------
 [[tool.bumpversion.files]]
-filename = "rust/ffi/node/Cargo.toml"
-search = "\nversion = \"{current_version}\""
+filename = "rust/lancedb/Cargo.toml"
 replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
 
 [[tool.bumpversion.files]]
-filename = "rust/lancedb/Cargo.toml"
-search = "\nversion = \"{current_version}\""
+filename = "nodejs/Cargo.toml"
 replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""

.cargo/config.toml

@@ -19,7 +19,7 @@ rustflags = [
     "-Wclippy::string_add_assign",
     "-Wclippy::string_add",
     "-Wclippy::string_lit_as_bytes",
-    "-Wclippy::string_to_string",
+    "-Wclippy::implicit_clone",
     "-Wclippy::use_self",
     "-Dclippy::cargo",
     "-Dclippy::dbg_macro",
@@ -31,6 +31,13 @@ rustflags = [
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 
+[target.x86_64-unknown-linux-musl]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
+
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"
+rustflags = ["-C", "target-feature=-crt-static"]
+
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
 
@@ -38,3 +45,7 @@ rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm
 # not found errors on systems that are missing it.
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-Ctarget-feature=+crt-static"]
+
+# Experimental target for Arm64 Windows
+[target.aarch64-pc-windows-msvc]
+rustflags = ["-Ctarget-feature=+crt-static"]

.github/ISSUE_TEMPLATE/documentation.yml

@@ -18,6 +18,6 @@ body:
       label: Link
       description: >
         Provide a link to the existing documentation, if applicable.
-      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
+      placeholder: ex. https://lancedb.com/docs/tables/...
     validations:
       required: false

.github/actions/create-failure-issue/action.yml

@@ -0,0 +1,45 @@
+name: Create Failure Issue
+description: Creates a GitHub issue if any jobs in the workflow failed
+
+inputs:
+  job-results:
+    description: 'JSON string of job results from needs context'
+    required: true
+  workflow-name:
+    description: 'Name of the workflow'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Check for failures and create issue
+      shell: bash
+      env:
+        JOB_RESULTS: ${{ inputs.job-results }}
+        WORKFLOW_NAME: ${{ inputs.workflow-name }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        # Check if any job failed
+        if echo "$JOB_RESULTS" | jq -e 'to_entries | any(.value.result == "failure")' > /dev/null; then
+          echo "Detected job failures, creating issue..."
+
+          # Extract failed job names
+          FAILED_JOBS=$(echo "$JOB_RESULTS" | jq -r 'to_entries | map(select(.value.result == "failure")) | map(.key) | join(", ")')
+
+          # Create issue with workflow name, failed jobs, and run URL
+          gh issue create \
+            --title "$WORKFLOW_NAME Failed ($FAILED_JOBS)" \
+            --body "The workflow **$WORKFLOW_NAME** failed during execution.
+
+        **Failed jobs:** $FAILED_JOBS
+
+        **Run URL:** $RUN_URL
+
+        Please investigate the failed jobs and address any issues." \
+            --label "ci"
+
+          echo "Issue created successfully"
+        else
+          echo "No job failures detected, skipping issue creation"
+        fi

.github/workflows/build_linux_wheel/action.yml

@@ -31,13 +31,13 @@ runs:
       with:
         command: build
         working-directory: python
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         target: x86_64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          yum install -y openssl-devel \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip
     - name: Build Arm Manylinux Wheel
@@ -46,18 +46,13 @@ runs:
       with:
         command: build
         working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         target: aarch64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          apt install -y unzip
-          if [ $(uname -m) = "x86_64" ]; then
-            PROTOC_ARCH="x86_64"
-          else
-            PROTOC_ARCH="aarch_64"
-          fi
-          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
+          yum install -y clang \
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -20,7 +20,7 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
+        # TODO: pass through interpreter
         args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         working-directory: python
-        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -26,9 +26,9 @@ runs:
       with:
         command: build
         args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         working-directory: python
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: windows-wheels
         path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -5,8 +5,8 @@ on:
     tags-ignore:
       # We don't publish pre-releases for Rust. Crates.io is just a source
       # distribution, so we don't need to publish pre-releases.
-      - 'v*-beta*'
-      - '*-v*' # for example, python-vX.Y.Z
+      - "v*-beta*"
+      - "*-v*" # for example, python-vX.Y.Z
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
@@ -19,6 +19,8 @@ env:
 jobs:
   build:
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
     timeout-minutes: 30
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -31,6 +33,22 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
+      - uses: rust-lang/crates-io-auth-action@v1
+        id: auth
       - name: Publish the package
         run: |
-          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          cargo publish -p lancedb --all-features --token ${{ steps.auth.outputs.token }}
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/codex-update-lance-dependency.yml

@@ -0,0 +1,127 @@
+name: Codex Update Lance Dependency
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        description: "Tag name from Lance"
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag name from Lance"
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: read
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Show inputs
+        run: |
+          echo "tag = ${{ inputs.tag }}"
+
+      - name: Checkout Repo LanceDB
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: true
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install Codex CLI
+        run: npm install -g @openai/codex
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          components: clippy, rustfmt
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler libssl-dev
+
+      - name: Install cargo-info
+        run: cargo install cargo-info
+
+      - name: Install Python dependencies
+        run: python3 -m pip install --upgrade pip packaging
+
+      - name: Configure git user
+        run: |
+          git config user.name "lancedb automation"
+          git config user.email "robot@lancedb.com"
+
+      - name: Run Codex to update Lance dependency
+        env:
+          TAG: ${{ inputs.tag }}
+          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
+        run: |
+          set -euo pipefail
+          VERSION="${TAG#refs/tags/}"
+          VERSION="${VERSION#v}"
+          BRANCH_NAME="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
+
+          cat <<EOF >/tmp/codex-prompt.txt
+          You are running inside the lancedb repository on a GitHub Actions runner. Update the Lance dependency to version ${VERSION} and prepare a pull request for maintainers to review.
+
+          Follow these steps exactly:
+          1. Use script "ci/set_lance_version.py" to update Lance dependencies. The script already refreshes Cargo metadata, so allow it to finish even if it takes time.
+          2. Run "cargo clippy --workspace --tests --all-features -- -D warnings". If diagnostics appear, fix them yourself and rerun clippy until it exits cleanly. Do not skip any warnings.
+          3. After clippy succeeds, run "cargo fmt --all" to format the workspace.
+          4. Ensure the repository is clean except for intentional changes. Inspect "git status --short" and "git diff" to confirm the dependency update and any required fixes.
+          5. Create and switch to a new branch named "${BRANCH_NAME}" (replace any duplicated hyphens if necessary).
+          6. Stage all relevant files with "git add -A". Commit using the message "chore: update lance dependency to v${VERSION}".
+          7. Push the branch to origin. If the branch already exists, force-push your changes.
+          8. env "GH_TOKEN" is available, use "gh" tools for github related operations like creating pull request.
+          9. Create a pull request targeting "main" with title "chore: update lance dependency to v${VERSION}". In the body, summarize the dependency bump, clippy/fmt verification, and link the triggering tag (${TAG}).
+          10. After creating the PR, display the PR URL, "git status --short", and a concise summary of the commands run and their results.
+
+          Constraints:
+          - Use bash commands; avoid modifying GitHub workflow files other than through the scripted task above.
+          - Do not merge the PR.
+          - If any command fails, diagnose and fix the issue instead of aborting.
+          EOF
+
+          printenv OPENAI_API_KEY | codex login --with-api-key
+          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"
+
+      - name: Trigger sophon dependency update
+        env:
+          TAG: ${{ inputs.tag }}
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          VERSION="${TAG#refs/tags/}"
+          VERSION="${VERSION#v}"
+          LANCEDB_BRANCH="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
+
+          echo "Triggering sophon workflow with:"
+          echo "  lance_ref: ${TAG#refs/tags/}"
+          echo "  lancedb_ref: ${LANCEDB_BRANCH}"
+
+          gh workflow run codex-bump-lancedb-lance.yml \
+            --repo lancedb/sophon \
+            -f lance_ref="${TAG#refs/tags/}" \
+            -f lancedb_ref="${LANCEDB_BRANCH}"
+
+      - name: Show latest sophon workflow run
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          echo "Latest sophon workflow run:"
+          gh run list --repo lancedb/sophon --workflow codex-bump-lancedb-lance.yml --limit 1 --json databaseId,url,displayTitle

.github/workflows/docs.yml

@@ -18,53 +18,51 @@ concurrency:
   group: "pages"
   cancel-in-progress: true
 
+env:
+  # This reduces the disk space needed for the build
+  RUSTFLAGS: "-C debuginfo=0"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
+
 jobs:
   # Single deploy job since we're just deploying
   build:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install dependecies needed for ubuntu
+      - name: Install dependencies needed for ubuntu
         run: |
           sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default        
+          rustup update && rustup default
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: "docs/requirements.txt"
+      - uses: Swatinem/rust-cache@v2
       - name: Build Python
         working-directory: python
         run: |
-          python -m pip install -e .
-          python -m pip install -r ../docs/requirements.txt
+          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .
+          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
       - name: Set up node
         uses: actions/setup-node@v3
         with:
           node-version: 20
           cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
+          cache-dependency-path: docs/package-lock.json
       - name: Install node dependencies
-        working-directory: node
+        working-directory: nodejs
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build node
-        working-directory: node
-        run: |
-          npm ci
-          npm run build
-          npm run tsc
-      - name: Create markdown files
-        working-directory: node
-        run: |
-          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         working-directory: docs
         run: |
@@ -72,9 +70,9 @@ jobs:
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.github/workflows/docs_test.yml

@@ -1,100 +0,0 @@
-name: Documentation Code Testing
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-  pull_request:
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  test-python:
-    name: Test doc python code
-    runs-on: "warp-ubuntu-latest-x64-4x"
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
-    - name: Install dependecies needed for ubuntu
-      run: |
-        sudo apt install -y protobuf-compiler libssl-dev
-        rustup update && rustup default
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.11
-        cache: "pip"
-        cache-dependency-path: "docs/test/requirements.txt"
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
-    - name: Build Python
-      working-directory: docs/test
-      run:
-        python -m pip install -r requirements.txt
-    - name: Create test files
-      run: |
-        cd docs/test
-        python md_testing.py
-    - name: Test
-      run: |
-        cd docs/test/python
-        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
-  test-node:
-    name: Test doc nodejs code
-    runs-on: "warp-ubuntu-latest-x64-4x"
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
-    - name: Set up Node
-      uses: actions/setup-node@v4
-      with:
-        node-version: 20
-    - name: Install dependecies needed for ubuntu
-      run: |
-        sudo apt install -y protobuf-compiler libssl-dev
-        rustup update && rustup default
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
-    - name: Install node dependencies
-      run: |
-        sudo swapoff -a
-        sudo fallocate -l 8G /swapfile
-        sudo chmod 600 /swapfile
-        sudo mkswap /swapfile
-        sudo swapon /swapfile
-        sudo swapon --show
-        cd node
-        npm ci
-        npm run build-release
-        cd ../docs
-        npm install
-    - name: Test
-      env:
-        LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
-        LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
-      run: |
-        cd docs
-        npm t

.github/workflows/java-publish.yml

@@ -0,0 +1,127 @@
+name: Build and publish Java packages
+on:
+  release:
+    types: [released]
+  pull_request:
+    paths:
+      - .github/workflows/java-publish.yml
+
+jobs:
+  macos-arm64:
+    name: Build on MacOS Arm64
+    runs-on: macos-14
+    timeout-minutes: 45
+    defaults:
+      run:
+        working-directory: ./java/core/lancedb-jni
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          brew install protobuf
+      - name: Build release
+        run: |
+          cargo build --release
+      - uses: actions/upload-artifact@v4
+        with:
+          name: liblancedb_jni_darwin_aarch64.zip
+          path: target/release/liblancedb_jni.dylib
+          retention-days: 1
+          if-no-files-found: error
+  linux-arm64:
+    name: Build on Linux Arm64
+    runs-on: warp-ubuntu-2204-arm64-8x
+    timeout-minutes: 45
+    defaults:
+      run:
+        working-directory: ./java/core/lancedb-jni
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-workspaces: "./java/core/lancedb-jni"
+          # Disable full debug symbol generation to speed up CI build and keep memory down
+          # "1" means line tables only, which is useful for panic tracebacks.
+          rustflags: "-C debuginfo=1"
+      - name: Install dependencies
+        run: |
+          sudo apt -y -qq update
+          sudo apt install -y protobuf-compiler libssl-dev pkg-config
+      - name: Build release
+        run: |
+          cargo build --release
+      - uses: actions/upload-artifact@v4
+        with:
+          name: liblancedb_jni_linux_aarch64.zip
+          path: target/release/liblancedb_jni.so
+          retention-days: 1
+          if-no-files-found: error
+  linux-x86:
+    runs-on: warp-ubuntu-2204-x64-8x
+    timeout-minutes: 30
+    needs: [macos-arm64, linux-arm64]
+    defaults:
+      run:
+        working-directory: ./java
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+      - name: Set up Java 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 8
+          cache: "maven"
+          server-id: ossrh
+          server-username: SONATYPE_USER
+          server-password: SONATYPE_TOKEN
+          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
+          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
+      - name: Install dependencies
+        run: |
+          sudo apt -y -qq update
+          sudo apt install -y protobuf-compiler libssl-dev pkg-config
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+      - name: Copy native libs
+        run: |
+          mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64
+          cp ../liblancedb_jni_darwin_aarch64.zip/liblancedb_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblancedb_jni.dylib
+          cp ../liblancedb_jni_linux_aarch64.zip/liblancedb_jni.so ./core/target/classes/nativelib/linux-aarch64/liblancedb_jni.so
+      - name: Dry run
+        if: github.event_name == 'pull_request'
+        run: |
+          mvn --batch-mode -DskipTests -Drust.release.build=true package
+      - name: Set github
+        run: |
+          git config --global user.email "LanceDB Github Runner"
+          git config --global user.name "dev+gha@lancedb.com"
+      - name: Publish with Java 8
+        if: github.event_name == 'release'
+        run: |
+          echo "use-agent" >> ~/.gnupg/gpg.conf
+          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
+          export GPG_TTY=$(tty)
+          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
+        env:
+          SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
+          SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [linux-arm64, linux-x86, macos-arm64]
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/java.yml

@@ -35,6 +35,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: java/core/lancedb-jni
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
       - name: Run cargo fmt
         run: cargo fmt --check
         working-directory: ./java/core/lancedb-jni
@@ -68,6 +71,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: java/core/lancedb-jni
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
       - name: Run cargo fmt
         run: cargo fmt --check
         working-directory: ./java/core/lancedb-jni
@@ -110,4 +116,3 @@ jobs:
           -Djdk.reflect.useDirectMethodHandle=false \
           -Dio.netty.tryReflectionSetAccessible=true"
           JAVA_HOME=$JAVA_17 mvn clean test
-  

.github/workflows/lance-release-timer.yml

@@ -0,0 +1,62 @@
+name: Lance Release Timer
+
+on:
+  schedule:
+    - cron: "*/10 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: write
+
+concurrency:
+  group: lance-release-timer
+  cancel-in-progress: false
+
+jobs:
+  trigger-update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check for new Lance tag
+        id: check
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          python3 ci/check_lance_release.py --github-output "$GITHUB_OUTPUT"
+
+      - name: Look for existing PR
+        if: steps.check.outputs.needs_update == 'true'
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          TITLE="chore: update lance dependency to v${{ steps.check.outputs.latest_version }}"
+          COUNT=$(gh pr list --search "\"$TITLE\" in:title" --state open --limit 1 --json number --jq 'length')
+          if [ "$COUNT" -gt 0 ]; then
+            echo "Open PR already exists for $TITLE"
+            echo "pr_exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No existing PR for $TITLE"
+            echo "pr_exists=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Trigger codex update workflow
+        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          TAG=${{ steps.check.outputs.latest_tag }}
+          gh workflow run codex-update-lance-dependency.yml -f tag=refs/tags/$TAG
+
+      - name: Show latest codex workflow run
+        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          gh run list --workflow codex-update-lance-dependency.yml --limit 1 --json databaseId,url,displayTitle

.github/workflows/license-header-check.yml

@@ -0,0 +1,31 @@
+name: Check license headers
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - rust/**
+      - python/**
+      - nodejs/**
+      - java/**
+      - .github/workflows/license-header-check.yml
+jobs:
+  check-licenses:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Install license-header-checker
+        working-directory: /tmp
+        run: |
+            curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
+            mv /tmp/bin/license-header-checker /usr/local/bin/
+      - name: Check license headers (rust)
+        run: license-header-checker -a -v ./rust/license_header.txt ./ rs && [[ -z `git status -s` ]]
+      - name: Check license headers (python)
+        run: license-header-checker -a -v ./python/license_header.txt python py && [[ -z `git status -s` ]]
+      - name: Check license headers (typescript)
+        run: license-header-checker -a -v ./nodejs/license_header.txt nodejs ts && [[ -z `git status -s` ]]
+      - name: Check license headers (java)
+        run: license-header-checker -a -v ./nodejs/license_header.txt java java && [[ -z `git status -s` ]]

.github/workflows/make-release-commit.yml

@@ -30,7 +30,7 @@ on:
         default: true
         type: boolean
       other:
-        description: 'Make a Node/Rust release'
+        description: 'Make a Node/Rust/Java release'
         required: true
         default: true
         type: boolean
@@ -43,7 +43,7 @@ on:
 jobs:
   make-release:
     # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
     steps:
@@ -57,15 +57,14 @@ jobs:
           # trigger any workflows watching for new tags. See:
           # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
           token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+      - name: Validate Lance dependency is at stable version
+        if: ${{ inputs.type == 'stable' }}
+        run: python ci/validate_stable_lance.py
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
       - name: Bump Python version
         if: ${{ inputs.python }}
         working-directory: python
@@ -85,6 +84,7 @@ jobs:
         run: |
           pip install bump-my-version PyGithub packaging
           bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
+          bash ci/update_lockfiles.sh --amend
       - name: Push new version tag
         if: ${{ !inputs.dry_run }}
         uses: ad-m/github-push-action@master
@@ -93,7 +93,3 @@ jobs:
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
           branch: ${{ github.ref }}
           tags: true
-      - uses: ./.github/workflows/update_package_lock
-        if: ${{ !inputs.dry_run && inputs.other }}
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/node.yml

@@ -1,147 +0,0 @@
-name: Node
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - node/**
-      - rust/ffi/node/**
-      - .github/workflows/node.yml
-      - docker-compose.yml
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  #
-  # Use native CPU to accelerate tests if possible, especially for f16
-  # target-cpu=haswell fixes failing ci build
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  linux:
-    name: Linux (Node ${{ matrix.node-version }})
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        node-version: [ "18", "20" ]
-    runs-on: "ubuntu-22.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - uses: Swatinem/rust-cache@v2
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
-    - name: Build
-      run: |
-        npm ci
-        npm run build
-        npm run pack-build
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
-    - name: Test
-      run: npm run test
-  macos:
-    timeout-minutes: 30
-    runs-on: "macos-13"
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - uses: Swatinem/rust-cache@v2
-    - name: Install dependencies
-      run: brew install protobuf
-    - name: Build
-      run: |
-        npm ci
-        npm run build
-        npm run pack-build
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
-    - name: Test
-      run: |
-        npm run test
-  aws-integtest:
-    timeout-minutes: 45
-    runs-on: "ubuntu-22.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    env:
-      AWS_ACCESS_KEY_ID: ACCESSKEY
-      AWS_SECRET_ACCESS_KEY: SECRETKEY
-      AWS_DEFAULT_REGION: us-west-2
-      # this one is for s3
-      AWS_ENDPOINT: http://localhost:4566
-      # this one is for dynamodb
-      DYNAMODB_ENDPOINT: http://localhost:4566
-      ALLOW_HTTP: true
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - name: start local stack
-      run: docker compose -f ../docker-compose.yml up -d --wait
-    - name: create s3
-      run: aws s3 mb s3://lancedb-integtest --endpoint $AWS_ENDPOINT
-    - name: create ddb
-      run: |
-        aws dynamodb create-table \
-          --table-name lancedb-integtest \
-          --attribute-definitions '[{"AttributeName": "base_uri", "AttributeType": "S"}, {"AttributeName": "version", "AttributeType": "N"}]' \
-          --key-schema '[{"AttributeName": "base_uri", "KeyType": "HASH"}, {"AttributeName": "version", "KeyType": "RANGE"}]' \
-          --provisioned-throughput '{"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}' \
-          --endpoint-url $DYNAMODB_ENDPOINT
-    - uses: Swatinem/rust-cache@v2
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
-    - name: Build
-      run: |
-        npm ci
-        npm run build
-        npm run pack-build
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
-    - name: Test
-      run: npm run integration-test

.github/workflows/nodejs.yml

@@ -6,6 +6,7 @@ on:
       - main
   pull_request:
     paths:
+      - Cargo.toml
       - nodejs/**
       - .github/workflows/nodejs.yml
       - docker-compose.yml
@@ -15,9 +16,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
 
 jobs:
@@ -42,17 +40,25 @@ jobs:
         node-version: 20
         cache: 'npm'
         cache-dependency-path: nodejs/package-lock.json
-    - uses: Swatinem/rust-cache@v2
+    - uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        components: rustfmt, clippy
     - name: Install dependencies
       run: |
         sudo apt update
         sudo apt install -y protobuf-compiler libssl-dev
-    - name: Lint
+    - uses: Swatinem/rust-cache@v2
+    - name: Format Rust
+      run: cargo fmt --all -- --check
+    - name: Lint Rust
+      run: cargo clippy --profile ci --all --all-features -- -D warnings
+    - name: Lint Typescript
       run: |
-        cargo fmt --all -- --check
-        cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint-ci
+    - name: Lint examples
+      working-directory: nodejs/examples
+      run: npm ci && npm run lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -73,7 +79,7 @@ jobs:
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'
-        cache-dependency-path: node/package-lock.json
+        cache-dependency-path: nodejs/package-lock.json
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
@@ -83,7 +89,8 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
+        npm run build:debug -- --profile ci
+        npm run tsc
     - name: Setup localstack
       working-directory: .
       run: docker compose up --detach --wait
@@ -91,6 +98,30 @@ jobs:
       env:
         S3_TEST: "1"
       run: npm run test
+    - name: Setup examples
+      working-directory: nodejs/examples
+      run: npm ci
+    - name: Test examples
+      working-directory: ./
+      env:
+        OPENAI_API_KEY: test
+        OPENAI_BASE_URL: http://0.0.0.0:8000
+      run: |
+        python ci/mock_openai.py &
+        cd nodejs/examples
+        npm test
+    - name: Check docs
+      run: |
+        # We run this as part of the job because the binary needs to be built
+        # first to export the types of the native code.
+        set -e
+        npm ci
+        npm run docs
+        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
+          echo "Docs need to be updated"
+          echo "Run 'npm run docs', fix any warnings, and commit the changes."
+          exit 1
+        fi
   macos:
     timeout-minutes: 30
     runs-on: "macos-14"
@@ -107,7 +138,7 @@ jobs:
       with:
         node-version: 20
         cache: 'npm'
-        cache-dependency-path: node/package-lock.json
+        cache-dependency-path: nodejs/package-lock.json
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
@@ -116,7 +147,8 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
+        npm run build:debug -- --profile ci
+        npm run tsc
     - name: Test
       run: |
         npm run test

.github/workflows/npm-publish.yml

@@ -1,399 +1,32 @@
 name: NPM Publish
 
+env:
+  MACOSX_DEPLOYMENT_TARGET: '10.13'
+  CARGO_INCREMENTAL: '0'
+
+permissions:
+  contents: write
+  id-token: write
+
 on:
   push:
+    branches:
+      - main
     tags:
       - "v*"
+  pull_request:
+    # This should trigger a dry run (we skip the final publish step)
+    paths:
+      - .github/workflows/npm-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  node:
-    name: vectordb Typescript
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: "npm"
-          cache-dependency-path: node/package-lock.json
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
-        run: |
-          npm ci
-          npm run tsc
-          npm pack
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-package
-          path: |
-            node/vectordb-*.tgz
-
-  node-macos:
-    name: vectordb ${{ matrix.config.arch }}
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-14
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-darwin-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-darwin*.tgz
-
-  nodejs-macos:
-    name: lancedb ${{ matrix.config.arch }}
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-14
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build MacOS native nodejs modules
-        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-darwin-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-
-  node-linux:
-    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: warp-ubuntu-latest-arm64-4x
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      # To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
-      - name: Build Linux Artifacts
-        run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-linux-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-linux*.tgz
-
-  nodejs-linux:
-    name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
-      - name: Build Linux Artifacts
-        run: |
-          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-      # The generic files are the same in all distros so we just pick
-      # one to do the upload.
-      - name: Upload Generic Artifacts
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-dist
-          path: |
-            nodejs/dist/*
-            !nodejs/dist/*.node
-
-  node-windows:
-    name: vectordb ${{ matrix.target }}
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-windows
-          path: |
-            node/dist/lancedb-vectordb-win32*.tgz
-
-  nodejs-windows:
-    name: lancedb ${{ matrix.target }}
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows
-          path: |
-            nodejs/dist/*.node
-
-  release:
-    name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux, node-windows]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: node-*
-      - name: Display structure of downloaded files
-        run: ls -R
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: "https://registry.npmjs.org"
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        run: |
-          # Tag beta as "preview" instead of default "latest". See lancedb 
-          # npm publish step for more info.
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            PUBLISH_ARGS="--tag preview"
-          fi
-
-          mv */*.tgz .
-          for filename in *.tgz; do
-            npm publish $PUBLISH_ARGS $filename
-          done
-      - name: Notify Slack Action
-        uses: ravsamhq/notify-slack-action@2.3.0
-        if: ${{ always() }}
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-          notification_title: "{workflow} is failing"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-
-  release-nodejs:
-    name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          name: nodejs-dist
-          path: nodejs/dist
-      - uses: actions/download-artifact@v4
-        name: Download arch-specific binaries
-        with:
-          pattern: nodejs-*
-          path: nodejs/nodejs-artifacts
-          merge-multiple: true
-      - name: Display structure of downloaded files
-        run: find .
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: "https://registry.npmjs.org"
-      - name: Install napi-rs
-        run: npm install -g @napi-rs/cli
-      - name: Prepare artifacts
-        run: npx napi artifacts -d nodejs-artifacts
-      - name: Display structure of staged files
-        run: find npm
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        # By default, things are published to the latest tag. This is what is
-        # installed by default if the user does not specify a version. This is
-        # good for stable releases, but for pre-releases, we want to publish to
-        # the "preview" tag so they can install with `npm install lancedb@preview`.
-        # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420
-        run: |
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            npm publish --access public --tag preview
-          else
-            npm publish --access public
-          fi
-      - name: Notify Slack Action
-        uses: ravsamhq/notify-slack-action@2.3.0
-        if: ${{ always() }}
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-          notification_title: "{workflow} is failing"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-
-  update-package-lock:
-    needs: [release]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-  update-package-lock-nodejs:
-    needs: [release-nodejs]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -458,3 +91,285 @@ jobs:
           generate_release_notes: false
           name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
           body: ${{ steps.release_notes.outputs.changelog }}
+
+  build-lancedb:
+    strategy:
+      fail-fast: false
+      matrix:
+        settings:
+          - target: aarch64-apple-darwin
+            host: macos-latest
+            features: fp16kernels
+            pre_build: brew install protobuf
+          - target: x86_64-pc-windows-msvc
+            host: windows-latest
+            features: ","
+            pre_build: |-
+              choco install --no-progress protoc ninja nasm
+              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
+              # There is an issue where choco doesn't add nasm to the path
+              export PATH="$PATH:/c/Program Files/NASM"
+              nasm -v
+          - target: aarch64-pc-windows-msvc
+            host: windows-latest
+            features: ","
+            pre_build: |-
+                choco install --no-progress protoc
+                rustup target add aarch64-pc-windows-msvc
+          - target: x86_64-unknown-linux-gnu
+            host: ubuntu-latest
+            features: fp16kernels
+            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
+            pre_build: |-
+              set -e &&
+              apt-get update &&
+              apt-get install -y protobuf-compiler pkg-config
+          - target: x86_64-unknown-linux-musl
+            # This one seems to need some extra memory
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
+            features: fp16kernels
+            pre_build: |-
+              set -e &&
+              apk add protobuf-dev curl &&
+              ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
+              ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
+              CC=gcc &&
+              CXX=g++
+          - target: aarch64-unknown-linux-gnu
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
+            features: "fp16kernels"
+            pre_build: |-
+              set -e &&
+              apt-get update &&
+              apt-get install -y protobuf-compiler pkg-config &&
+              # https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
+              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
+              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
+              rustup target add aarch64-unknown-linux-gnu
+          - target: aarch64-unknown-linux-musl
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
+            features: ","
+            pre_build: |-
+              set -e &&
+              apk add protobuf-dev &&
+              rustup target add aarch64-unknown-linux-musl &&
+              export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
+              export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
+    name: build - ${{ matrix.settings.target }}
+    runs-on: ${{ matrix.settings.host }}
+    defaults:
+      run:
+        working-directory: nodejs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        if: ${{ !matrix.settings.docker }}
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: nodejs/package-lock.json
+      - name: Install
+        uses: dtolnay/rust-toolchain@stable
+        if: ${{ !matrix.settings.docker }}
+        with:
+          toolchain: stable
+          targets: ${{ matrix.settings.target }}
+      - name: Cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            target/
+          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
+      - name: Setup toolchain
+        run: ${{ matrix.settings.setup }}
+        if: ${{ matrix.settings.setup }}
+        shell: bash
+      - name: Install dependencies
+        run: npm ci
+      - name: Build in docker
+        uses: addnab/docker-run-action@v3
+        if: ${{ matrix.settings.docker }}
+        with:
+          image: ${{ matrix.settings.docker }}
+          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
+                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
+                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
+                    -v ${{ github.workspace }}:/build -w /build/nodejs"
+          run: |
+            set -e
+            ${{ matrix.settings.pre_build }}
+            npx napi build --platform  --release --no-const-enum \
+              --features ${{ matrix.settings.features }} \
+              --target ${{ matrix.settings.target }} \
+              --dts ../lancedb/native.d.ts \
+              --js ../lancedb/native.js \
+              --strip \
+              dist/
+      - name: Build
+        run: |
+          ${{ matrix.settings.pre_build }}
+          npx napi build --platform  --release --no-const-enum \
+              --features ${{ matrix.settings.features }} \
+              --target ${{ matrix.settings.target }} \
+              --dts ../lancedb/native.d.ts \
+              --js ../lancedb/native.js \
+              --strip \
+              $EXTRA_ARGS \
+              dist/
+        if: ${{ !matrix.settings.docker }}
+        shell: bash
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: lancedb-${{ matrix.settings.target }}
+          path: nodejs/dist/*.node
+          if-no-files-found: error
+      # The generic files are the same in all distros so we just pick
+      # one to do the upload.
+      - name: Make generic artifacts
+        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
+        run: npm run tsc
+      - name: Upload Generic Artifacts
+        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-dist
+          path: |
+            nodejs/dist/*
+            !nodejs/dist/*.node
+  test-lancedb:
+    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
+    needs:
+      - build-lancedb
+    strategy:
+      fail-fast: false
+      matrix:
+        settings:
+          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
+          # - host: windows-latest
+          #   target: x86_64-pc-windows-msvc
+          - host: macos-latest
+            target: aarch64-apple-darwin
+          - target: x86_64-unknown-linux-gnu
+            host: ubuntu-latest
+          - target: aarch64-unknown-linux-gnu
+            host: buildjet-16vcpu-ubuntu-2204-arm
+        node:
+          - '20'
+    runs-on: ${{ matrix.settings.host }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: npm
+          cache-dependency-path: nodejs/package-lock.json
+      - name: Install dependencies
+        run: npm ci
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: lancedb-${{ matrix.settings.target }}
+          path: nodejs/dist/
+          # For testing purposes:
+          # run-id: 13982782871
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+          # For testing purposes:
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+          # run-id: 13982782871
+      - name: List packages
+        run: ls -R dist
+      - name: Move built files
+        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
+      - name: Test bindings
+        run: npm test
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    needs:
+      - test-lancedb
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: nodejs/package-lock.json
+          registry-url: "https://registry.npmjs.org"
+      - name: Install dependencies
+        run: npm ci
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+          # For testing purposes:
+          # run-id: 13982782871
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+      - uses: actions/download-artifact@v4
+        name: Download arch-specific binaries
+        with:
+          pattern: lancedb-*
+          path: nodejs/nodejs-artifacts
+          merge-multiple: true
+          # For testing purposes:
+          # run-id: 13982782871
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+      - name: Display structure of downloaded files
+        run: find dist && find nodejs-artifacts
+      - name: Move artifacts
+        run: npx napi artifacts -d nodejs-artifacts
+      - name: List packages
+        run: find npm
+      - name: Publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        run: |
+          ARGS="--access public"
+          if [[ $DRY_RUN == "true" ]]; then
+            ARGS="$ARGS --dry-run"
+          fi
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            ARGS="$ARGS --tag preview"
+          fi
+          npm publish $ARGS
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [build-lancedb, test-lancedb, publish]
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/pypi-publish.yml

@@ -4,6 +4,14 @@ on:
   push:
     tags:
       - 'python-v*'
+  pull_request:
+    # This should trigger a dry run (we skip the final publish step)
+    paths:
+      - .github/workflows/pypi-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
+
+env:
+  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
 
 jobs:
   linux:
@@ -15,15 +23,21 @@ jobs:
           - platform: x86_64
             manylinux: "2_17"
             extra_args: ""
+            runner: ubuntu-22.04
           - platform: x86_64
             manylinux: "2_28"
             extra_args: "--features fp16kernels"
+            runner: ubuntu-22.04
           - platform: aarch64
-            manylinux: "2_24"
+            manylinux: "2_17"
             extra_args: ""
-          # We don't build fp16 kernels for aarch64, because it uses 
-          # cross compilation image, which doesn't have a new enough compiler.
-    runs-on: "ubuntu-22.04"
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: ubuntu-2404-8x-arm64
+          - platform: aarch64
+            manylinux: "2_28"
+            extra_args: "--features fp16kernels"
+            runner: ubuntu-2404-8x-arm64
+    runs-on: ${{ matrix.config.runner }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -40,19 +54,18 @@ jobs:
           arm-build: ${{ matrix.config.platform == 'aarch64' }}
           manylinux: ${{ matrix.config.manylinux }}
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   mac:
-    timeout-minutes: 60
+    timeout-minutes: 90
     runs-on: ${{ matrix.config.runner }}
     strategy:
       matrix:
         config:
-          - target: x86_64-apple-darwin
-            runner: macos-13
           - target: aarch64-apple-darwin
-            runner: macos-14
+            runner: warp-macos-14-arm64-6x
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.15
     steps:
@@ -69,6 +82,7 @@ jobs:
           python-minor-version: 8
           args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
@@ -83,17 +97,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - uses: ./.github/workflows/build_windows_wheel
         with:
           python-minor-version: 8
           args: "--release --strip"
           vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/python-v')
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -158,3 +174,17 @@ jobs:
           generate_release_notes: false
           name: Python LanceDB v${{ steps.extract_version.outputs.version }}
           body: ${{ steps.python_release_notes.outputs.changelog }}
+  report-failure:
+    name: Report Workflow Failure
+    runs-on: ubuntu-latest
+    needs: [linux, mac, windows]
+    permissions:
+      contents: read
+      issues: write
+    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/create-failure-issue
+        with:
+          job-results: ${{ toJSON(needs) }}
+          workflow-name: ${{ github.workflow }}

.github/workflows/python.yml

@@ -6,6 +6,7 @@ on:
       - main
   pull_request:
     paths:
+      - Cargo.toml
       - python/**
       - .github/workflows/python.yml
 
@@ -13,6 +14,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  # Color output for pytest is off by default.
+  PYTEST_ADDOPTS: "--color=yes"
+  FORCE_COLOR: "1"
+  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
+  RUST_BACKTRACE: "1"
+
 jobs:
   lint:
     name: "Lint"
@@ -30,16 +38,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install ruff
         run: |
-          pip install ruff==0.5.4
+          pip install ruff==0.9.9
       - name: Format check
         run: ruff format --check .
       - name: Lint
         run: ruff check .
-  doctest:
-    name: "Doctest"
+
+  type-check:
+    name: "Type Check"
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
     defaults:
@@ -54,18 +63,44 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
+      - name: Install protobuf compiler
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+          pip install toml
+      - name: Install dependencies
+        run: |
+          python ../ci/parse_requirements.py pyproject.toml --extras dev,tests,embeddings > requirements.txt
+          pip install -r requirements.txt
+      - name: Run pyright
+        run: pyright
+
+  doctest:
+    name: "Doctest"
+    timeout-minutes: 30
+    runs-on: ubuntu-2404-8x-x64
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
           cache: "pip"
       - name: Install protobuf
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
       - name: Install
         run: |
-          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
+          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
           pip install tantivy
           pip install mlx
       - name: Doctest
@@ -75,8 +110,8 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        python-minor-version: ["9", "11"]
-    runs-on: "ubuntu-22.04"
+        python-minor-version: ["9", "12"]
+    runs-on: "ubuntu-24.04"
     defaults:
       run:
         shell: bash
@@ -94,27 +129,23 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.${{ matrix.python-minor-version }}
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
       - uses: ./.github/workflows/build_linux_wheel
+        with:
+          args: --profile ci
       - uses: ./.github/workflows/run_tests
         with:
           integration: true
+      - name: Test without pylance or pandas
+        run: |
+          pip uninstall -y pylance pandas
+          pytest -vv python/tests/test_table.py
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
   platform:
-    name: "Mac: ${{ matrix.config.name }}"
+    name: "Mac"
     timeout-minutes: 30
-    strategy:
-      matrix:
-        config:
-          - name: x86
-            runner: macos-13
-          - name: Arm
-            runner: macos-14
-    runs-on: "${{ matrix.config.runner }}"
+    runs-on: macos-14
     defaults:
       run:
         shell: bash
@@ -127,18 +158,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
+          python-version: "3.12"
       - uses: ./.github/workflows/build_mac_wheel
+        with:
+          args: --profile ci
       - uses: ./.github/workflows/run_tests
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
   windows:
     name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 30
+    timeout-minutes: 60
     strategy:
       matrix:
         config:
@@ -157,18 +187,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
+          python-version: "3.12"
       - uses: ./.github/workflows/build_windows_wheel
+        with:
+          args: --profile ci
       - uses: ./.github/workflows/run_tests
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-24.04"
     defaults:
       run:
         shell: bash
@@ -189,7 +218,8 @@ jobs:
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
-          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
+          pip install pyarrow==16
+          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests
         run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -15,7 +15,7 @@ runs:
     - name: Install lancedb
       shell: bash
       run: |
-        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
+        pip3 install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
     - name: Setup localstack for integration tests
       if: ${{ inputs.integration == 'true' }}
       shell: bash
@@ -24,8 +24,8 @@ runs:
     - name: pytest (with integration)
       shell: bash
       if: ${{ inputs.integration == 'true' }}
-      run: pytest -m "not slow" -x -v --durations=30 python/python/tests
+      run: pytest -m "not slow" -vv --durations=30 python/python/tests
     - name: pytest (no integration tests)
       shell: bash
       if: ${{ inputs.integration != 'true' }}
-      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests
+      run: pytest -m "not slow and not s3_test" -vv --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -18,79 +18,111 @@ env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
 
 jobs:
   lint:
     timeout-minutes: 30
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     defaults:
       run:
         shell: bash
-        working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
+      CC: clang-18
+      CXX: clang++-18
     steps:
-    - uses: actions/checkout@v4
-      with:
+      - uses: actions/checkout@v4
+        with:
           fetch-depth: 0
           lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: rust
-    - name: Install dependencies
-      run: |
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-    - name: Run format
-      run: cargo fmt --all -- --check
-    - name: Run clippy
-      run: cargo clippy --all --all-features -- -D warnings
+      - name: Run format
+        run: cargo fmt --all -- --check
+      - name: Run clippy
+        run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
+
+  build-no-lock:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      # Need up-to-date compilers for kernels
+      CC: clang
+      CXX: clang++
+    steps:
+      - uses: actions/checkout@v4
+      # Building without a lock file often requires the latest Rust version since downstream
+      # dependencies may have updated their minimum Rust version.
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: "stable"
+      # Remove cargo.lock to force a fresh build
+      - name: Remove Cargo.lock
+        run: rm -f Cargo.lock
+      - uses: rui314/setup-mold@v1
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build all
+        run: |
+          cargo build --profile ci --benches --all-features --tests
+
   linux:
     timeout-minutes: 30
     # To build all features, we need more disk space than is available
-    # on the GitHub-provided runner. This is mostly due to the the
+    # on the free OSS github runner. This is mostly due to the the
     # sentence-transformers feature.
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-2404-4x-x64
     defaults:
       run:
         shell: bash
         working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
+      CC: clang-18
+      CXX: clang++-18
+      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
     steps:
-    - uses: actions/checkout@v4
-      with:
+      - uses: actions/checkout@v4
+        with:
           fetch-depth: 0
           lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-          workspaces: rust
-    - name: Install dependencies
-      run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Start S3 integration test environment
-      working-directory: .
-      run: docker compose up --detach --wait
-    - name: Build
-      run: cargo build --all-features
-    - name: Run tests
-      run: cargo test --all-features
-    - name: Run examples
-      run: cargo run --example simple
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: sudo apt install -y protobuf-compiler libssl-dev
+      - uses: rui314/setup-mold@v1
+      - name: Make Swap
+        run: |
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+      - name: Build
+        run: cargo build --profile ci --all-features --tests --locked --examples
+      - name: Run feature tests
+        run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
+      - name: Run examples
+        run: cargo run  --profile ci --example simple --locked
+      - name: Run remote tests
+        # Running this requires access to secrets, so skip if this is
+        # a PR from a fork.
+        if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
+        run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
+
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: [ "macos-13", "macos-14" ]
+        mac-runner: ["macos-14", "macos-15"]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
@@ -99,38 +131,85 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-            fetch-depth: 0
-            lfs: true
+          fetch-depth: 0
+          lfs: true
       - name: CPU features
         run: sysctl -a | grep cpu
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install dependencies
         run: brew install protobuf
-      - name: Build
-        run: cargo build --all-features
       - name: Run tests
-        # Run with everything except the integration tests.
-        run: cargo test --features remote,fp16kernels
+        run: |
+          # Don't run the s3 integration tests since docker isn't available
+          # on this image.
+          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
+            | jq -r '.packages[] | .features | keys | .[]' \
+            | grep -v s3-test | sort | uniq | paste -s -d "," -`
+          cargo test --profile ci --features $ALL_FEATURES --locked
+
   windows:
     runs-on: windows-2022
+    strategy:
+      matrix:
+        target:
+          - x86_64-pc-windows-msvc
+          - aarch64-pc-windows-msvc
+    defaults:
+      run:
+        working-directory: rust/lancedb
     steps:
       - uses: actions/checkout@v4
+      - name: Set target
+        run: rustup target add ${{ matrix.target }}
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Run tests
+        run: choco install --no-progress protoc
+      - name: Build
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build
-          cargo test
+          cargo build --profile ci --features remote --tests --locked --target ${{ matrix.target }}
+      - name: Run tests
+        # Can only run tests when target matches host
+        if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
+        run: |
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
+          cargo test --profile ci --features remote --locked
+
+  msrv:
+    # Check the minimum supported Rust version
+    name: MSRV Check - Rust v${{ matrix.msrv }}
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
+    env:
+      # Need up-to-date compilers for kernels
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install ${{ matrix.msrv }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ matrix.msrv }}
+      - uses: Swatinem/rust-cache@v2
+      - name: Downgrade  dependencies
+        # These packages have newer requirements for MSRV
+        run: |
+          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
+          cargo update -p aws-sdk-dynamodb --precise 1.55.0
+          cargo update -p aws-config --precise 1.5.10
+          cargo update -p aws-sdk-kms --precise 1.51.0
+          cargo update -p aws-sdk-s3 --precise 1.65.0
+          cargo update -p aws-sdk-sso --precise 1.50.0
+          cargo update -p aws-sdk-ssooidc --precise 1.51.0
+          cargo update -p aws-sdk-sts --precise 1.51.0
+          cargo update -p home --precise 0.5.9
+      - name: cargo +${{ matrix.msrv }} check
+        run: cargo check --profile ci --workspace --tests --benches --all-features

.github/workflows/trigger-vectordb-recipes.yml

@@ -1,26 +0,0 @@
-name: Trigger vectordb-recipers workflow
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    paths:
-      - .github/workflows/trigger-vectordb-recipes.yml
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Trigger vectordb-recipes workflow
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
-          script: |
-            const result = await github.rest.actions.createWorkflowDispatch({
-                owner: 'lancedb',
-                repo: 'vectordb-recipes',
-                workflow_id: 'examples-test.yml',
-                ref: 'main'
-            });
-            console.log(result);

.github/workflows/update_package_lock/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock
-description: "Update node's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./node
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/update_package_lock_nodejs/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock_nodejs
-description: "Update nodejs's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./nodejs
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/upload_wheel/action.yml

@@ -17,11 +17,12 @@ runs:
     run: |
       python -m pip install --upgrade pip
       pip install twine
+      python3 -m pip install --upgrade pkginfo
   - name: Choose repo
     shell: bash
     id: choose_repo
     run: |
-      if [ ${{ github.ref }} == "*beta*" ]; then
+      if [[ ${{ github.ref }} == *beta* ]]; then
         echo "repo=fury" >> $GITHUB_OUTPUT
       else
         echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -32,7 +33,7 @@ runs:
       FURY_TOKEN: ${{ inputs.fury_token }}
       PYPI_TOKEN: ${{ inputs.pypi_token }}
     run: |
-      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
+      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
         WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
         echo "Uploading $WHEEL to Fury"
         curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/

.gitignore

@@ -1,4 +1,5 @@
 .idea
+*.swp
 **/*.whl
 *.egg-info
 **/__pycache__
@@ -9,7 +10,6 @@ venv
 .vscode
 .zed
 rust/target
-rust/Cargo.lock
 
 site
 
@@ -32,9 +32,6 @@ python/dist
 *.node
 **/node_modules
 **/.DS_Store
-node/dist
-node/examples/**/package-lock.json
-node/examples/**/dist
 nodejs/lancedb/native*
 dist
 
@@ -42,5 +39,3 @@ dist
 target
 
 **/sccache.log
-
-Cargo.lock

.pre-commit-config.yaml

@@ -1,21 +1,27 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
--   repo: https://github.com/astral-sh/ruff-pre-commit
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.2.2
+    rev: v0.9.9
     hooks:
-    - id: ruff
-- repo: local
-  hooks:
-    - id: local-biome-check
-      name: biome check
-      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
-      language: system
-      types: [text]
-      files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*
+      - id: ruff
+  # - repo: https://github.com/RobertCraigie/pyright-python
+  #   rev: v1.1.395
+  #   hooks:
+  #     - id: pyright
+  #       args: ["--project", "python"]
+  #       additional_dependencies: [pyarrow-stubs]
+  - repo: local
+    hooks:
+      - id: local-biome-check
+        name: biome check
+        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+        language: system
+        types: [text]
+        files: "nodejs/.*"
+        exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*

AGENTS.md

@@ -0,0 +1,101 @@
+LanceDB is a database designed for retrieval, including vector, full-text, and hybrid search.
+It is a wrapper around Lance. There are two backends: local (in-process like SQLite) and
+remote (against LanceDB Cloud).
+
+The core of LanceDB is written in Rust. There are bindings in Python, Typescript, and Java.
+
+Project layout:
+
+* `rust/lancedb`: The LanceDB core Rust implementation.
+* `python`: The Python bindings, using PyO3.
+* `nodejs`: The Typescript bindings, using napi-rs
+* `java`: The Java bindings
+
+Common commands:
+
+* Check for compiler errors: `cargo check --quiet --features remote --tests --examples`
+* Run tests: `cargo test --quiet --features remote --tests`
+* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
+* Lint: `cargo clippy --quiet --features remote --tests --examples`
+* Format: `cargo fmt --all`
+
+Before committing changes, run formatting.
+
+## Coding tips
+
+* When writing Rust doctests for things that require a connection or table reference,
+  write them as a function instead of a fully executable test. This allows type checking
+  to run but avoids needing a full test environment. For example:
+    ```rust
+    /// ```
+    /// use lance_index::scalar::FullTextSearchQuery;
+    /// use lancedb::query::{QueryBase, ExecutableQuery};
+    ///
+    /// # use lancedb::Table;
+    /// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let results = table.query()
+    ///     .full_text_search(FullTextSearchQuery::new("hello world".into()))
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ```
+
+## Example plan: adding a new method on Table
+
+Adding a new method involves first adding it to the Rust core, then exposing it
+in the Python and TypeScript bindings. There are both local and remote tables.
+Remote tables are implemented via a HTTP API and require the `remote` cargo
+feature flag to be enabled. Python has both sync and async methods.
+
+Rust core changes:
+
+1. Add method on `Table` struct in `rust/lancedb/src/table.rs` (calls `BaseTable` trait).
+2. Add method to `BaseTable` trait in `rust/lancedb/src/table.rs`.
+3. Implement new trait method on `NativeTable` in `rust/lancedb/src/table.rs`.
+    * Test with unit test in `rust/lancedb/src/table.rs`.
+4. Implement new trait method on `RemoteTable` in `rust/lancedb/src/remote/table.rs`.
+    * Test with unit test in `rust/lancedb/src/remote/table.rs` against mocked endpoint.
+
+Python bindings changes:
+
+1. Add PyO3 method binding in `python/src/table.rs`. Run `make develop` to compile bindings.
+2. Add types for PyO3 method in `python/python/lancedb/_lancedb.pyi`.
+3. Add method to `AsyncTable` class in `python/python/lancedb/table.py`.
+4. Add abstract method to `Table` abstract base class in `python/python/lancedb/table.py`.
+5. Add concrete sync method to `LanceTable` class in `python/python/lancedb/table.py`.
+    * Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
+6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
+7. Add unit test in `python/tests/test_table.py`.
+
+TypeScript bindings changes:
+
+1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
+2. Run `npm run build` to generate TypeScript definitions.
+3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
+4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
+    * Note: despite the name, this class is also used for remote tables.
+5. Add test in `nodejs/__test__/table.test.ts`.
+6. Run `npm run docs` to generate TypeScript documentation.
+
+## Review Guidelines
+
+Please consider the following when reviewing code contributions.
+
+### Rust API design
+* Design public APIs so they can be evolved easily in the future without breaking
+  changes. Often this means using builder patterns or options structs instead of
+  long argument lists.
+* For public APIs, prefer inputs that use `Into<T>` or `AsRef<T>` traits to allow
+  more flexible inputs. For example, use `name: Into<String>` instead of `name: String`,
+  so we don't have to write `func("my_string".to_string())`.
+
+### Testing
+* Ensure all new public APIs have documentation and examples.
+* Ensure that all bugfixes and features have corresponding tests. **We do not merge
+  code without tests.**
+
+### Documentation
+* New features must include updates to the rust documentation comments. Link to
+  relevant structs and methods to increase the value of documentation.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

CONTRIBUTING.md

@@ -0,0 +1,78 @@
+# Contributing to LanceDB
+
+LanceDB is an open-source project and we welcome contributions from the community.
+This document outlines the process for contributing to LanceDB.
+
+## Reporting Issues
+
+If you encounter a bug or have a feature request, please open an issue on the
+[GitHub issue tracker](https://github.com/lancedb/lancedb).
+
+## Picking an issue
+
+We track issues on the GitHub issue tracker. If you are looking for something to
+work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
+
+If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
+
+## Configuring Git
+
+First, fork the repository on GitHub, then clone your fork:
+
+```bash
+git clone https://github.com/<username>/lancedb.git
+cd lancedb
+```
+
+Then add the main repository as a remote:
+
+```bash
+git remote add upstream https://github.com/lancedb/lancedb.git
+git fetch upstream
+```
+
+## Setting up your development environment
+
+We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
+
+* [Python](python/CONTRIBUTING.md)
+* [Typescript](nodejs/CONTRIBUTING.md)
+<!-- TODO: add Java contributing guide -->
+* [Documentation](docs/README.md)
+
+
+## Best practices for pull requests
+
+For the best chance of having your pull request accepted, please follow these guidelines:
+
+1. Unit test all bug fixes and new features. Your code will not be merged if it
+   doesn't have tests.
+1. If you change the public API, update the documentation in the `docs` directory.
+1. Aim to minimize the number of changes in each pull request. Keep to solving
+   one problem at a time, when possible.
+1. Before marking a pull request ready-for-review, do a self review of your code.
+   Is it clear why you are making the changes? Are the changes easy to understand?
+1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
+    * New feature: `feat: adding foo API`
+    * Bug fix: `fix: issue with foo API`
+    * Documentation change: `docs: adding foo API documentation`
+1. If your pull request is a work in progress, leave the pull request as a draft.
+   We will assume the pull request is ready for review when it is opened.
+1. When writing tests, test the error cases. Make sure they have understandable
+   error messages.
+
+## Project structure
+
+The core library is written in Rust. The Python, Typescript, and Java libraries
+are wrappers around the Rust library.
+
+* `src/lancedb`: Rust library source code
+* `python`: Python package source code
+* `nodejs`: Typescript package source code
+* `node`: **Deprecated** Typescript package source code
+* `java`: Java package source code
+* `docs`: Documentation source code
+
+## Release process
+
+For information on the release process, see: [release_process.md](release_process.md)

Cargo.toml

@@ -1,11 +1,5 @@
 [workspace]
-members = [
-    "rust/ffi/node",
-    "rust/lancedb",
-    "nodejs",
-    "python",
-    "java/core/lancedb-jni",
-]
+members = ["rust/lancedb", "nodejs", "python", "java/core/lancedb-jni"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -18,35 +12,68 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
+rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.16.1", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.16.1" }
-lance-linalg = { "version" = "=0.16.1" }
-lance-testing = { "version" = "=0.16.1" }
-lance-datafusion = { "version" = "=0.16.1" }
-lance-encoding = { "version" = "=0.16.1" }
+lance = { "version" = "=1.1.0-beta.1", default-features = false, "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=1.1.0-beta.1", default-features = false, "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=1.1.0-beta.1", default-features = false, "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "52.2", optional = false }
-arrow-array = "52.2"
-arrow-data = "52.2"
-arrow-ipc = "52.2"
-arrow-ord = "52.2"
-arrow-schema = "52.2"
-arrow-arith = "52.2"
-arrow-cast = "52.2"
+arrow = { version = "56.2", optional = false }
+arrow-array = "56.2"
+arrow-data = "56.2"
+arrow-ipc = "56.2"
+arrow-ord = "56.2"
+arrow-schema = "56.2"
+arrow-select = "56.2"
+arrow-cast = "56.2"
 async-trait = "0"
-chrono = "0.4.35"
-datafusion-physical-plan = "40.0"
-half = { "version" = "=2.4.1", default-features = false, features = [
+datafusion = { version = "50.1", default-features = false }
+datafusion-catalog = "50.1"
+datafusion-common = { version = "50.1", default-features = false }
+datafusion-execution = "50.1"
+datafusion-expr = "50.1"
+datafusion-physical-plan = "50.1"
+env_logger = "0.11"
+half = { "version" = "2.6.0", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
-object_store = "0.10.2"
+moka = { version = "0.12", features = ["future"] }
+object_store = "0.12.0"
 pin-project = "1.0.7"
-snafu = "0.7.4"
+rand = "0.9"
+snafu = "0.8"
 url = "2"
 num-traits = "0.2"
 regex = "1.10"
 lazy_static = "1"
+semver = "1.0.25"
+chrono = "0.4"
+
+[profile.ci]
+debug = "line-tables-only"
+inherits = "dev"
+incremental = false
+
+# This rule applies to every package except workspace members (dependencies
+# such as `arrow` and `tokio`). It disables debug info and related features on
+# dependencies so their binaries stay smaller, improving cache reuse.
+[profile.ci.package."*"]
+debug = false
+debug-assertions = false
+strip = "debuginfo"
+incremental = false

README.md

@@ -1,85 +1,97 @@
+<a href="https://cloud.lancedb.com" target="_blank">
+  <img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
+</a>
 <div align="center">
-<p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
+[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
+[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
+[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
+[![Discord](https://img.shields.io/badge/-Discord-100000?style=for-the-badge&logo=discord&logoColor=white&labelColor=645cfb&color=645cfb)](https://discord.gg/zMM32dvNtd)
+[![Twitter](https://img.shields.io/badge/-Twitter-100000?style=for-the-badge&logo=x&logoColor=white&labelColor=645cfb&color=645cfb)](https://twitter.com/lancedb)
+[![LinkedIn](https://img.shields.io/badge/-LinkedIn-100000?style=for-the-badge&logo=linkedin&logoColor=white&labelColor=645cfb&color=645cfb)](https://www.linkedin.com/company/lancedb/)
 
-**Developer-friendly, database for multimodal AI**
 
-<a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-<a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/)
-[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd)
-[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
+<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
 
-</p>
+# **The Multimodal AI Lakehouse**
 
-<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
+[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
+
+**The ultimate multimodal data platform for AI/ML applications.** 
+
+LanceDB is designed for fast, scalable, and production-ready vector search. It is built on top of the Lance columnar format. You can store, index, and search over petabytes of multimodal data and vectors with ease. 
+LanceDB is a central location where developers can build, train and analyze their AI workloads.
 
-</p>
 </div>
 
-<hr />
+<br>
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
+## **Demo: Multimodal Search by Keyword, Vector or with SQL**
+<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
 
-The key features of LanceDB include:
+## **Star LanceDB to get updates!**
 
-* Production-scale vector search with no servers to manage.
+<details>
+<summary>⭐ Click here ⭐  to see how fast we're growing!</summary>
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=lancedb/lancedb&theme=dark&type=Date">
+  <img width="100%" src="https://api.star-history.com/svg?repos=lancedb/lancedb&theme=dark&type=Date">
+</picture>
+</details>
 
-* Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
+## **Key Features**:
 
-* Support for vector similarity search, full-text search and SQL.
+- **Fast Vector Search**: Search billions of vectors in milliseconds with state-of-the-art indexing.
+- **Comprehensive Search**: Support for vector similarity search, full-text search and SQL.
+- **Multimodal Support**: Store, query and filter vectors, metadata and multimodal data (text, images, videos, point clouds, and more).
+- **Advanced Features**: Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure. GPU support in building vector index.
 
-* Native Python and Javascript/Typescript support.
+### **Products**:
+- **Open Source & Local**: 100% open source, runs locally or in your cloud. No vendor lock-in.
+- **Cloud and Enterprise**: Production-scale vector search with no servers to manage. Complete data sovereignty and security.
 
-* Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
+### **Ecosystem**:
+- **Columnar Storage**: Built on the Lance columnar format for efficient storage and analytics.
+- **Seamless Integration**: Python, Node.js, Rust, and REST APIs for easy integration. Native Python and Javascript/Typescript support.
+- **Rich Ecosystem**: Integrations with [**LangChain** 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [**LlamaIndex** 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-* GPU support in building vector index(*).
+## **How to Install**:
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally. 
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+**API & SDK:** We also support Python, Typescript and Rust SDKs
 
-## Quick Start
+| Interface | Documentation |
+|-----------|---------------|
+| Python SDK | https://lancedb.github.io/lancedb/python/python/ |
+| Typescript SDK | https://lancedb.github.io/lancedb/js/globals/ |
+| Rust SDK | https://docs.rs/lancedb/latest/lancedb/index.html |
+| REST API | https://docs.lancedb.com/api-reference/introduction |
 
-**Javascript**
-```shell
-npm install @lancedb/lancedb
-```
+## **Join Us and Contribute**
 
-```javascript
-import * as lancedb from "@lancedb/lancedb";
+We welcome contributions from everyone! Whether you're a developer, researcher, or just someone who wants to help out. 
 
-const db = await lancedb.connect("data/sample-lancedb");
-const table = await db.createTable("vectors", [
-	{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-	{ id: 2, vector: [1.1, 1.2], item: "bar", price: 50 },
-], {mode: 'overwrite'});
+If you have any suggestions or feature requests, please feel free to open an issue on GitHub or discuss it on our [**Discord**](https://discord.gg/G5DcmnZWKB) server.
+
+[**Check out the GitHub Issues**](https://github.com/lancedb/lancedb/issues) if you would like to work on the features that are planned for the future. If you have any suggestions or feature requests, please feel free to open an issue on GitHub. 
+
+## **Contributors**
+
+<a href="https://github.com/lancedb/lancedb/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=lancedb/lancedb" />
+</a>
 
 
-const query = table.vectorSearch([0.1, 0.3]).limit(2);
-const results = await query.toArray();
+## **Stay in Touch With Us**
+<div align="center">
 
-// You can also search for rows by specific criteria without involving a vector search.
-const rowsByCriteria = await table.query().where("price >= 10").toArray();
-```
+</br>
 
-**Python**
-```shell
-pip install lancedb
-```
+[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
+[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
+[![Discord](https://img.shields.io/badge/-Discord-100000?style=for-the-badge&logo=discord&logoColor=white&labelColor=645cfb&color=645cfb)](https://discord.gg/zMM32dvNtd)
+[![Twitter](https://img.shields.io/badge/-Twitter-100000?style=for-the-badge&logo=x&logoColor=white&labelColor=645cfb&color=645cfb)](https://twitter.com/lancedb)
+[![LinkedIn](https://img.shields.io/badge/-LinkedIn-100000?style=for-the-badge&logo=linkedin&logoColor=white&labelColor=645cfb&color=645cfb)](https://www.linkedin.com/company/lancedb/)
 
-```python
-import lancedb
-
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-table = db.create_table("my_table",
-                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_pandas()
-```
-
-## Blogs, Tutorials & Videos
-* 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>
+</div>

ci/build_linux_artifacts.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-ARCH=${1:-x86_64}
-
-# We pass down the current user so that when we later mount the local files 
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_node
-docker build \
-    -t lancedb-node-manylinux \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
-
-# We turn on memory swap to avoid OOM killer
-docker run \
-    -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
-    lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH

ci/build_linux_artifacts_nodejs.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-ARCH=${1:-x86_64}
-
-# We pass down the current user so that when we later mount the local files
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_node
-docker build \
-    -t lancedb-node-manylinux-$ARCH \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
-
-# We turn on memory swap to avoid OOM killer
-docker run \
-    -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
-    lancedb-node-manylinux-$ARCH \
-    bash ci/manylinux_node/build_lancedb.sh $ARCH

ci/build_macos_artifacts.sh

@@ -1,34 +0,0 @@
-# Builds the macOS artifacts (node binaries).
-# Usage: ./ci/build_macos_artifacts.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-set -e
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/ffi/node
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd node
-    echo "Building node library for $1"
-    npm run build-release -- --target $1
-    npm run pack-build -- --target $1
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/build_macos_artifacts_nodejs.sh

@@ -1,34 +0,0 @@
-# Builds the macOS artifacts (nodejs binaries).
-# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-set -e
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/lancedb
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd nodejs
-    echo "Building nodejs library for $1"
-    export RUST_TARGET=$1
-    npm run build-release
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/build_windows_artifacts.ps1

@@ -1,41 +0,0 @@
-# Builds the Windows artifacts (node binaries).
-# Usage:  .\ci\build_windows_artifacts.ps1 [target]
-# Targets supported:
-# - x86_64-pc-windows-msvc
-# - i686-pc-windows-msvc
-
-function Prebuild-Rust {
-    param (
-        [string]$target
-    )
-
-    # Building here for the sake of easier debugging.
-    Push-Location -Path "rust/ffi/node"
-    Write-Host "Building rust library for $target"
-    $env:RUST_BACKTRACE=1
-    cargo build --release --target $target
-    Pop-Location
-}
-
-function Build-NodeBinaries {
-    param (
-        [string]$target
-    )
-
-    Push-Location -Path "node"
-    Write-Host "Building node library for $target"
-    npm run build-release -- --target $target
-    npm run pack-build -- --target $target
-    Pop-Location
-}
-
-$targets = $args[0]
-if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
-}
-
-Write-Host "Building artifacts for targets: $targets"
-foreach ($target in $targets) {
-    Prebuild-Rust $target
-    Build-NodeBinaries $target
-}

ci/build_windows_artifacts_nodejs.ps1

@@ -1,41 +0,0 @@
-# Builds the Windows artifacts (nodejs binaries).
-# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
-# Targets supported:
-# - x86_64-pc-windows-msvc
-# - i686-pc-windows-msvc
-
-function Prebuild-Rust {
-    param (
-        [string]$target
-    )
-
-    # Building here for the sake of easier debugging.
-    Push-Location -Path "rust/lancedb"
-    Write-Host "Building rust library for $target"
-    $env:RUST_BACKTRACE=1
-    cargo build --release --target $target
-    Pop-Location
-}
-
-function Build-NodeBinaries {
-    param (
-        [string]$target
-    )
-
-    Push-Location -Path "nodejs"
-    Write-Host "Building nodejs library for $target"
-    $env:RUST_TARGET=$target
-    npm run build-release
-    Pop-Location
-}
-
-$targets = $args[0]
-if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
-}
-
-Write-Host "Building artifacts for targets: $targets"
-foreach ($target in $targets) {
-    Prebuild-Rust $target
-    Build-NodeBinaries $target
-}

ci/check_lance_release.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Determine whether a newer Lance tag exists and expose results for CI."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, List, Sequence, Tuple, Union
+
+try:  # Python >=3.11
+    import tomllib  # type: ignore
+except ModuleNotFoundError:  # pragma: no cover - fallback for older Python
+    import tomli as tomllib  # type: ignore
+
+LANCE_REPO = "lance-format/lance"
+
+SEMVER_RE = re.compile(
+    r"^\s*(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
+    r"(?:-(?P<prerelease>[0-9A-Za-z.-]+))?"
+    r"(?:\+[0-9A-Za-z.-]+)?\s*$"
+)
+
+
+@dataclass(frozen=True)
+class SemVer:
+    major: int
+    minor: int
+    patch: int
+    prerelease: Tuple[Union[int, str], ...]
+
+    def __lt__(self, other: "SemVer") -> bool:  # pragma: no cover - simple comparison
+        if (self.major, self.minor, self.patch) != (other.major, other.minor, other.patch):
+            return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
+        if self.prerelease == other.prerelease:
+            return False
+        if not self.prerelease:
+            return False  # release > anything else
+        if not other.prerelease:
+            return True
+        for left, right in zip(self.prerelease, other.prerelease):
+            if left == right:
+                continue
+            if isinstance(left, int) and isinstance(right, int):
+                return left < right
+            if isinstance(left, int):
+                return True
+            if isinstance(right, int):
+                return False
+            return str(left) < str(right)
+        return len(self.prerelease) < len(other.prerelease)
+
+    def __eq__(self, other: object) -> bool:  # pragma: no cover - trivial
+        if not isinstance(other, SemVer):
+            return NotImplemented
+        return (
+            self.major == other.major
+            and self.minor == other.minor
+            and self.patch == other.patch
+            and self.prerelease == other.prerelease
+        )
+
+
+def parse_semver(raw: str) -> SemVer:
+    match = SEMVER_RE.match(raw)
+    if not match:
+        raise ValueError(f"Unsupported version format: {raw}")
+    prerelease = match.group("prerelease")
+    parts: Tuple[Union[int, str], ...] = ()
+    if prerelease:
+        parsed: List[Union[int, str]] = []
+        for piece in prerelease.split("."):
+            if piece.isdigit():
+                parsed.append(int(piece))
+            else:
+                parsed.append(piece)
+        parts = tuple(parsed)
+    return SemVer(
+        major=int(match.group("major")),
+        minor=int(match.group("minor")),
+        patch=int(match.group("patch")),
+        prerelease=parts,
+    )
+
+
+@dataclass
+class TagInfo:
+    tag: str  # e.g. v1.0.0-beta.2
+    version: str  # e.g. 1.0.0-beta.2
+    semver: SemVer
+
+
+def run_command(cmd: Sequence[str]) -> str:
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"Command {' '.join(cmd)} failed with {result.returncode}: {result.stderr.strip()}"
+        )
+    return result.stdout.strip()
+
+
+def fetch_remote_tags() -> List[TagInfo]:
+    output = run_command(
+        [
+            "gh",
+            "api",
+            "-X",
+            "GET",
+            f"repos/{LANCE_REPO}/git/refs/tags",
+            "--paginate",
+            "--jq",
+            ".[].ref",
+        ]
+    )
+    tags: List[TagInfo] = []
+    for line in output.splitlines():
+        ref = line.strip()
+        if not ref.startswith("refs/tags/v"):
+            continue
+        tag = ref.split("refs/tags/")[-1]
+        version = tag.lstrip("v")
+        try:
+            tags.append(TagInfo(tag=tag, version=version, semver=parse_semver(version)))
+        except ValueError:
+            continue
+    if not tags:
+        raise RuntimeError("No Lance tags could be parsed from GitHub API output")
+    return tags
+
+
+def read_current_version(repo_root: Path) -> str:
+    cargo_path = repo_root / "Cargo.toml"
+    with cargo_path.open("rb") as fh:
+        data = tomllib.load(fh)
+    try:
+        deps = data["workspace"]["dependencies"]
+        entry = deps["lance"]
+    except KeyError as exc:  # pragma: no cover - configuration guard
+        raise RuntimeError("Failed to locate workspace.dependencies.lance in Cargo.toml") from exc
+
+    if isinstance(entry, str):
+        raw_version = entry
+    elif isinstance(entry, dict):
+        raw_version = entry.get("version", "")
+    else:  # pragma: no cover - defensive
+        raise RuntimeError("Unexpected lance dependency format")
+
+    raw_version = raw_version.strip()
+    if not raw_version:
+        raise RuntimeError("lance dependency does not declare a version")
+    return raw_version.lstrip("=")
+
+
+def determine_latest_tag(tags: Iterable[TagInfo]) -> TagInfo:
+    return max(tags, key=lambda tag: tag.semver)
+
+
+def write_outputs(args: argparse.Namespace, payload: dict) -> None:
+    target = getattr(args, "github_output", None)
+    if not target:
+        return
+    with open(target, "a", encoding="utf-8") as handle:
+        for key, value in payload.items():
+            handle.write(f"{key}={value}\n")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--repo-root",
+        default=Path(__file__).resolve().parents[1],
+        type=Path,
+        help="Path to the lancedb repository root",
+    )
+    parser.add_argument(
+        "--github-output",
+        default=os.environ.get("GITHUB_OUTPUT"),
+        help="Optional file path for writing GitHub Action outputs",
+    )
+    args = parser.parse_args(argv)
+
+    repo_root = Path(args.repo_root)
+    current_version = read_current_version(repo_root)
+    current_semver = parse_semver(current_version)
+
+    tags = fetch_remote_tags()
+    latest = determine_latest_tag(tags)
+    needs_update = latest.semver > current_semver
+
+    payload = {
+        "current_version": current_version,
+        "current_tag": f"v{current_version}",
+        "latest_version": latest.version,
+        "latest_tag": latest.tag,
+        "needs_update": "true" if needs_update else "false",
+    }
+
+    print(json.dumps(payload))
+    write_outputs(args, payload)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/create_lancedb_test_connection.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+export RUST_LOG=info
+exec ./lancedb server --port 0 --sql-port 0  --data-dir "${1}"

ci/manylinux_node/Dockerfile

@@ -1,31 +0,0 @@
-# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
-# This container allows building the node modules native libraries in an 
-# environment with a very old glibc, so that we are compatible with a wide
-# range of linux distributions.
-ARG ARCH=x86_64
-
-FROM quay.io/pypa/manylinux_2_28_${ARCH}
-
-ARG ARCH=x86_64
-ARG DOCKER_USER=default_user
-
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
-# Protobuf is also installed as root.
-COPY install_protobuf.sh install_protobuf.sh
-RUN ./install_protobuf.sh ${ARCH}
-
-ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user, but only if it doesn't exist
-RUN echo ${ARCH} && id -u ${DOCKER_USER} >/dev/null 2>&1 || adduser --user-group --create-home --uid ${DOCKER_USER} build_user
-
-# We switch to the user to install Rust and Node, since those like to be 
-# installed at the user level.
-USER ${DOCKER_USER}
-
-COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
-RUN cp /prepare_manylinux_node.sh $HOME/ && \
-    cd $HOME && \
-    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_node/build_lancedb.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
-
-cd nodejs
-npm ci
-npm run build-release

ci/manylinux_node/build_vectordb.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-# Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else 
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
-
-cd node
-npm ci
-npm run build-release
-npm run pack-build

ci/manylinux_node/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1v \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

ci/manylinux_node/install_protobuf.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# Installs protobuf compiler. Should be run as root.
-set -e
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=x86_64
-else
-    # gnu target
-    ARCH=aarch_64
-fi
-
-PB_REL=https://github.com/protocolbuffers/protobuf/releases
-PB_VERSION=23.1
-curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_node/prepare_manylinux_node.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-install_node() {
-    echo "Installing node..."
-
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-
-    source "$HOME"/.bashrc
-
-    nvm install --no-progress 18
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-install_node
-install_rust

ci/mock_openai.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
+import argparse
+import json
+import http.server
+
+
+class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers["Content-Length"])
+        post_data = self.rfile.read(content_length)
+        post_data = json.loads(post_data.decode("utf-8"))
+        # See: https://platform.openai.com/docs/api-reference/embeddings/create
+
+        if isinstance(post_data["input"], str):
+            num_inputs = 1
+        else:
+            num_inputs = len(post_data["input"])
+
+        model = post_data.get("model", "text-embedding-ada-002")
+
+        data = []
+        for i in range(num_inputs):
+            data.append({
+                "object": "embedding",
+                "embedding": [0.1] * 1536,
+                "index": i,
+            })
+
+        response = {
+            "object": "list",
+            "data": data,
+            "model": model,
+            "usage": {
+                "prompt_tokens": 0,
+                "total_tokens": 0,
+            }
+        }
+
+        self.send_response(200)
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode("utf-8"))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
+    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
+    args = parser.parse_args()
+    port = args.port
+
+    print(f"server started on port {port}. Press Ctrl-C to stop.")
+    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
+
+    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
+        server.serve_forever()

ci/parse_requirements.py

@@ -0,0 +1,41 @@
+import argparse
+import toml
+
+
+def parse_dependencies(pyproject_path, extras=None):
+    with open(pyproject_path, "r") as file:
+        pyproject = toml.load(file)
+
+    dependencies = pyproject.get("project", {}).get("dependencies", [])
+    for dependency in dependencies:
+        print(dependency)
+
+    optional_dependencies = pyproject.get("project", {}).get(
+        "optional-dependencies", {}
+    )
+
+    if extras:
+        for extra in extras.split(","):
+            for dep in optional_dependencies.get(extra, []):
+                print(dep)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate requirements.txt from pyproject.toml"
+    )
+    parser.add_argument("path", type=str, help="Path to pyproject.toml")
+    parser.add_argument(
+        "--extras",
+        type=str,
+        help="Comma-separated list of extras to include",
+        default="",
+    )
+
+    args = parser.parse_args()
+
+    parse_dependencies(args.path, args.extras)
+
+
+if __name__ == "__main__":
+    main()

ci/run_with_docker_compose.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#
+# A script for running the given command together with a docker compose environment.
+#
+
+# Bring down the docker setup once the command is done running.
+tear_down() {
+    docker compose -p fixture down
+}
+trap tear_down EXIT
+
+set +xe
+
+# Clean up any existing docker setup and bring up a new one.
+docker compose -p fixture up --detach --wait || exit 1
+
+"${@}"

ci/run_with_test_connection.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+#
+# A script for running the given command together with the lancedb cli.
+#
+
+die() {
+    echo $?
+    exit 1
+}
+
+check_command_exists() {
+    command="${1}"
+    which ${command} &> /dev/null || \
+        die "Unable to locate command: ${command}. Did you install it?"
+}
+
+if [[ ! -e ./lancedb ]]; then
+    if [[ -v SOPHON_READ_TOKEN ]]; then
+        INPUT="lancedb-linux-x64"
+        gh release \
+            --repo lancedb/lancedb \
+            download ci-support-binaries \
+            --pattern "${INPUT}" \
+            || die "failed to fetch cli."
+        check_command_exists openssl
+        openssl enc -aes-256-cbc \
+            -d -pbkdf2 \
+            -pass "env:SOPHON_READ_TOKEN" \
+            -in "${INPUT}" \
+            -out ./lancedb-linux-x64.tar.gz \
+            || die "openssl failed"
+        TARGET="${INPUT}.tar.gz"
+    else
+        ARCH="x64"
+        if [[ $OSTYPE == 'darwin'* ]]; then
+            UNAME=$(uname -m)
+            if [[ $UNAME == 'arm64' ]]; then
+                ARCH='arm64'
+            fi
+            OSTYPE="macos"
+        elif [[ $OSTYPE == 'linux'* ]]; then
+            if [[ $UNAME == 'aarch64' ]]; then
+                ARCH='arm64'
+            fi
+            OSTYPE="linux"
+        else
+            die "unknown OSTYPE: $OSTYPE"
+        fi
+
+        check_command_exists gh
+        TARGET="lancedb-${OSTYPE}-${ARCH}.tar.gz"
+        gh release \
+            --repo lancedb/sophon \
+            download lancedb-cli-v0.0.3 \
+            --pattern "${TARGET}" \
+            || die "failed to fetch cli."
+    fi
+
+    check_command_exists tar
+    tar xvf "${TARGET}" || die "tar failed."
+    [[ -e ./lancedb ]] || die "failed to extract lancedb."
+fi
+
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+export CREATE_LANCEDB_TEST_CONNECTION_SCRIPT="${SCRIPT_DIR}/create_lancedb_test_connection.sh"
+
+"${@}"

ci/set_lance_version.py

@@ -0,0 +1,270 @@
+import argparse
+import re
+import sys
+import json
+
+LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
+
+
+def run_command(command: str) -> str:
+    """
+    Run a shell command and return stdout as a string.
+    If exit code is not 0, raise an exception with the stderr output.
+    """
+    import subprocess
+
+    result = subprocess.run(command, shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise Exception(f"Command failed with error: {result.stderr.strip()}")
+    return result.stdout.strip()
+
+
+def get_latest_stable_version() -> str:
+    version_line = run_command("cargo info lance | grep '^version:'")
+    # Example output: "version: 0.35.0 (latest 0.37.0)"
+    match = re.search(r'\(latest ([0-9.]+)\)', version_line)
+    if match:
+        return match.group(1)
+    # Fallback: use the first version after 'version:'
+    return version_line.split("version:")[1].split()[0].strip()
+
+
+def get_latest_preview_version() -> str:
+    lance_tags = run_command(
+        f"git ls-remote --tags {LANCE_GIT_URL} | grep 'refs/tags/v[0-9beta.-]\\+$'"
+    ).splitlines()
+    lance_tags = (
+        tag.split("refs/tags/")[1]
+        for tag in lance_tags
+        if "refs/tags/" in tag and "beta" in tag
+    )
+    from packaging.version import Version
+
+    latest = max(
+        (tag[1:] for tag in lance_tags if tag.startswith("v")), key=lambda t: Version(t)
+    )
+    return str(latest)
+
+
+def extract_features(line: str) -> list:
+    """
+    Extracts the features from a line in Cargo.toml.
+    Example: 'lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }'
+    Returns: ['dynamodb']
+    """
+    import re
+
+    match = re.search(r'"features"\s*=\s*\[\s*(.*?)\s*\]', line, re.DOTALL)
+    if match:
+        features_str = match.group(1)
+        return [f.strip().strip('"') for f in features_str.split(",") if f.strip()]
+    return []
+
+
+def extract_default_features(line: str) -> bool:
+    """
+    Checks if default-features = false is present in a line in Cargo.toml.
+    Example: 'lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }'
+    Returns: True if default-features = false is present, False otherwise
+    """
+    import re
+
+    match = re.search(r'default-features\s*=\s*false', line)
+    return match is not None
+
+
+def dict_to_toml_line(package_name: str, config: dict) -> str:
+    """
+    Converts a configuration dictionary to a TOML dependency line.
+    Dictionary insertion order is preserved (Python 3.7+), so the caller
+    controls the order of fields in the output.
+
+    Args:
+        package_name: The name of the package (e.g., "lance", "lance-io")
+        config: Dictionary with keys like "version", "path", "git", "tag", "features", "default-features"
+                The order of keys in this dict determines the order in the output.
+
+    Returns:
+        A properly formatted TOML line with a trailing newline
+    """
+    # If only version is specified, use simple format
+    if len(config) == 1 and "version" in config:
+        return f'{package_name} = "{config["version"]}"\n'
+
+    # Otherwise, use inline table format
+    parts = []
+    for key, value in config.items():
+        if key == "default-features" and not value:
+            parts.append("default-features = false")
+        elif key == "features":
+            parts.append(f'"features" = {json.dumps(value)}')
+        elif isinstance(value, str):
+            parts.append(f'"{key}" = "{value}"')
+        else:
+            # This shouldn't happen with our current usage
+            parts.append(f'"{key}" = {json.dumps(value)}')
+
+    return f'{package_name} = {{ {", ".join(parts)} }}\n'
+
+
+def update_cargo_toml(line_updater):
+    """
+    Updates the Cargo.toml file by applying the line_updater function to each line.
+    The line_updater function should take a line as input and return the updated line.
+    """
+    with open("Cargo.toml", "r") as f:
+        lines = f.readlines()
+
+    new_lines = []
+    lance_line = ""
+    is_parsing_lance_line = False
+    for line in lines:
+        if line.startswith("lance"):
+            # Check if this is a single-line or multi-line entry
+            # Single-line entries either:
+            # 1. End with } (complete inline table)
+            # 2. End with " (simple version string)
+            # Multi-line entries start with { but don't end with }
+            if line.strip().endswith("}") or line.strip().endswith('"'):
+                # Single-line entry - process immediately
+                new_lines.append(line_updater(line))
+            elif "{" in line and not line.strip().endswith("}"):
+                # Multi-line entry - start accumulating
+                lance_line = line
+                is_parsing_lance_line = True
+            else:
+                # Single-line entry without quotes or braces (shouldn't happen but handle it)
+                new_lines.append(line_updater(line))
+        elif is_parsing_lance_line:
+            lance_line += line
+            if line.strip().endswith("}"):
+                new_lines.append(line_updater(lance_line))
+                lance_line = ""
+                is_parsing_lance_line = False
+        else:
+            # Keep the line unchanged
+            new_lines.append(line)
+
+    with open("Cargo.toml", "w") as f:
+        f.writelines(new_lines)
+
+
+def set_stable_version(version: str):
+    """
+    Sets lines to
+    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }
+    lance-io = { "version" = "=0.29.0", default-features = false }
+    ...
+    """
+
+    def line_updater(line: str) -> str:
+        package_name = line.split("=", maxsplit=1)[0].strip()
+
+        # Build config in desired order: version, default-features, features
+        config = {"version": f"={version}"}
+
+        if extract_default_features(line):
+            config["default-features"] = False
+
+        features = extract_features(line)
+        if features:
+            config["features"] = features
+
+        return dict_to_toml_line(package_name, config)
+
+    update_cargo_toml(line_updater)
+
+
+def set_preview_version(version: str):
+    """
+    Sets lines to
+    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.29.0-beta.2", "git" = LANCE_GIT_URL }
+    lance-io = { "version" = "=0.29.0", default-features = false, "tag" = "v0.29.0-beta.2", "git" = LANCE_GIT_URL }
+    ...
+    """
+
+    def line_updater(line: str) -> str:
+        package_name = line.split("=", maxsplit=1)[0].strip()
+        # Build config in desired order: version, default-features, features, tag, git
+        config = {"version": f"={version}"}
+
+        if extract_default_features(line):
+            config["default-features"] = False
+
+        features = extract_features(line)
+        if features:
+            config["features"] = features
+
+        config["tag"] = f"v{version}"
+        config["git"] = LANCE_GIT_URL
+
+        return dict_to_toml_line(package_name, config)
+
+    update_cargo_toml(line_updater)
+
+
+def set_local_version():
+    """
+    Sets lines to
+    lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
+    lance-io = { "path" = "../lance/rust/lance-io", default-features = false }
+    ...
+    """
+
+    def line_updater(line: str) -> str:
+        package_name = line.split("=", maxsplit=1)[0].strip()
+
+        # Build config in desired order: path, default-features, features
+        config = {"path": f"../lance/rust/{package_name}"}
+
+        if extract_default_features(line):
+            config["default-features"] = False
+
+        features = extract_features(line)
+        if features:
+            config["features"] = features
+
+        return dict_to_toml_line(package_name, config)
+
+    update_cargo_toml(line_updater)
+
+
+parser = argparse.ArgumentParser(description="Set the version of the Lance package.")
+parser.add_argument(
+    "version",
+    type=str,
+    help="The version to set for the Lance package. Use 'stable' for the latest stable version, 'preview' for latest preview version, or a specific version number (e.g., '0.1.0'). You can also specify 'local' to use a local path.",
+)
+args = parser.parse_args()
+
+if args.version == "stable":
+    latest_stable_version = get_latest_stable_version()
+    print(
+        f"Found latest stable version: \033[1mv{latest_stable_version}\033[0m",
+        file=sys.stderr,
+    )
+    set_stable_version(latest_stable_version)
+elif args.version == "preview":
+    latest_preview_version = get_latest_preview_version()
+    print(
+        f"Found latest preview version: \033[1mv{latest_preview_version}\033[0m",
+        file=sys.stderr,
+    )
+    set_preview_version(latest_preview_version)
+elif args.version == "local":
+    set_local_version()
+else:
+    # Parse the version number.
+    version = args.version
+    # Ignore initial v if present.
+    if version.startswith("v"):
+        version = version[1:]
+
+    if "beta" in version:
+        set_preview_version(version)
+    else:
+        set_stable_version(version)
+
+print("Updating lockfiles...", file=sys.stderr, end="")
+run_command("cargo metadata > /dev/null")
+print(" done.", file=sys.stderr)

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+
+# dbghelp.lib fwpuclnt.lib arm64rt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# ARM intrinsics
+# original dir: MSVC/
+
+# '__n128x4' redefined in arm_neon.h
+# "arm64_neon.h" included from intrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+# .lib
+
+# _Interlocked intrinsics
+# must always link with arm64rt.lib
+# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
+# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
+
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib

ci/sysroot-x86_64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
+
+# dbghelp.lib fwpuclnt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# x86 intrinsics
+# original dir: MSVC/
+
+# '_mm_movemask_epi8' defined in emmintrin.h
+# '__v4sf' defined in xmmintrin.h
+# '__v2si' defined in mmintrin.h
+# '__m128d' redefined in immintrin.h
+# '__m128i' redefined in intrin.h
+# '_mm_comlt_epu8' defined in ammintrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+# .lib
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib

ci/update_lockfiles.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+AMEND=false
+
+for arg in "$@"; do
+  if [[ "$arg" == "--amend" ]]; then
+    AMEND=true
+  fi
+done
+
+# This updates the lockfile without building
+cargo metadata --quiet > /dev/null
+
+pushd nodejs || exit 1
+npm install --package-lock-only --silent
+popd
+
+if git diff --quiet --exit-code; then
+  echo "No lockfile changes to commit; skipping amend."
+elif $AMEND; then
+  git add Cargo.lock nodejs/package-lock.json
+  git commit --amend --no-edit
+else
+  git add Cargo.lock nodejs/package-lock.json
+  git commit -m "Update lockfiles"
+fi

ci/validate_stable_lance.py

@@ -0,0 +1,34 @@
+import tomllib
+
+found_preview_lance = False
+
+with open("Cargo.toml", "rb") as f:
+    cargo_data = tomllib.load(f)
+
+    for name, dep in cargo_data["workspace"]["dependencies"].items():
+        if name == "lance" or name.startswith("lance-"):
+            if isinstance(dep, str):
+                version = dep
+            elif isinstance(dep, dict):
+                # Version doesn't have the beta tag in it, so we instead look
+                # at the git tag.
+                version = dep.get('tag', dep.get('version'))
+            else:
+                raise ValueError("Unexpected type for dependency: " + str(dep))
+
+            if "beta" in version:
+                found_preview_lance = True
+                print(f"Dependency '{name}' is a preview version: {version}")
+
+with open("python/pyproject.toml", "rb") as f:
+    py_proj_data = tomllib.load(f)
+
+    for dep in py_proj_data["project"]["dependencies"]:
+        if dep.startswith("pylance"):
+            if "b" in dep:
+                found_preview_lance = True
+                print(f"Dependency '{dep}' is a preview version")
+            break  # Only one pylance dependency
+
+if found_preview_lance:
+    raise ValueError("Found preview version of Lance in dependencies")

docs/README.md

@@ -1,44 +1,89 @@
 # LanceDB Documentation
 
-LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
+LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs).
 
-Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
 unreleased features.
 
 ## Building the docs
 
 ### Setup
-1. Install LanceDB. From LanceDB repo root: `pip install -e python`
-2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
-3. Make sure you have node and npm setup
-4. Make sure protobuf and libssl are installed
+1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
+   Run `make develop` to install the Python package.
+2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
 
-### Building node module and create markdown files
+### Preview the docs
 
-See [Javascript docs README](./src/javascript/README.md)
-
-### Build docs
-From LanceDB repo root:
-
-Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-### Run local server
-
-You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
-
-```bash
+```shell
 cd docs
 mkdocs serve
 ```
 
-### Run doctest for typescript example
+If you want to just generate the HTML files:
 
-```bash
-cd lancedb/docs
-npm i
-npm run build
-npm run all
+```shell
+PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
+```
+
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+## Adding examples
+
+To make sure examples are correct, we put examples in test files so they can be
+run as part of our test suites.
+
+You can see the tests are at:
+
+* Python: `python/python/tests/docs`
+* Typescript: `nodejs/examples/`
+
+### Checking python examples
+
+```shell
+cd python
+pytest -vv python/tests/docs
+```
+
+### Checking typescript examples
+
+The `@lancedb/lancedb` package must be built before running the tests:
+
+```shell
+pushd nodejs
+npm ci
+npm run build
+popd
+```
+
+Then you can run the examples by going to the `nodejs/examples` directory and
+running the tests like a normal npm package:
+
+```shell
+pushd nodejs/examples
+npm ci
+npm test
+popd
+```
+
+## API documentation
+
+### Python
+
+The Python API documentation is organized based on the file `docs/src/python/python.md`.
+We manually add entries there so we can control the organization of the reference page.
+**However, this means any new types must be manually added to the file.** No additional
+steps are needed to generate the API documentation.
+
+### Typescript
+
+The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
+
+When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
+The new files should be checked into the repository.
+
+```shell
+pushd nodejs
+npm run docs
+popd
 ```

docs/mkdocs.yml

@@ -4,6 +4,9 @@ repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
+watch:
+  - src
+  - ../python/python
 
 theme:
   name: "material"
@@ -26,6 +29,7 @@ theme:
     - content.code.copy
     - content.tabs.link
     - content.action.edit
+    - content.tooltips
     - toc.follow
     - navigation.top
     - navigation.tabs
@@ -33,9 +37,10 @@ theme:
     - navigation.footer
     - navigation.tracking
     - navigation.instant
+    - content.footnote.tooltips
   icon:
     repo: fontawesome/brands/github
-  custom_dir: overrides
+    annotation: material/arrow-right-circle
 
 plugins:
   - search
@@ -43,7 +48,9 @@ plugins:
   - mkdocstrings:
       handlers:
         python:
-          paths: [../python]
+          # Ensure the handler points to the real package root
+          # so it reads local sources at python/python/lancedb
+          paths: [../python/python]
           options:
             docstring_style: numpy
             heading_level: 3
@@ -52,17 +59,42 @@ plugins:
             show_signature_annotations: true
             show_root_heading: true
             members_order: source
+            docstring_section_style: list
+            signature_crossrefs: true
+            separate_signature: true
           import:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
-  - mkdocs-jupyter
+            - https://docs.pydantic.dev/latest/objects.inv
   - render_swagger:
-      allow_arbitrary_locations : true
+      allow_arbitrary_locations: true
+#   - redirects:
+#       redirect_maps:
+#         # Redirect the home page and other top-level markdown files. This enables maximum SEO benefit
+#         # other sub-pages are handled by the ingected js in overrides/partials/header.html
+#         'index.md': 'https://lancedb.com/docs/'
+#         'guides/tables.md': 'https://lancedb.com/docs/tables/'
+#         'ann_indexes.md': 'https://lancedb.com/docs/indexing/'
+#         'basic.md': 'https://lancedb.com/docs/quickstart/'
+#         'faq.md': 'https://lancedb.com/docs/faq/'
+#         'embeddings/understanding_embeddings.md': 'https://lancedb.com/docs/embedding/'
+#         'integrations.md': 'https://lancedb.com/docs/integrations/'
+#         'examples.md': 'https://lancedb.com/docs/tutorials/'
+#         'concepts/vector_search.md': 'https://lancedb.com/docs/search/vector-search/'
+#         'troubleshooting.md': 'https://lancedb.com/docs/troubleshooting/'
+#         'guides/storage.md': 'https://lancedb.com/docs/storage/integrations'
+
+
 
 markdown_extensions:
   - admonition
   - footnotes
+  - pymdownx.critic
+  - pymdownx.caret
+  - pymdownx.keys
+  - pymdownx.mark
+  - pymdownx.tilde
   - pymdownx.details
   - pymdownx.highlight:
       anchor_linenums: true
@@ -76,191 +108,29 @@ markdown_extensions:
   - pymdownx.tabbed:
       alternate_style: true
   - md_in_html
+  - abbr
   - attr_list
+  - pymdownx.snippets
+  - pymdownx.emoji:
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+  - markdown.extensions.toc:
+      baselevel: 1
+      permalink: ""
 
 nav:
-  - Home:
-      - LanceDB: index.md
-      - 🏃🏼‍♂️ Quick start: basic.md
-      - 📚 Concepts:
-          - Vector search: concepts/vector_search.md
-          - Indexing: concepts/index_ivfpq.md
-          - Storage: concepts/storage.md
-          - Data management: concepts/data_management.md
-      - 🔨 Guides:
-          - Working with tables: guides/tables.md
-          - Building an ANN index: ann_indexes.md
-          - Vector Search: search.md
-          - Full-text search: fts.md
-          - Hybrid search:
-              - Overview: hybrid_search/hybrid_search.md
-              - Comparing Rerankers: hybrid_search/eval.md
-              - Airbnb financial data example: notebooks/hybrid_search.ipynb
-          - Reranking:
-              - Quickstart: reranking/index.md
-              - Cohere Reranker: reranking/cohere.md
-              - Linear Combination Reranker: reranking/linear_combination.md
-              - Reciprocal Rank Fusion Reranker: reranking/rrf.md
-              - Cross Encoder Reranker: reranking/cross_encoder.md
-              - ColBERT Reranker: reranking/colbert.md
-              - Jina Reranker: reranking/jina.md
-              - OpenAI Reranker: reranking/openai.md
-              - Building Custom Rerankers: reranking/custom_reranker.md
-              - Example: notebooks/lancedb_reranking.ipynb
-          - Filtering: sql.md
-          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
-          - Configuring Storage: guides/storage.md
-          - Migration Guide: migration.md
-          - Tuning retrieval performance:
-              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
-              - Reranking: guides/tuning_retrievers/2_reranking.md
-              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
-      - 🧬 Managing embeddings:
-          - Overview: embeddings/index.md
-          - Embedding functions: embeddings/embedding_functions.md
-          - Available models: embeddings/default_embedding_functions.md
-          - User-defined embedding functions: embeddings/custom_embedding_function.md
-          - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
-          - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
-      - 🔌 Integrations:
-          - Tools and data formats: integrations/index.md
-          - Pandas and PyArrow: python/pandas_and_pyarrow.md
-          - Polars: python/polars_arrow.md
-          - DuckDB: python/duckdb.md
-          - LangChain:
-            - LangChain 🔗: integrations/langchain.md
-            - LangChain demo: notebooks/langchain_demo.ipynb
-            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-          - LlamaIndex 🦙:
-            - LlamaIndex docs: integrations/llamaIndex.md
-            - LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
-          - Pydantic: python/pydantic.md
-          - Voxel51: integrations/voxel51.md
-          - PromptTools: integrations/prompttools.md
-      - 🎯 Examples:
-          - Overview: examples/index.md
-          - 🐍 Python:
-              - Overview: examples/examples_python.md
-              - Build From Scratch: examples/python_examples/build_from_scratch.md
-              - Multimodal: examples/python_examples/multimodal.md
-              - Rag: examples/python_examples/rag.md
-              - Miscellaneous: 
-                  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-                  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-                  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-                  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-                  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-          - 👾 JavaScript:
-              - Overview: examples/examples_js.md
-              - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-              - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-              - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-          - 🦀 Rust:
-              - Overview: examples/examples_rust.md
-      - 💭 FAQs: faq.md
-      - ⚙️ API reference:
-          - 🐍 Python: python/python.md
-          - 👾 JavaScript (vectordb): javascript/modules.md
-          - 👾 JavaScript (lancedb): js/globals.md
-          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
-      - ☁️ LanceDB Cloud:
-          - Overview: cloud/index.md
-          - API reference:
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
-
-  - Quick start: basic.md
-  - Concepts:
-      - Vector search: concepts/vector_search.md
-      - Indexing: concepts/index_ivfpq.md
-      - Storage: concepts/storage.md
-      - Data management: concepts/data_management.md
-  - Guides:
-      - Working with tables: guides/tables.md
-      - Building an ANN index: ann_indexes.md
-      - Vector Search: search.md
-      - Full-text search: fts.md
-      - Hybrid search:
-          - Overview: hybrid_search/hybrid_search.md
-          - Comparing Rerankers: hybrid_search/eval.md
-          - Airbnb financial data example: notebooks/hybrid_search.ipynb
-      - Reranking:
-          - Quickstart: reranking/index.md
-          - Cohere Reranker: reranking/cohere.md
-          - Linear Combination Reranker: reranking/linear_combination.md
-          - Reciprocal Rank Fusion Reranker: reranking/rrf.md
-          - Cross Encoder Reranker: reranking/cross_encoder.md
-          - ColBERT Reranker: reranking/colbert.md
-          - Jina Reranker: reranking/jina.md
-          - OpenAI Reranker: reranking/openai.md
-          - Building Custom Rerankers: reranking/custom_reranker.md
-          - Example: notebooks/lancedb_reranking.ipynb
-      - Filtering: sql.md
-      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
-      - Configuring Storage: guides/storage.md
-      - Migration Guide: migration.md
-      - Tuning retrieval performance:
-          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
-          - Reranking: guides/tuning_retrievers/2_reranking.md
-          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
-  - Managing Embeddings:
-      - Overview: embeddings/index.md
-      - Embedding functions: embeddings/embedding_functions.md
-      - Available models: embeddings/default_embedding_functions.md
-      - User-defined embedding functions: embeddings/custom_embedding_function.md
-      - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
-      - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
-  - Integrations:
-      - Overview: integrations/index.md
-      - Pandas and PyArrow: python/pandas_and_pyarrow.md
-      - Polars: python/polars_arrow.md
-      - DuckDB: python/duckdb.md
-      - LangChain 🦜️🔗↗: integrations/langchain.md
-      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-      - LlamaIndex 🦙↗: integrations/llamaIndex.md
-      - Pydantic: python/pydantic.md
-      - Voxel51: integrations/voxel51.md
-      - PromptTools: integrations/prompttools.md
-  - Examples:
-      - examples/index.md
-      - 🐍 Python:
-          - Overview: examples/examples_python.md
-          - Build From Scratch: examples/python_examples/build_from_scratch.md
-          - Multimodal: examples/python_examples/multimodal.md
-          - Rag: examples/python_examples/rag.md
-          - Miscellaneous: 
-              - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-              - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-              - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-      - 👾 JavaScript:
-          - Overview: examples/examples_js.md
-          - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-          - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-          - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-      - 🦀 Rust:
-          - Overview: examples/examples_rust.md
   - API reference:
-      - Overview: api_reference.md
+      - Overview: index.md
       - Python: python/python.md
-      - Javascript (vectordb): javascript/modules.md
-      - Javascript (lancedb): js/globals.md
+      - Javascript/TypeScript: js/globals.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
-  - LanceDB Cloud:
-      - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
 
 extra_css:
   - styles/global.css
   - styles/extra.css
 
 extra_javascript:
-  - "extra_js/init_ask_ai_widget.js"
+  - "extra_js/reo.js"
 
 extra:
   analytics:

docs/openapi.yml

@@ -38,6 +38,13 @@ components:
       required: true
       schema:
         type: string
+    index_name:
+      name: index_name
+      in: path
+      description: name of the index
+      required: true
+      schema:
+        type: string
   responses:
     invalid_request:
       description: Invalid request
@@ -164,7 +171,7 @@ paths:
                 distance_type:
                   type: string
                   description: |
-                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
+                    The distance metric to use for search. l2, Cosine, Dot and Hamming are supported. Default is l2.
                 bypass_vector_index:
                   type: boolean
                   description: |
@@ -443,7 +450,7 @@ paths:
                   type: string
                   nullable: false
                   description: |
-                    The metric type to use for the index. L2, Cosine, Dot are supported.
+                    The metric type to use for the index. l2, Cosine, Dot are supported.
                 index_type:
                   type: string
       responses:
@@ -485,3 +492,22 @@ paths:
           $ref: "#/components/responses/unauthorized"
         "404":
           $ref: "#/components/responses/not_found"
+  /v1/table/{name}/index/{index_name}/drop/:
+    post:
+      description: Drop an index from the table
+      tags:
+        - Tables
+      summary: Drop an index from the table
+      operationId: dropIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+        - $ref: "#/components/parameters/index_name"
+      responses:
+        "200":
+          description: Index successfully dropped
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"

docs/overrides/partials/header.html

@@ -1,176 +0,0 @@
-<!--
-  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to
-  deal in the Software without restriction, including without limitation the
-  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-  sell copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-  IN THE SOFTWARE.
--->
-
-{% set class = "md-header" %}
-{% if "navigation.tabs.sticky" in features %}
-  {% set class = class ~ " md-header--shadow md-header--lifted" %}
-{% elif "navigation.tabs" not in features %}
-  {% set class = class ~ " md-header--shadow" %}
-{% endif %}
-
-<!-- Header -->
-<header class="{{ class }}" data-md-component="header">
-  <nav
-    class="md-header__inner md-grid"
-    aria-label="{{ lang.t('header') }}"
-  >
-
-    <!-- Link to home -->
-    <a
-      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
-      title="{{ config.site_name | e }}"
-      class="md-header__button md-logo"
-      aria-label="{{ config.site_name }}"
-      data-md-component="logo"
-    >
-      {% include "partials/logo.html" %}
-    </a>
-
-    <!-- Button to open drawer -->
-    <label class="md-header__button md-icon" for="__drawer">
-      {% include ".icons/material/menu" ~ ".svg" %}
-    </label>
-
-    <!-- Header title -->
-    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
-      <div class="md-header__ellipsis">
-        <div class="md-header__topic">
-          <span class="md-ellipsis">
-            {{ config.site_name }}
-          </span>
-        </div>
-        <div class="md-header__topic" data-md-component="header-topic">
-          <span class="md-ellipsis">
-            {% if page.meta and page.meta.title %}
-              {{ page.meta.title }}
-            {% else %}
-              {{ page.title }}
-            {% endif %}
-          </span>
-        </div>
-      </div>
-    </div>
-
-    <!-- Color palette -->
-    {% if config.theme.palette %}
-      {% if not config.theme.palette is mapping %}
-        <form class="md-header__option" data-md-component="palette">
-          {% for option in config.theme.palette %}
-            {% set scheme  = option.scheme  | d("default", true) %}
-            {% set primary = option.primary | d("indigo", true) %}
-            {% set accent  = option.accent  | d("indigo", true) %}
-            <input
-              class="md-option"
-              data-md-color-media="{{ option.media }}"
-              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
-              data-md-color-primary="{{ primary | replace(' ', '-') }}"
-              data-md-color-accent="{{ accent | replace(' ', '-') }}"
-              {% if option.toggle %}
-                aria-label="{{ option.toggle.name }}"
-              {% else  %}
-                aria-hidden="true"
-              {% endif %}
-              type="radio"
-              name="__palette"
-              id="__palette_{{ loop.index }}"
-            />
-            {% if option.toggle %}
-              <label
-                class="md-header__button md-icon"
-                title="{{ option.toggle.name }}"
-                for="__palette_{{ loop.index0 or loop.length }}"
-                hidden
-              >
-                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
-              </label>
-            {% endif %}
-          {% endfor %}
-        </form>
-      {% endif %}
-    {% endif %}
-
-    <!-- Site language selector -->
-    {% if config.extra.alternate %}
-      <div class="md-header__option">
-        <div class="md-select">
-          {% set icon = config.theme.icon.alternate or "material/translate" %}
-          <button
-            class="md-header__button md-icon"
-            aria-label="{{ lang.t('select.language') }}"
-          >
-            {% include ".icons/" ~ icon ~ ".svg" %}
-          </button>
-          <div class="md-select__inner">
-            <ul class="md-select__list">
-              {% for alt in config.extra.alternate %}
-                <li class="md-select__item">
-                  <a
-                    href="{{ alt.link | url }}"
-                    hreflang="{{ alt.lang }}"
-                    class="md-select__link"
-                  >
-                    {{ alt.name }}
-                  </a>
-                </li>
-              {% endfor %}
-            </ul>
-          </div>
-        </div>
-      </div>
-    {% endif %}
-
-    <!-- Button to open search modal -->
-    {% if "material/search" in config.plugins %}
-      <label class="md-header__button md-icon" for="__search">
-        {% include ".icons/material/magnify.svg" %}
-      </label>
-
-      <!-- Search interface -->
-      {% include "partials/search.html" %}
-    {% endif %}
-
-    <div style="margin-left: 10px; margin-right: 5px;">
-        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
-            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
-        </a>
-    </div>
-    <div style="margin-left: 5px; margin-right: 5px;">
-        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
-            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
-        </a>
-    </div>
-
-    <!-- Repository information -->
-    {% if config.repo_url %}
-      <div class="md-header__source" style="margin-left: -5px !important;">
-        {% include "partials/source.html" %}
-      </div>
-    {% endif %}
-  </nav>
-
-  <!-- Navigation tabs (sticky) -->
-  {% if "navigation.tabs.sticky" in features %}
-    {% if "navigation.tabs" in features %}
-      {% include "partials/tabs.html" %}
-    {% endif %}
-  {% endif %}
-</header>

docs/package-lock.json

@@ -19,7 +19,7 @@
     },
     "../node": {
       "name": "vectordb",
-      "version": "0.4.6",
+      "version": "0.21.2-beta.0",
       "cpu": [
         "x64",
         "arm64"
@@ -31,9 +31,7 @@
         "win32"
       ],
       "dependencies": {
-        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -46,6 +44,7 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -62,15 +61,19 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "*",
+        "typescript": "^5.1.0",
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.6",
-        "@lancedb/vectordb-darwin-x64": "0.4.6",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
+        "@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
       }
     },
     "../node/node_modules/apache-arrow": {

docs/requirements.txt

@@ -1,6 +1,9 @@
 mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
-mkdocstrings[python]==0.20.0
+mkdocs-autorefs<=1.0
+mkdocstrings[python]==0.25.2
+griffe
 mkdocs-render-swagger-plugin
 pydantic
+mkdocs-redirects

docs/src/ann_indexes.md

@@ -1,281 +0,0 @@
-# Approximate Nearest Neighbor (ANN) Indexes
-
-An ANN or a vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity via the chosen distance metric.
-By constructing a vector index, the search space is effectively narrowed down, avoiding the need
-for brute-force scanning of the entire vector space.
-A vector index is faster but less accurate than exhaustive search (kNN or flat search).
-LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
-
-## Disk-based Index
-
-Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
-the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
-See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
-
-## Creating an IVF_PQ Index
-
-Lance supports `IVF_PQ` index type by default.
-
-=== "Python"
-
-    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
-
-    ```python
-    import lancedb
-    import numpy as np
-    uri = "data/sample-lancedb"
-    db = lancedb.connect(uri)
-
-    # Create 10,000 sample vectors
-    data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
-
-    # Add the vectors to a table
-    tbl = db.create_table("my_vectors", data=data)
-
-    # Create and train the index - you need to have enough data in the table for an effective training step
-    tbl.create_index(num_partitions=256, num_sub_vectors=96)
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
-
-        ```typescript
-        --8<--- "nodejs/examples/ann_indexes.ts:import"
-
-        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
-        ```
-
-    === "vectordb (deprecated)"
-
-        Creating indexes is done via the [lancedb.Table.createIndex](../javascript/interfaces/Table.md/#createIndex) method.
-
-        ```typescript
-        --8<--- "docs/src/ann_indexes.ts:import"
-
-        --8<-- "docs/src/ann_indexes.ts:ingest"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
-    ```
-
-    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
-
-The following IVF_PQ paramters can be specified:
-
-- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
-  We also support "cosine" and "dot" distance as well.
-- **num_partitions**: The number of partitions in the index. The default is the square root
-  of the number of rows.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
-
-- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
-  a single PQ code. The default is the dimension of the vector divided by 16.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
-
-<figure markdown>
-  ![IVF PQ](./assets/ivf_pq.png)
-  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
-</figure>
-
-### Use GPU to build vector index
-
-Lance Python SDK has experimental GPU support for creating IVF index.
-Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
-
-You can specify the GPU device to train IVF partitions via
-
-- **accelerator**: Specify to `cuda` or `mps` (on Apple Silicon) to enable GPU training.
-
-=== "Linux"
-
-    <!-- skip-test -->
-    ``` { .python .copy }
-    # Create index using CUDA on Nvidia GPUs.
-    tbl.create_index(
-        num_partitions=256,
-        num_sub_vectors=96,
-        accelerator="cuda"
-    )
-    ```
-
-=== "MacOS"
-
-    <!-- skip-test -->
-    ```python
-    # Create index using MPS on Apple Silicon.
-    tbl.create_index(
-        num_partitions=256,
-        num_sub_vectors=96,
-        accelerator="mps"
-    )
-    ```
-
-Troubleshooting:
-
-If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
-PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
-
-## Querying an ANN Index
-
-Querying vector indexes is done via the [search](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.search) function.
-
-There are a couple of parameters that can be used to fine-tune the search:
-
-- **limit** (default: 10): The amount of results that will be returned
-- **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
-  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
-  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
-- **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
-  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
-  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
-
-=== "Python"
-
-    ```python
-    tbl.search(np.random.random((1536))) \
-        .limit(2) \
-        .nprobes(20) \
-        .refine_factor(10) \
-        .to_pandas()
-    ```
-
-    ```text
-                                              vector       item       _distance
-    0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-    1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search1"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/ann_indexes.ts:search1"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
-    ```
-
-    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
-
-The search will return the data requested in addition to the distance of each item.
-
-### Filtering (where clause)
-
-You can further filter the elements returned by a search using a where clause.
-
-=== "Python"
-
-    ```python
-    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search2"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```javascript
-        --8<-- "docs/src/ann_indexes.ts:search2"
-        ```
-
-### Projections (select clause)
-
-You can select the columns returned by the query using a select clause.
-
-=== "Python"
-
-    ```python
-    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
-    ```
-
-
-    ```text
-                                                vector _distance
-    0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-    1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-    ...
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search3"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/ann_indexes.ts:search3"
-        ```
-
-## FAQ
-
-### Why do I need to manually create an index?
-
-Currently, LanceDB does _not_ automatically create the ANN index.
-LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
-datasets of the order of ~100K vectors don't require index creation. If you can live with up to
-100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
-
-### When is it necessary to create an ANN vector index?
-
-`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
-In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
-We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
-vector indices are usually not necessary.
-
-For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
-
-### How big is my index, and how many memory will it take?
-
-In LanceDB, all vector indices are **disk-based**, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
-
-For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
-Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
-
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index?
-
-`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
-Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
-On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
-
-`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
-PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
-less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
-more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/ann_indexes.ts

@@ -1,53 +0,0 @@
-// --8<-- [start:import]
-import * as vectordb from "vectordb";
-// --8<-- [end:import]
-
-(async () => {
-  // --8<-- [start:ingest]
-  const db = await vectordb.connect("data/sample-lancedb");
-
-  let data = [];
-  for (let i = 0; i < 10_000; i++) {
-    data.push({
-      vector: Array(1536).fill(i),
-      id: `${i}`,
-      content: "",
-      longId: `${i}`,
-    });
-  }
-  const table = await db.createTable("my_vectors", data);
-  await table.createIndex({
-    type: "ivf_pq",
-    column: "vector",
-    num_partitions: 16,
-    num_sub_vectors: 48,
-  });
-  // --8<-- [end:ingest]
-
-  // --8<-- [start:search1]
-  const results_1 = await table
-    .search(Array(1536).fill(1.2))
-    .limit(2)
-    .nprobes(20)
-    .refineFactor(10)
-    .execute();
-  // --8<-- [end:search1]
-
-  // --8<-- [start:search2]
-  const results_2 = await table
-    .search(Array(1536).fill(1.2))
-    .where("id != '1141'")
-    .limit(2)
-    .execute();
-  // --8<-- [end:search2]
-
-  // --8<-- [start:search3]
-  const results_3 = await table
-    .search(Array(1536).fill(1.2))
-    .select(["id"])
-    .limit(2)
-    .execute();
-  // --8<-- [end:search3]
-
-  console.log("Ann indexes: done");
-})();

docs/src/api_reference.md

@@ -1,8 +0,0 @@
-# API Reference
-
-The API reference for the LanceDB client SDKs are available at the following locations:
-
-- [Python](python/python.md)
-- [JavaScript (legacy vectordb package)](javascript/modules.md)
-- [JavaScript (newer @lancedb/lancedb package)](js/globals.md)
-- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/assets/open_hf_space.svg

@@ -0,0 +1,22 @@
+<svg width="147" height="20" viewBox="0 0 147 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="145.482" height="19" rx="9.5" fill="white" stroke="#EFEFEF"/>
+<path d="M14.1863 10.9251V12.7593H16.0205V10.9251H14.1863Z" fill="#FF3270"/>
+<path d="M17.8707 10.9251V12.7593H19.7049V10.9251H17.8707Z" fill="#861FFF"/>
+<path d="M14.1863 7.24078V9.07496H16.0205V7.24078H14.1863Z" fill="#097EFF"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M12.903 6.77179C12.903 6.32194 13.2676 5.95728 13.7175 5.95728C14.1703 5.95728 15.2556 5.95728 16.1094 5.95728C16.7538 5.95728 17.2758 6.47963 17.2758 7.12398V9.6698H19.8217C20.4661 9.6698 20.9884 10.1922 20.9884 10.8365C20.9884 11.6337 20.9884 12.4309 20.9884 13.2282C20.9884 13.678 20.6237 14.0427 20.1738 14.0427H17.3039H16.5874H13.7175C13.2676 14.0427 12.903 13.678 12.903 13.2282V9.71653V9.64174V6.77179ZM14.1863 7.24066V9.07485H16.0205V7.24066H14.1863ZM14.1863 12.7593V10.9251H16.0205V12.7593H14.1863ZM17.8708 12.7593V10.9251H19.705V12.7593H17.8708Z" fill="black"/>
+<path d="M18.614 8.35468L20.7796 6.18905M20.7796 6.18905V7.66073M20.7796 6.18905L19.2724 6.18905" stroke="black" stroke-width="0.686298" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M31.6082 13.9838C30.8546 13.9838 30.1895 13.802 29.6132 13.4385C29.0368 13.066 28.5846 12.5429 28.2565 11.869C27.9373 11.1862 27.7777 10.3749 27.7777 9.43501C27.7777 8.49511 27.9373 7.69265 28.2565 7.02762C28.5846 6.3626 29.0368 5.85275 29.6132 5.49807C30.1895 5.14339 30.8546 4.96605 31.6082 4.96605C32.3708 4.96605 33.0403 5.14339 33.6166 5.49807C34.193 5.85275 34.6408 6.3626 34.96 7.02762C35.2881 7.69265 35.4521 8.49511 35.4521 9.43501C35.4521 10.3749 35.2881 11.1862 34.96 11.869C34.6408 12.5429 34.193 13.066 33.6166 13.4385C33.0403 13.802 32.3708 13.9838 31.6082 13.9838ZM31.6082 12.6404C32.291 12.6404 32.8363 12.3523 33.2442 11.7759C33.6521 11.1907 33.856 10.4104 33.856 9.43501C33.856 8.45964 33.6521 7.69708 33.2442 7.14733C32.8363 6.58871 32.291 6.3094 31.6082 6.3094C30.9255 6.3094 30.3802 6.58871 29.9723 7.14733C29.5644 7.69708 29.3605 8.45964 29.3605 9.43501C29.3605 10.4104 29.5644 11.1907 29.9723 11.7759C30.3802 12.3523 30.9255 12.6404 31.6082 12.6404Z" fill="#2C3236"/>
+<path d="M37.0592 16.4045V7.29363H38.3227L38.4291 7.98526H38.4823C38.7572 7.75472 39.0631 7.55521 39.4 7.38674C39.7459 7.21826 40.0961 7.13403 40.4508 7.13403C41.2665 7.13403 41.8961 7.43551 42.3395 8.03846C42.7917 8.64142 43.0178 9.44831 43.0178 10.4591C43.0178 11.204 42.8848 11.8424 42.6188 12.3744C42.3528 12.8976 42.0069 13.2966 41.5813 13.5715C41.1646 13.8463 40.7124 13.9838 40.2247 13.9838C39.9409 13.9838 39.6572 13.9217 39.3734 13.7976C39.0897 13.6646 38.8148 13.4872 38.5488 13.2656L38.5887 14.3562V16.4045H37.0592ZM39.9055 12.7202C40.3399 12.7202 40.7035 12.5296 40.9961 12.1483C41.2887 11.767 41.435 11.2084 41.435 10.4724C41.435 9.81629 41.3242 9.30644 41.1025 8.94289C40.8808 8.57935 40.5217 8.39757 40.0252 8.39757C39.5641 8.39757 39.0853 8.64142 38.5887 9.1291V12.1749C38.8281 12.37 39.0587 12.5119 39.2803 12.6005C39.502 12.6803 39.7104 12.7202 39.9055 12.7202Z" fill="#2C3236"/>
+<path d="M47.3598 13.9838C46.7568 13.9838 46.2115 13.8508 45.7238 13.5848C45.2361 13.3099 44.8504 12.9197 44.5667 12.4143C44.2829 11.9 44.141 11.2838 44.141 10.5656C44.141 9.85619 44.2829 9.24437 44.5667 8.73009C44.8593 8.2158 45.2361 7.82122 45.6972 7.54634C46.1583 7.27147 46.6415 7.13403 47.147 7.13403C47.741 7.13403 48.2376 7.26703 48.6366 7.53304C49.0356 7.79018 49.3371 8.15373 49.541 8.62368C49.745 9.08476 49.847 9.62122 49.847 10.233C49.847 10.5523 49.8248 10.8005 49.7805 10.9779H45.6307C45.7016 11.5542 45.91 12.002 46.2558 12.3212C46.6016 12.6404 47.0361 12.8 47.5593 12.8C47.843 12.8 48.1046 12.7601 48.344 12.6803C48.5923 12.5917 48.8361 12.472 49.0755 12.3212L49.5942 13.2789C49.2839 13.4828 48.9381 13.6513 48.5568 13.7843C48.1755 13.9173 47.7765 13.9838 47.3598 13.9838ZM45.6174 9.94043H48.5169C48.5169 9.43501 48.4061 9.04043 48.1844 8.75669C47.9627 8.46408 47.6302 8.31777 47.1869 8.31777C46.8056 8.31777 46.4642 8.45964 46.1627 8.74339C45.8701 9.01826 45.6883 9.41728 45.6174 9.94043Z" fill="#2C3236"/>
+<path d="M51.3078 13.8242V7.29363H52.5714L52.6778 8.17147H52.731C53.0236 7.88772 53.3428 7.64388 53.6886 7.43994C54.0344 7.236 54.429 7.13403 54.8724 7.13403C55.5728 7.13403 56.0827 7.36014 56.4019 7.81235C56.7211 8.26457 56.8807 8.90299 56.8807 9.72762V13.8242H55.3512V9.92713C55.3512 9.38624 55.2714 9.00496 55.1118 8.78329C54.9522 8.56161 54.6906 8.45078 54.327 8.45078C54.0433 8.45078 53.7906 8.52171 53.5689 8.66358C53.3561 8.79659 53.1123 8.99609 52.8374 9.2621V13.8242H51.3078Z" fill="#2C3236"/>
+<path d="M61.4131 13.8242V7.29363H62.9426V13.8242H61.4131ZM62.1845 6.14979C61.9096 6.14979 61.6879 6.06999 61.5195 5.91038C61.351 5.75078 61.2668 5.53797 61.2668 5.27196C61.2668 5.01482 61.351 4.80644 61.5195 4.64684C61.6879 4.48723 61.9096 4.40743 62.1845 4.40743C62.4594 4.40743 62.6811 4.48723 62.8495 4.64684C63.018 4.80644 63.1022 5.01482 63.1022 5.27196C63.1022 5.53797 63.018 5.75078 62.8495 5.91038C62.6811 6.06999 62.4594 6.14979 62.1845 6.14979Z" fill="#2C3236"/>
+<path d="M64.8941 13.8242V7.29363H66.1576L66.264 8.17147H66.3172C66.6098 7.88772 66.929 7.64388 67.2748 7.43994C67.6207 7.236 68.0152 7.13403 68.4586 7.13403C69.1591 7.13403 69.6689 7.36014 69.9881 7.81235C70.3074 8.26457 70.467 8.90299 70.467 9.72762V13.8242H68.9374V9.92713C68.9374 9.38624 68.8576 9.00496 68.698 8.78329C68.5384 8.56161 68.2768 8.45078 67.9133 8.45078C67.6295 8.45078 67.3768 8.52171 67.1551 8.66358C66.9423 8.79659 66.6985 8.99609 66.4236 9.2621V13.8242H64.8941Z" fill="#2C3236"/>
+<path d="M75.1323 13.8242V5.12565H76.6752V8.62368H80.1998V5.12565H81.7427V13.8242H80.1998V9.96703H76.6752V13.8242H75.1323Z" fill="#2C3236"/>
+<path d="M83.9517 13.8242V5.12565H89.2054V6.4291H85.4945V8.88969H88.6601V10.1931H85.4945V13.8242H83.9517Z" fill="#2C3236"/>
+<path d="M95.9349 13.9838C95.3497 13.9838 94.7822 13.8729 94.2324 13.6513C93.6915 13.4296 93.2127 13.1148 92.796 12.7069L93.7004 11.6562C94.0108 11.9488 94.3654 12.1882 94.7645 12.3744C95.1635 12.5518 95.5625 12.6404 95.9615 12.6404C96.458 12.6404 96.8349 12.5385 97.092 12.3345C97.3492 12.1306 97.4778 11.8601 97.4778 11.5232C97.4778 11.1596 97.3492 10.8981 97.092 10.7385C96.8438 10.5789 96.5245 10.4148 96.1344 10.2463L94.9374 9.72762C94.6536 9.60348 94.3743 9.44388 94.0994 9.2488C93.8334 9.05373 93.6117 8.80546 93.4344 8.50398C93.2659 8.2025 93.1817 7.83895 93.1817 7.41334C93.1817 6.95225 93.3058 6.53994 93.5541 6.17639C93.8113 5.80398 94.1571 5.51137 94.5915 5.29856C95.0349 5.07689 95.5403 4.96605 96.1078 4.96605C96.6132 4.96605 97.1009 5.06802 97.5709 5.27196C98.0408 5.46703 98.4442 5.73304 98.7812 6.06999L97.9965 7.05423C97.7216 6.82368 97.429 6.64191 97.1186 6.5089C96.8172 6.3759 96.4802 6.3094 96.1078 6.3094C95.6999 6.3094 95.3674 6.4025 95.1103 6.58871C94.862 6.76605 94.7379 7.01432 94.7379 7.33353C94.7379 7.55521 94.7999 7.74142 94.9241 7.89215C95.0571 8.03403 95.23 8.15816 95.4428 8.26457C95.6556 8.36211 95.8817 8.45964 96.1211 8.55718L97.3048 9.0493C97.8191 9.27097 98.2403 9.56358 98.5684 9.92713C98.8965 10.2818 99.0605 10.7739 99.0605 11.4035C99.0605 11.8734 98.9364 12.3035 98.6881 12.6936C98.4398 13.0838 98.0807 13.3986 97.6108 13.638C97.1497 13.8685 96.591 13.9838 95.9349 13.9838Z" fill="#2C3236"/>
+<path d="M100.509 16.4045V7.29363H101.773L101.879 7.98526H101.932C102.207 7.75472 102.513 7.55521 102.85 7.38674C103.196 7.21826 103.546 7.13403 103.901 7.13403C104.717 7.13403 105.346 7.43551 105.79 8.03846C106.242 8.64142 106.468 9.44831 106.468 10.4591C106.468 11.204 106.335 11.8424 106.069 12.3744C105.803 12.8976 105.457 13.2966 105.031 13.5715C104.615 13.8463 104.162 13.9838 103.675 13.9838C103.391 13.9838 103.107 13.9217 102.824 13.7976C102.54 13.6646 102.265 13.4872 101.999 13.2656L102.039 14.3562V16.4045H100.509ZM103.356 12.7202C103.79 12.7202 104.154 12.5296 104.446 12.1483C104.739 11.767 104.885 11.2084 104.885 10.4724C104.885 9.81629 104.774 9.30644 104.553 8.94289C104.331 8.57935 103.972 8.39757 103.475 8.39757C103.014 8.39757 102.535 8.64142 102.039 9.1291V12.1749C102.278 12.37 102.509 12.5119 102.73 12.6005C102.952 12.6803 103.16 12.7202 103.356 12.7202Z" fill="#2C3236"/>
+<path d="M109.444 13.9838C108.876 13.9838 108.411 13.8064 108.047 13.4518C107.692 13.0971 107.515 12.636 107.515 12.0685C107.515 11.368 107.821 10.8271 108.433 10.4458C109.045 10.0557 110.02 9.78969 111.359 9.64782C111.35 9.30201 111.257 9.00496 111.08 8.75669C110.911 8.49954 110.605 8.37097 110.162 8.37097C109.843 8.37097 109.528 8.43304 109.218 8.55718C108.916 8.68132 108.619 8.83206 108.326 9.0094L107.768 7.98526C108.131 7.75472 108.539 7.55521 108.991 7.38674C109.452 7.21826 109.94 7.13403 110.454 7.13403C111.27 7.13403 111.878 7.37787 112.277 7.86555C112.685 8.34437 112.888 9.04043 112.888 9.95373V13.8242H111.625L111.518 13.1059H111.465C111.173 13.3542 110.858 13.5626 110.521 13.7311C110.193 13.8995 109.834 13.9838 109.444 13.9838ZM109.936 12.7867C110.202 12.7867 110.441 12.7247 110.654 12.6005C110.876 12.4675 111.111 12.2902 111.359 12.0685V10.6055C110.472 10.7207 109.856 10.8936 109.51 11.1242C109.164 11.3458 108.991 11.6207 108.991 11.9488C108.991 12.2414 109.08 12.4542 109.257 12.5872C109.435 12.7202 109.661 12.7867 109.936 12.7867Z" fill="#2C3236"/>
+<path d="M117.446 13.9838C116.851 13.9838 116.315 13.8508 115.836 13.5848C115.366 13.3099 114.989 12.9197 114.706 12.4143C114.431 11.9 114.293 11.2838 114.293 10.5656C114.293 9.83846 114.444 9.2222 114.746 8.71679C115.047 8.2025 115.446 7.81235 115.943 7.54634C116.448 7.27147 116.989 7.13403 117.565 7.13403C117.982 7.13403 118.346 7.20496 118.656 7.34684C118.966 7.48871 119.241 7.66161 119.48 7.86555L118.736 8.86309C118.567 8.71235 118.394 8.59708 118.217 8.51728C118.04 8.42861 117.849 8.38427 117.645 8.38427C117.122 8.38427 116.692 8.58378 116.355 8.98279C116.027 9.38181 115.863 9.9094 115.863 10.5656C115.863 11.2128 116.022 11.736 116.342 12.135C116.67 12.534 117.091 12.7335 117.605 12.7335C117.862 12.7335 118.102 12.6803 118.323 12.5739C118.554 12.4587 118.762 12.3256 118.948 12.1749L119.574 13.1857C119.272 13.4518 118.935 13.6513 118.563 13.7843C118.19 13.9173 117.818 13.9838 117.446 13.9838Z" fill="#2C3236"/>
+<path d="M123.331 13.9838C122.728 13.9838 122.183 13.8508 121.695 13.5848C121.207 13.3099 120.822 12.9197 120.538 12.4143C120.254 11.9 120.112 11.2838 120.112 10.5656C120.112 9.85619 120.254 9.24437 120.538 8.73009C120.83 8.2158 121.207 7.82122 121.668 7.54634C122.13 7.27147 122.613 7.13403 123.118 7.13403C123.712 7.13403 124.209 7.26703 124.608 7.53304C125.007 7.79018 125.308 8.15373 125.512 8.62368C125.716 9.08476 125.818 9.62122 125.818 10.233C125.818 10.5523 125.796 10.8005 125.752 10.9779H121.602C121.673 11.5542 121.881 12.002 122.227 12.3212C122.573 12.6404 123.007 12.8 123.53 12.8C123.814 12.8 124.076 12.7601 124.315 12.6803C124.563 12.5917 124.807 12.472 125.047 12.3212L125.565 13.2789C125.255 13.4828 124.909 13.6513 124.528 13.7843C124.147 13.9173 123.748 13.9838 123.331 13.9838ZM121.589 9.94043H124.488C124.488 9.43501 124.377 9.04043 124.156 8.75669C123.934 8.46408 123.601 8.31777 123.158 8.31777C122.777 8.31777 122.435 8.45964 122.134 8.74339C121.841 9.01826 121.66 9.41728 121.589 9.94043Z" fill="#2C3236"/>
+<path d="M129.101 13.9838C128.658 13.9838 128.215 13.8995 127.771 13.7311C127.328 13.5537 126.947 13.3365 126.627 13.0793L127.346 12.0951C127.638 12.3168 127.931 12.4941 128.223 12.6271C128.516 12.7601 128.826 12.8266 129.154 12.8266C129.509 12.8266 129.771 12.7513 129.939 12.6005C130.108 12.4498 130.192 12.2636 130.192 12.0419C130.192 11.8557 130.121 11.705 129.979 11.5897C129.846 11.4656 129.673 11.3591 129.46 11.2705C129.248 11.1729 129.026 11.0798 128.795 10.9912C128.512 10.8848 128.228 10.7562 127.944 10.6055C127.669 10.4458 127.443 10.2463 127.266 10.0069C127.088 9.75866 127 9.45274 127 9.0892C127 8.51284 127.213 8.04289 127.638 7.67935C128.064 7.3158 128.64 7.13403 129.367 7.13403C129.828 7.13403 130.241 7.21383 130.604 7.37344C130.968 7.53304 131.282 7.71482 131.548 7.91876L130.844 8.84979C130.613 8.68132 130.378 8.54831 130.139 8.45078C129.908 8.34437 129.664 8.29117 129.407 8.29117C129.079 8.29117 128.835 8.36211 128.676 8.50398C128.516 8.63698 128.436 8.80545 128.436 9.0094C128.436 9.26654 128.569 9.46161 128.835 9.59462C129.101 9.72762 129.412 9.85619 129.766 9.98033C130.068 10.0867 130.36 10.2197 130.644 10.3793C130.928 10.5301 131.163 10.7296 131.349 10.9779C131.544 11.2261 131.642 11.5542 131.642 11.9621C131.642 12.5207 131.424 12.9995 130.99 13.3986C130.555 13.7887 129.926 13.9838 129.101 13.9838Z" fill="#2C3236"/>
+</svg>

docs/src/basic.md

@@ -1,584 +0,0 @@
-# Quick start
-
-!!! info "LanceDB can be run in a number of ways:"
-
-    * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Directly from a client application like a Jupyter notebook for analytical workloads
-    * Deployed as a remote serverless database
-
-![](assets/lancedb_embedded_explanation.png)
-
-## Installation
-
-=== "Python"
-
-      ```shell
-      pip install lancedb
-      ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```shell
-        npm install @lancedb/lancedb
-        ```
-        !!! note "Bundling `@lancedb/lancedb` apps with Webpack"
-
-            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-            ```javascript
-            /** @type {import('next').NextConfig} */
-            module.exports = ({
-            webpack(config) {
-                config.externals.push({ '@lancedb/lancedb': '@lancedb/lancedb' })
-                return config;
-            }
-            })
-            ```
-
-        !!! note "Yarn users"
-
-            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
-
-            ```shell
-            yarn add apache-arrow
-            ```
-
-    === "vectordb (deprecated)"
-
-        ```shell
-        npm install vectordb
-        ```
-        !!! note "Bundling `vectordb` apps with Webpack"
-
-            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-            ```javascript
-            /** @type {import('next').NextConfig} */
-            module.exports = ({
-            webpack(config) {
-                config.externals.push({ vectordb: 'vectordb' })
-                return config;
-            }
-            })
-            ```
-
-        !!! note "Yarn users"
-
-            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
-
-            ```shell
-            yarn add apache-arrow
-            ```
-
-=== "Rust"
-
-    ```shell
-    cargo add lancedb
-    ```
-
-    !!! info "To use the lancedb create, you first need to install protobuf."
-
-    === "macOS"
-
-        ```shell
-        brew install protobuf
-        ```
-
-    === "Ubuntu/Debian"
-
-        ```shell
-        sudo apt install -y protobuf-compiler libssl-dev
-        ```
-
-    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
-
-### Preview releases
-
-Stable releases are created about every 2 weeks. For the latest features and bug
-fixes, you can install the preview release. These releases receive the same
-level of testing as stable releases, but are not guaranteed to be available for
-more than 6 months after they are released. Once your application is stable, we
-recommend switching to stable releases.
-
-=== "Python"
-
-      ```shell
-      pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
-      ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```shell
-        npm install @lancedb/lancedb@preview
-        ```
-    === "vectordb (deprecated)"
-
-        ```shell
-        npm install vectordb@preview
-        ```
-
-=== "Rust"
-
-    We don't push preview releases to crates.io, but you can referent the tag
-    in GitHub within your Cargo dependencies:
-
-    ```toml
-    [dependencies]
-    lancedb = { git = "https://github.com/lancedb/lancedb.git", tag = "vX.Y.Z-beta.N" }
-    ```
-
-## Connect to a database
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:imports"
-    --8<-- "python/python/tests/docs/test_basic.py:connect"
-
-    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
-    ```
-
-    !!! note "Asynchronous Python API"
-
-        The asynchronous Python API is new and has some slight differences compared
-        to the synchronous API.  Feel free to start using the asynchronous version.
-        Once all features have migrated we will start to move the synchronous API to
-        use the same syntax as the asynchronous API.  To help with this migration we
-        have created a [migration guide](migration.md) detailing the differences.
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        import * as lancedb from "@lancedb/lancedb";
-        import * as arrow from "apache-arrow";
-
-        --8<-- "nodejs/examples/basic.ts:connect"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:open_db"
-        ```
-
-=== "Rust"
-
-    ```rust
-    #[tokio::main]
-    async fn main() -> Result<()> {
-        --8<-- "rust/lancedb/examples/simple.rs:connect"
-    }
-    ```
-
-    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
-
-LanceDB will create the directory if it doesn't exist (including parent directories).
-
-If you need a reminder of the uri, you can call `db.uri()`.
-
-## Create a table
-
-### Create a table from initial data
-
-If you have data to insert into the table at creation time, you can simultaneously create a
-table and insert the data into it. The schema of the data will be used as the schema of the
-table.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table"
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
-    ```
-
-    If the table already exists, LanceDB will raise an error by default.
-    If you want to overwrite the table, you can pass in `mode="overwrite"`
-    to the `create_table` method.
-
-    You can also pass in a pandas DataFrame directly:
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
-    ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:create_table"
-        ```
-
-        If the table already exists, LanceDB will raise an error by default.
-        If you want to overwrite the table, you can pass in `mode:"overwrite"`
-        to the `createTable` function.
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_table"
-    ```
-
-    If the table already exists, LanceDB will raise an error by default.  See
-    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
-    for details on how to overwrite (or open) existing tables instead.
-
-    !!! Providing table records in Rust
-
-        The Rust SDK currently expects data to be provided as an Arrow
-        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
-        Support for additional formats (such as serde or polars) is on the roadmap.
-
-!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
-
-!!! info "Automatic embedding generation with Embedding API"
-    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
-
-### Create an empty table
-
-Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema, so that you can add
-data to the table at a later time (as long as it conforms to the schema). This is
-similar to a `CREATE TABLE` statement in SQL.
-
-=== "Python"
-
-      ```python
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
-      ```
-
-    !!! note "You can define schema in Pydantic"
-        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_empty_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
-    ```
-
-## Open an existing table
-
-Once created, you can open a table as follows:
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:open_table"
-    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
-    ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:open_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        const tbl = await db.openTable("myTable");
-        ```
-
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
-    ```
-
-If you forget the name of your table, you can always get a listing of all table names:
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:table_names"
-    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
-    ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:table_names"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        console.log(await db.tableNames());
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:list_names"
-    ```
-
-## Add data to a table
-
-After a table has been created, you can always add more data to it as follows:
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:add_data"
-    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
-    ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:add_data"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:add"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:add"
-    ```
-
-## Search for nearest neighbors
-
-Once you've embedded the query, you can find its nearest neighbors as follows:
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
-    ```
-
-    This returns a pandas DataFrame with the results.
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:vector_search"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:search"
-        ```
-
-=== "Rust"
-
-    ```rust
-    use futures::TryStreamExt;
-
-    --8<-- "rust/lancedb/examples/simple.rs:search"
-    ```
-
-    !!! Query vectors in Rust
-        Rust does not yet support automatic execution of embedding functions.  You will need to
-        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
-        https://github.com/lancedb/lancedb/issues/994
-
-        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
-        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
-
-By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
-For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
-LanceDB allows you to create an ANN index on a table as follows:
-
-=== "Python"
-
-    ```py
-    --8<-- "python/python/tests/docs/test_basic.py:create_index"
-    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
-    ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_index"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```{.typescript .ignore}
-        --8<-- "docs/src/basic_legacy.ts:create_index"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_index"
-    ```
-
-!!! note "Why do I need to create an index manually?"
-LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
-for really fast retrievals via a disk-based index, and the second is that data and query workloads can
-be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
-to fine-tune index size, query latency and accuracy. See the section on
-[ANN indexes](ann_indexes.md) for more details.
-
-## Delete rows from a table
-
-Use the `delete()` method on tables to delete rows from a table. To choose
-which rows to delete, provide a filter that matches on the metadata columns.
-This can delete any number of rows that match the filter.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
-    ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:delete_rows"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:delete"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:delete"
-    ```
-
-The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause (`only_if()` in Rust) on a search. They can be as
-simple or complex as needed. To see what expressions are supported, see the
-[SQL filters](sql.md) section.
-
-=== "Python"
-
-      Read more: [lancedb.table.Table.delete][]
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        Read more: [lancedb.Table.delete](javascript/interfaces/Table.md#delete)
-
-    === "vectordb (deprecated)"
-
-        Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
-
-=== "Rust"
-
-      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
-
-## Drop a table
-
-Use the `drop_table()` method on the database to remove a table.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-    ```
-
-    This permanently removes the table and is not recoverable, unlike deleting rows.
-    By default, if the table does not exist an exception is raised. To suppress this,
-    you can pass in `ignore_missing=True`.
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.ts:drop_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:drop_table"
-        ```
-
-        This permanently removes the table and is not recoverable, unlike deleting rows.
-        If the table does not exist an exception is raised.
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
-    ```
-
-
-## Using the Embedding API
-You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
-    ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/embedding.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/openai.rs:imports"
-    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
-
-
-## What's next
-
-This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
-
-If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.
-
-[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/basic_legacy.ts

@@ -1,126 +0,0 @@
-// --8<-- [start:import]
-import * as lancedb from "vectordb";
-import {
-  Schema,
-  Field,
-  Float32,
-  FixedSizeList,
-  Int32,
-  Float16,
-} from "apache-arrow";
-import * as arrow from "apache-arrow";
-// --8<-- [end:import]
-import * as fs from "fs";
-import { Table as ArrowTable, Utf8 } from "apache-arrow";
-
-const example = async () => {
-  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
-  // --8<-- [start:open_db]
-  const lancedb = require("vectordb");
-  const uri = "data/sample-lancedb";
-  const db = await lancedb.connect(uri);
-  // --8<-- [end:open_db]
-
-  // --8<-- [start:create_table]
-  const tbl = await db.createTable(
-    "myTable",
-    [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ],
-    { writeMode: lancedb.WriteMode.Overwrite },
-  );
-  // --8<-- [end:create_table]
-  {
-    // --8<-- [start:create_table_with_schema]
-    const schema = new arrow.Schema([
-      new arrow.Field(
-        "vector",
-        new arrow.FixedSizeList(
-          2,
-          new arrow.Field("item", new arrow.Float32(), true),
-        ),
-      ),
-      new arrow.Field("item", new arrow.Utf8(), true),
-      new arrow.Field("price", new arrow.Float32(), true),
-    ]);
-    const data = [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ];
-    const tbl = await db.createTable({
-      name: "myTableWithSchema",
-      data,
-      schema,
-    });
-    // --8<-- [end:create_table_with_schema]
-  }
-
-  // --8<-- [start:add]
-  const newData = Array.from({ length: 500 }, (_, i) => ({
-    vector: [i, i + 1],
-    item: "fizz",
-    price: i * 0.1,
-  }));
-  await tbl.add(newData);
-  // --8<-- [end:add]
-
-  // --8<-- [start:create_index]
-  await tbl.createIndex({
-    type: "ivf_pq",
-    num_partitions: 2,
-    num_sub_vectors: 2,
-  });
-  // --8<-- [end:create_index]
-
-  // --8<-- [start:create_empty_table]
-  const schema = new arrow.Schema([
-    new arrow.Field("id", new arrow.Int32()),
-    new arrow.Field("name", new arrow.Utf8()),
-  ]);
-
-  const empty_tbl = await db.createTable({ name: "empty_table", schema });
-  // --8<-- [end:create_empty_table]
-  {
-    // --8<-- [start:create_f16_table]
-    const dim = 16;
-    const total = 10;
-    const schema = new Schema([
-      new Field("id", new Int32()),
-      new Field(
-        "vector",
-        new FixedSizeList(dim, new Field("item", new Float16(), true)),
-        false,
-      ),
-    ]);
-    const data = lancedb.makeArrowTable(
-      Array.from(Array(total), (_, i) => ({
-        id: i,
-        vector: Array.from(Array(dim), Math.random),
-      })),
-      { schema },
-    );
-    const table = await db.createTable("f16_tbl", data);
-    // --8<-- [end:create_f16_table]
-  }
-
-  // --8<-- [start:search]
-  const query = await tbl.search([100, 100]).limit(2).execute();
-  // --8<-- [end:search]
-  console.log(query);
-
-  // --8<-- [start:delete]
-  await tbl.delete('item = "fizz"');
-  // --8<-- [end:delete]
-
-  // --8<-- [start:drop_table]
-  await db.dropTable("myTable");
-  // --8<-- [end:drop_table]
-};
-
-async function main() {
-  await example();
-  console.log("Basic example: done");
-}
-
-main();

docs/src/cloud/index.md

@@ -1,17 +0,0 @@
-# About LanceDB Cloud
-
-LanceDB Cloud is a SaaS (software-as-a-service) solution that runs serverless in the cloud, clearly separating storage from compute. It's designed to be highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
-
-[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
-
-## Architecture
-
-LanceDB Cloud provides the same underlying fast vector store that powers the OSS version, but without the need to maintain your own infrastructure. Because it's serverless, you only pay for the storage you use, and you can scale compute up and down as needed depending on the size of your data and its associated index.
-
-![](../assets/lancedb_cloud.png)
-
-## Transitioning from the OSS to the Cloud version
-
-The OSS version of LanceDB is designed to be embedded in your application, and it runs in-process. This makes it incredibly simple to self-host your own AI retrieval workflows for RAG and more and build and test out your concepts on your own infrastructure. The OSS version is forever free, and you can continue to build and integrate LanceDB into your existing backend applications without any added costs.
-
-Should you decide that you need a managed deployment in production, it's possible to seamlessly transition from the OSS to the cloud version by changing the connection string to point to a remote database instead of a local one. With LanceDB Cloud, you can take your AI application from development to production without major code changes or infrastructure burden.

docs/src/cloud/rest.md

@@ -1 +0,0 @@
-!!swagger ../../openapi.yml!!

docs/src/concepts/data_management.md

@@ -1,62 +0,0 @@
-# Data management
-
-This section covers concepts related to managing your data over time in LanceDB.
-
-## A primer on Lance
-
-Because LanceDB is built on top of the [Lance](https://lancedb.github.io/lance/) data format, it helps to understand some of its core ideas. Just like Apache Arrow, Lance is a fast columnar data format, but it has the added benefit of being versionable, query and train ML models on. Lance is designed to be used with simple and complex data types, like tabular data, images, videos audio, 3D point clouds (which are deeply nested) and more.
-
-The following concepts are important to keep in mind:
-
-- Data storage is columnar and is interoperable with other columnar formats (such as Parquet) via Arrow
-- Data is divided into fragments that represent a subset of the data
-- Data is versioned, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata
-
-!!! note
-    1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.  
-    2. Second, these versions exist to keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be in the middle of querying the old version. It's important to retain older versions for as long as they might be queried.
-
-## What are fragments?
-
-Fragments are chunks of data in a Lance dataset. Each fragment includes multiple files that contain several columns in the chunk of data that it represents.
-
-## Compaction
-
-As you insert more data, your dataset will grow and you'll need to perform *compaction* to maintain query throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments together to reduce the amount of metadata that needs to be managed, and to reduce the number of files that need to be opened while scanning the dataset.
-
-### How does compaction improve performance?
-
-Compaction performs the following tasks in the background:
-
-- Removes deleted rows from fragments
-- Removes dropped columns from fragments
-- Merges small fragments into larger ones
-
-Depending on the use case and dataset, optimal compaction will have different requirements. As a rule of thumb:
-
-- It’s always better to use *batch* inserts rather than adding 1 row at a time (to avoid too small fragments). If single-row inserts are unavoidable, run compaction on a regular basis to merge them into larger fragments.
-- Keep the number of fragments under 100, which is suitable for most use cases (for *really* large datasets of >500M rows, more fragments might be needed)
-
-## Deletion
-
-Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever deleted from that fragment, it will not have a deletion file). This is important to keep in mind because it means that the data is still there, and can be recovered if needed, as long as that version still exists based on your backup policy.
-
-## Reindexing
-
-Reindexing is the process of updating the index to account for new data, keeping good performance for queries. This applies to either a full-text search (FTS) index or a vector index. For ANN search, new data will always be included in query results, but queries on tables with unindexed data will fallback to slower search methods for the new parts of the table. This is another important operation to run periodically as your data grows, as it also improves performance. This is especially important if you're appending large amounts of data to an existing dataset.
-
-!!! tip
-    When adding new data to a dataset that has an existing index (either FTS or vector), LanceDB doesn't immediately update the index until a reindex operation is complete.
-
-Both LanceDB OSS and Cloud support reindexing, but the process (at least for now) is different for each, depending on the type of index.
-
-When a reindex job is triggered in the background, the entire data is reindexed, but in the interim as new queries come in, LanceDB will combine results from the existing index with exhaustive kNN search on the new data. This is done to ensure that you're still searching on all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
-
-### Vector reindex
-
-* LanceDB Cloud supports incremental reindexing, where a background process will trigger a new index build for you automatically when new data is added to a dataset
-* LanceDB OSS requires you to manually trigger a reindex operation -- we are working on adding incremental reindexing to LanceDB OSS as well
-
-### FTS reindex
-
-FTS reindexing is supported in both LanceDB OSS and Cloud, but requires that it's manually rebuilt once you have a significant enough amount of new data added that needs to be reindexed. We [updated](https://github.com/lancedb/lancedb/pull/762) Tantivy's default heap size from 128MB to 1GB in LanceDB to make it much faster to reindex, by up to 10x from the default settings.

docs/src/concepts/index_ivfpq.md

@@ -1,84 +0,0 @@
-# Understanding LanceDB's IVF-PQ index
-
-An ANN (Approximate Nearest Neighbors) index is a data structure that represents data in a way that makes it more efficient to search and retrieve. Using an ANN index is faster, but less accurate than kNN or brute force search because, in essence, the index is a lossy representation of the data.
-
-LanceDB is fundamentally different from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy.
-
-## IVF-PQ
-
-IVF-PQ is a composite index that combines inverted file index (IVF) and product quantization (PQ). The implementation in LanceDB provides several parameters to fine-tune the index's size, query throughput, latency and recall, which are described later in this section.
-
-### Product quantization
-
-Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search.
-
-Product quantization (PQ) works by dividing a large, high-dimensional vector of size into equally sized subvectors. Each subvector is assigned a "reproduction value" that maps to the nearest centroid of points for that subvector. The reproduction values are then assigned to a codebook using unique IDs, which can be used to reconstruct the original vector.
-
-![](../assets/ivfpq_pq_desc.png)
-
-It's important to remember that quantization is a *lossy process*, i.e., the reconstructed vector is not identical to the original vector. This results in a trade-off between the size of the index and the accuracy of the search results.
-
-As an example, consider starting with 128-dimensional vector consisting of 32-bit floats. Quantizing it to an 8-bit integer vector with 4 dimensions as in the image above, we can significantly reduce memory requirements.
-
-!!! example "Effect of quantization"
-
-    Original: `128 × 32 = 4096` bits
-    Quantized: `4 × 8 = 32` bits
-
-    Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial.
-
-### Inverted file index
-
-While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
-
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
-
-![](../assets/ivfpq_ivf_desc.webp)
-
-During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query.
-
-![](../assets/ivfpq_query_vector.webp)
-
-## Putting it all together
-
-We can combine the above concepts to understand how to build and query an IVF-PQ index in LanceDB.
-
-### Construct index
-
-There are three key parameters to set when constructing an IVF-PQ index:
-
-* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
-* `num_partitions`: The number of partitions in the IVF portion of the index.
-* `num_sub_vectors`: The number of sub-vectors that will be created during Product Quantization (PQ).
-
-In Python, the index can be created as follows:
-
-```python
-# Create and train the index for a 1536-dimensional vector
-# Make sure you have enough data in the table for an effective training step
-tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
-```
-
-The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See the [FAQs](#faq) below for best practices on choosing these parameters.
-
-
-### Query the index
-
-```python
-# Search using a random 1536-dimensional embedding
-tbl.search(np.random.random((1536))) \
-    .limit(2) \
-    .nprobes(20) \
-    .refine_factor(10) \
-    .to_pandas()
-```
-
-The above query will perform a search on the table `tbl` using the given query vector, with the following parameters:
-
-* `limit`: The number of results to return
-* `nprobes`: The number of probes determines the distribution of vector space. While a higher number enhances search accuracy, it also results in slower performance. Typically, setting `nprobes` to cover 5–10% of the dataset proves effective in achieving high recall with minimal latency.
-* `refine_factor`: Refine the results by reading extra elements and re-ranking them in memory. A higher number makes the search more accurate but also slower (see the [FAQ](../faq.md#do-i-need-to-set-a-refine-factor-when-using-an-index) page for more details on this).
-* `to_pandas()`: Convert the results to a pandas DataFrame
-
-And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
-To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.

docs/src/concepts/storage.md

@@ -1,80 +0,0 @@
-# Storage
-
-LanceDB is among the only vector databases built on top of multiple modular components designed from the ground-up to be efficient on disk. This gives it the unique benefit of being flexible enough to support multiple storage backends, including local NVMe, EBS, EFS and many other third-party APIs that connect to the cloud.
-
-It is important to understand the tradeoffs between cost and latency for your specific application and use case. This section will help you understand the tradeoffs between the different storage backends.
-
-## Storage options
-
-We've prepared a simple diagram to showcase the thought process that goes into choosing a storage backend when using LanceDB OSS, Cloud or Enterprise.
-
-![](../assets/lancedb_storage_tradeoffs.png)
-
-When architecting your system, you'd typically ask yourself the following questions to decide on a storage option:
-
-1. **Latency**: How fast do I need results? What do the p50 and also p95 look like?
-2. **Scalability**: Can I scale up the amount of data and QPS easily?
-3. **Cost**: To serve my application, what’s the all-in cost of *both* storage and serving infra?
-4. **Reliability/Availability**: How does replication work? Is disaster recovery addressed?
-
-## Tradeoffs
-
-This section reviews the characteristics of each storage option in four dimensions: latency, scalability, cost and reliability.
-
-**We begin with the lowest cost option, and end with the lowest latency option.**
-
-### 1. S3 / GCS / Azure Blob Storage
-
-!!! tip "Lowest cost, highest latency"
-
-    - **Latency** ⇒ Has the highest latency. p95 latency is also substantially worse than p50. In general you get results in the order of several hundred milliseconds
-    - **Scalability** ⇒ Infinite on storage, however, QPS will be limited by S3 concurrency limits
-    - **Cost** ⇒ Lowest (order of magnitude cheaper than other options)
-    - **Reliability/Availability** ⇒ Highly available, as blob storage like S3 are critical infrastructure that form the backbone of the internet.
-
-Another important point to note is that LanceDB is designed to separate storage from compute, and the underlying Lance format stores the data in numerous immutable fragments. Due to these factors, LanceDB is a great storage option that addresses the _N + 1_ query problem. i.e., when a high query throughput is required, query processes can run in a stateless manner and be scaled up and down as needed.
-
-### 2. EFS / GCS Filestore / Azure File Storage
-
-!!! info "Moderately low cost, moderately low latency (<100ms)"
-
-    - **Latency** ⇒ Much better than object/blob storage but not as good as EBS/Local disk; < 100ms p95 achievable
-    - **Scalability** ⇒ High, but the bottleneck will be the IOPs limit, but when scaling you can provision multiple EFS volumes
-    - **Cost** ⇒ Significantly more expensive than S3 but still very cost effective compared to in-memory dbs. Inactive data in EFS is also automatically tiered to S3-level costs.
-    - **Reliability/Availability** ⇒ Highly available, as query nodes can go down without affecting EFS.  However, EFS does not provide replication / backup - this must be managed manually.
-
-A recommended best practice is to keep a copy of the data on S3 for disaster recovery scenarios. If any downtime is unacceptable, then you would need another EFS with a copy of the data. This is still much cheaper than EC2 instances holding multiple copies of the data.
-
-### 3. Third-party storage solutions
-
-Solutions like [MinIO](https://blog.min.io/lancedb-trusted-steed-against-data-complexity/), WekaFS, etc. that deliver S3 compatible API with much better performance than S3.
-
-!!! info "Moderately low cost, moderately low latency (<100ms)"
-
-    - **Latency** ⇒ Should be similar latency to EFS, better than S3 (<100ms)
-    - **Scalability** ⇒ Up to the solutions architect, who can add as many nodes to their MinIO or other third-party provider's cluster as needed
-    - **Cost** ⇒ Definitely higher than S3. The cost can be marginally higher than EFS until you get to maybe >10TB scale with high utilization
-    - **Reliability/Availability** ⇒ These are all shareable by lots of nodes, quality/cost of replication/backup depends on the vendor
-
-
-### 4. EBS / GCP Persistent Disk / Azure Managed Disk
-
-!!! info "Very low latency (<30ms), higher cost"
-
-    - **Latency** ⇒ Very good, pretty close to local disk. You’re looking at <30ms latency in most cases
-    - **Scalability** ⇒ EBS is not shareable between instances. If deployed via k8s, it can be shared between pods that live on the same instance, but beyond that you would need to shard data or make an additional copy
-    - **Cost** ⇒ Higher than EFS. There are some hidden costs to EBS as well if you’re paying for IO.
-    - **Reliability/Availability** ⇒ Not shareable between instances but can be shared between pods on the same instance. Survives instance termination. No automatic backups.
-
-Just like EFS, an EBS or persistent disk setup requires more manual work to manage data sharding, backups and capacity.
-
-### 5. Local disk (SSD/NVMe)
-
-!!! danger "Lowest latency (<10ms), highest cost"
-
-    - **Latency** ⇒ Lowest latency with modern NVMe drives, <10ms p95
-    - **Scalability** ⇒ Difficult to scale on cloud. Also need additional copies / sharding if QPS needs to be higher
-    - **Cost** ⇒ Highest cost; the main issue with keeping your application and storage tightly integrated is that it’s just not really possible to scale this up in cloud environments
-    - **Reliability/Availability** ⇒ If the instance goes down, so does your data. You have to be _very_ diligent about backing up your data
-
-As a rule of thumb, local disk should be your storage option if you require absolutely *crazy low* latency and you’re willing to do a bunch of data management work to make it happen.

docs/src/concepts/vector_search.md

@@ -1,36 +0,0 @@
-# Vector search
-
-Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search.
-
-Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a vector database like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups.
-
-![](../assets/vector-db-basics.png)
-
-## Embeddings
-
-Modern machine learning models can be trained to convert raw data into embeddings, represented as arrays (or vectors) of floating point numbers of fixed dimensionality. What makes embeddings useful in practice is that the position of an embedding in vector space captures some of the semantics of the data, depending on the type of model and how it was trained. Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar.
-
-Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model. Projecting the vectors' principal components in 2D space results in groups of vectors that represent similar concepts clustering together, as shown below.
-
-![](../assets/embedding_intro.png)
-
-## Indexes
-
-Embeddings for a given dataset are made searchable via an **index**. The index is constructed by using data structures that store the embeddings such that it's very efficient to perform scans and lookups on them. A key distinguishing feature of LanceDB is it uses a disk-based index: IVF-PQ, which is a variant of the Inverted File Index (IVF) that uses Product Quantization (PQ) to compress the embeddings.
-
-See the [IVF-PQ](./index_ivfpq.md) page for more details on how it works.
-
-## Brute force search
-
-The simplest way to perform vector search is to perform a brute force search, without an index, where the distance between the query vector and all the vectors in the database are computed, with the top-k closest vectors returned. This is equivalent to a k-nearest neighbours (kNN) search in vector space.
-
-![](../assets/knn_search.png)
-
-As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in.
-
-## Approximate nearest neighbour (ANN) search
-
-Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases.
-
-
-

docs/src/embeddings/custom_embedding_function.md

@@ -1,240 +0,0 @@
-To use your own custom embedding function, you can follow these 2 simple steps:
-
-1. Create your embedding function by implementing the `EmbeddingFunction` interface
-2. Register your embedding function in the global `EmbeddingFunctionRegistry`.
-
-Let us see how this looks like in action.
-
-![](../assets/embeddings_api.png)
-
-`EmbeddingFunction` and `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embedding function, you don't have to worry about the finer details - simply focus on setting up the model and leave the rest to LanceDB.
-
-## `TextEmbeddingFunction` interface
-
-There is another optional layer of abstraction available: `TextEmbeddingFunction`. You can use this abstraction if your model isn't multi-modal in nature and only needs to operate on text. In such cases, both the source and vector fields will have the same work for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
-
-Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
-
-
-=== "Python"
-
-    ```python
-    from lancedb.embeddings import register
-    from lancedb.util import attempt_import_or_raise
-
-    @register("sentence-transformers")
-    class SentenceTransformerEmbeddings(TextEmbeddingFunction):
-        name: str = "all-MiniLM-L6-v2"
-        # set more default instance vars like device, etc.
-
-        def __init__(self, **kwargs):
-            super().__init__(**kwargs)
-            self._ndims = None
-
-        def generate_embeddings(self, texts):
-            return self._embedding_model().encode(list(texts), ...).tolist()
-
-        def ndims(self):
-            if self._ndims is None:
-                self._ndims = len(self.generate_embeddings("foo")[0])
-            return self._ndims
-
-        @cached(cache={})
-        def _embedding_model(self):
-            return sentence_transformers.SentenceTransformer(name)
-    ```
-
-=== "TypeScript"
-
-    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:imports"
-
-    --8<--- "nodejs/examples/custom_embedding_function.ts:embedding_impl"
-    ```
-
-
-This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and default settings.
-
-Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
-
-=== "Python"
-
-    ```python
-    from lancedb.pydantic import LanceModel, Vector
-
-    registry = EmbeddingFunctionRegistry.get_instance()
-    stransformer = registry.get("sentence-transformers").create()
-
-    class TextModelSchema(LanceModel):
-        vector: Vector(stransformer.ndims) = stransformer.VectorField()
-        text: str = stransformer.SourceField()
-
-    tbl = db.create_table("table", schema=TextModelSchema)
-
-    tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
-    result = tbl.search("world").limit(5)
-    ```
-
-=== "TypeScript"
-
-    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:call_custom_function"
-    ```
-
-!!! note
-
-    You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
-
-## Multi-modal embedding function example
-You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support.
-
-=== "Python"
-
-    LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
-
-    ```python
-    @register("open-clip")
-    class OpenClipEmbeddings(EmbeddingFunction):
-        name: str = "ViT-B-32"
-        pretrained: str = "laion2b_s34b_b79k"
-        device: str = "cpu"
-        batch_size: int = 64
-        normalize: bool = True
-        _model = PrivateAttr()
-        _preprocess = PrivateAttr()
-        _tokenizer = PrivateAttr()
-
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
-            model, _, preprocess = open_clip.create_model_and_transforms(
-                self.name, pretrained=self.pretrained
-            )
-            model.to(self.device)
-            self._model, self._preprocess = model, preprocess
-            self._tokenizer = open_clip.get_tokenizer(self.name)
-            self._ndims = None
-
-        def ndims(self):
-            if self._ndims is None:
-                self._ndims = self.generate_text_embeddings("foo").shape[0]
-            return self._ndims
-
-        def compute_query_embeddings(
-            self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
-        ) -> List[np.ndarray]:
-            """
-            Compute the embeddings for a given user query
-
-            Parameters
-            ----------
-            query : Union[str, PIL.Image.Image]
-                The query to embed. A query can be either text or an image.
-            """
-            if isinstance(query, str):
-                return [self.generate_text_embeddings(query)]
-            else:
-                PIL = attempt_import_or_raise("PIL", "pillow")
-                if isinstance(query, PIL.Image.Image):
-                    return [self.generate_image_embedding(query)]
-                else:
-                    raise TypeError("OpenClip supports str or PIL Image as query")
-
-        def generate_text_embeddings(self, text: str) -> np.ndarray:
-            torch = attempt_import_or_raise("torch")
-            text = self.sanitize_input(text)
-            text = self._tokenizer(text)
-            text.to(self.device)
-            with torch.no_grad():
-                text_features = self._model.encode_text(text.to(self.device))
-                if self.normalize:
-                    text_features /= text_features.norm(dim=-1, keepdim=True)
-                return text_features.cpu().numpy().squeeze()
-
-        def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-            """
-            Sanitize the input to the embedding function.
-            """
-            if isinstance(images, (str, bytes)):
-                images = [images]
-            elif isinstance(images, pa.Array):
-                images = images.to_pylist()
-            elif isinstance(images, pa.ChunkedArray):
-                images = images.combine_chunks().to_pylist()
-            return images
-
-        def compute_source_embeddings(
-            self, images: IMAGES, *args, **kwargs
-        ) -> List[np.array]:
-            """
-            Get the embeddings for the given images
-            """
-            images = self.sanitize_input(images)
-            embeddings = []
-            for i in range(0, len(images), self.batch_size):
-                j = min(i + self.batch_size, len(images))
-                batch = images[i:j]
-                embeddings.extend(self._parallel_get(batch))
-            return embeddings
-
-        def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
-            """
-            Issue concurrent requests to retrieve the image data
-            """
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                futures = [
-                    executor.submit(self.generate_image_embedding, image)
-                    for image in images
-                ]
-                return [future.result() for future in futures]
-
-        def generate_image_embedding(
-            self, image: Union[str, bytes, "PIL.Image.Image"]
-        ) -> np.ndarray:
-            """
-            Generate the embedding for a single image
-
-            Parameters
-            ----------
-            image : Union[str, bytes, PIL.Image.Image]
-                The image to embed. If the image is a str, it is treated as a uri.
-                If the image is bytes, it is treated as the raw image bytes.
-            """
-            torch = attempt_import_or_raise("torch")
-            # TODO handle retry and errors for https
-            image = self._to_pil(image)
-            image = self._preprocess(image).unsqueeze(0)
-            with torch.no_grad():
-                return self._encode_and_normalize_image(image)
-
-        def _to_pil(self, image: Union[str, bytes]):
-            PIL = attempt_import_or_raise("PIL", "pillow")
-            if isinstance(image, bytes):
-                return PIL.Image.open(io.BytesIO(image))
-            if isinstance(image, PIL.Image.Image):
-                return image
-            elif isinstance(image, str):
-                parsed = urlparse.urlparse(image)
-                # TODO handle drive letter on windows.
-                if parsed.scheme == "file":
-                    return PIL.Image.open(parsed.path)
-                elif parsed.scheme == "":
-                    return PIL.Image.open(image if os.name == "nt" else parsed.path)
-                elif parsed.scheme.startswith("http"):
-                    return PIL.Image.open(io.BytesIO(url_retrieve(image)))
-                else:
-                    raise NotImplementedError("Only local and http(s) urls are supported")
-
-        def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
-            """
-            encode a single image tensor and optionally normalize the output
-            """
-            image_features = self._model.encode_image(image_tensor)
-            if self.normalize:
-                image_features /= image_features.norm(dim=-1, keepdim=True)
-            return image_features.cpu().numpy().squeeze()
-    ```
-
-=== "TypeScript"
-
-    Coming Soon! See this [issue](https://github.com/lancedb/lancedb/issues/1482) to track the status!

docs/src/embeddings/default_embedding_functions.md

@@ -1,800 +0,0 @@
-There are various embedding functions available out of the box with LanceDB to manage your embeddings implicitly. We're actively working on adding other popular embedding APIs and models.
-
-## Text embedding functions
-Contains the text embedding functions registered by default.
-
-* Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential backoff. 
-* Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
-
-### Sentence transformers
-Allows you to set parameters when registering a `sentence-transformers` object.
-
-!!! info
-    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
-| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
-| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
-| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository |
-
-
-??? "Check out available sentence-transformer models here!"
-    ```markdown
-    - sentence-transformers/all-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-mpnet-base-v2
-    - sentence-transformers/gtr-t5-base
-    - sentence-transformers/LaBSE
-    - sentence-transformers/all-MiniLM-L6-v2
-    - sentence-transformers/bert-base-nli-max-tokens
-    - sentence-transformers/bert-base-nli-mean-tokens
-    - sentence-transformers/bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
-    - sentence-transformers/bert-large-nli-cls-token
-    - sentence-transformers/bert-large-nli-max-tokens
-    - sentence-transformers/bert-large-nli-mean-tokens
-    - sentence-transformers/bert-large-nli-stsb-mean-tokens
-    - sentence-transformers/distilbert-base-nli-max-tokens
-    - sentence-transformers/distilbert-base-nli-mean-tokens
-    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
-    - sentence-transformers/distilroberta-base-msmarco-v1
-    - sentence-transformers/distilroberta-base-msmarco-v2
-    - sentence-transformers/nli-bert-base-cls-pooling
-    - sentence-transformers/nli-bert-base-max-pooling
-    - sentence-transformers/nli-bert-base
-    - sentence-transformers/nli-bert-large-cls-pooling
-    - sentence-transformers/nli-bert-large-max-pooling
-    - sentence-transformers/nli-bert-large
-    - sentence-transformers/nli-distilbert-base-max-pooling
-    - sentence-transformers/nli-distilbert-base
-    - sentence-transformers/nli-roberta-base
-    - sentence-transformers/nli-roberta-large
-    - sentence-transformers/roberta-base-nli-mean-tokens
-    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
-    - sentence-transformers/roberta-large-nli-mean-tokens
-    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
-    - sentence-transformers/stsb-bert-base
-    - sentence-transformers/stsb-bert-large
-    - sentence-transformers/stsb-distilbert-base
-    - sentence-transformers/stsb-roberta-base
-    - sentence-transformers/stsb-roberta-large
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
-    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
-    - sentence-transformers/bert-base-nli-cls-token
-    - sentence-transformers/all-distilroberta-v1
-    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
-    - sentence-transformers/multi-qa-distilbert-cos-v1
-    - sentence-transformers/multi-qa-distilbert-dot-v1
-    - sentence-transformers/multi-qa-mpnet-base-cos-v1
-    - sentence-transformers/multi-qa-mpnet-base-dot-v1
-    - sentence-transformers/nli-distilroberta-base-v2
-    - sentence-transformers/all-MiniLM-L6-v1
-    - sentence-transformers/all-mpnet-base-v1
-    - sentence-transformers/all-mpnet-base-v2
-    - sentence-transformers/all-roberta-large-v1
-    - sentence-transformers/allenai-specter
-    - sentence-transformers/average_word_embeddings_glove.6B.300d
-    - sentence-transformers/average_word_embeddings_glove.840B.300d
-    - sentence-transformers/average_word_embeddings_komninos
-    - sentence-transformers/average_word_embeddings_levy_dependency
-    - sentence-transformers/clip-ViT-B-32-multilingual-v1
-    - sentence-transformers/clip-ViT-B-32
-    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
-    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
-    - sentence-transformers/distilroberta-base-paraphrase-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v2
-    - sentence-transformers/distiluse-base-multilingual-cased
-    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
-    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
-    - sentence-transformers/gtr-t5-large
-    - sentence-transformers/gtr-t5-xl
-    - sentence-transformers/gtr-t5-xxl
-    - sentence-transformers/msmarco-MiniLM-L-12-v3
-    - sentence-transformers/msmarco-MiniLM-L-6-v3
-    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
-    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
-    - sentence-transformers/msmarco-bert-base-dot-v5
-    - sentence-transformers/msmarco-bert-co-condensor
-    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
-    - sentence-transformers/msmarco-distilbert-base-tas-b
-    - sentence-transformers/msmarco-distilbert-base-v2
-    - sentence-transformers/msmarco-distilbert-base-v3
-    - sentence-transformers/msmarco-distilbert-base-v4
-    - sentence-transformers/msmarco-distilbert-cos-v5
-    - sentence-transformers/msmarco-distilbert-dot-v5
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
-    - sentence-transformers/msmarco-distilroberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-ance-firstp
-    - sentence-transformers/msmarco-roberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-v3
-    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-    - sentence-transformers/nli-mpnet-base-v2
-    - sentence-transformers/nli-roberta-base-v2
-    - sentence-transformers/nq-distilbert-base-v1
-    - sentence-transformers/paraphrase-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-MiniLM-L3-v2
-    - sentence-transformers/paraphrase-MiniLM-L6-v2
-    - sentence-transformers/paraphrase-TinyBERT-L6-v2
-    - sentence-transformers/paraphrase-albert-base-v2
-    - sentence-transformers/paraphrase-albert-small-v2
-    - sentence-transformers/paraphrase-distilroberta-base-v1
-    - sentence-transformers/paraphrase-distilroberta-base-v2
-    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
-    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
-    - sentence-transformers/quora-distilbert-base
-    - sentence-transformers/quora-distilbert-multilingual
-    - sentence-transformers/sentence-t5-base
-    - sentence-transformers/sentence-t5-large
-    - sentence-transformers/sentence-t5-xxl
-    - sentence-transformers/sentence-t5-xl
-    - sentence-transformers/stsb-distilroberta-base-v2
-    - sentence-transformers/stsb-mpnet-base-v2
-    - sentence-transformers/stsb-roberta-base-v2
-    - sentence-transformers/stsb-xlm-r-multilingual
-    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
-    - sentence-transformers/clip-ViT-L-14
-    - sentence-transformers/clip-ViT-B-16
-    - sentence-transformers/use-cmlm-multilingual
-    - sentence-transformers/all-MiniLM-L12-v1
-    ```
-
-!!! info
-    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
-    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
-
-!!! note "BAAI Embeddings example"
-    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-    )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
-
-
-### Huggingface embedding models
-We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
-
-Example usage - 
-```python
-import lancedb
-import pandas as pd
-
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-model = get_registry().get("huggingface").create(name='facebook/bart-base')
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
-table = db.create_table("greets", schema=Words)
-table.add(df)
-query = "old greeting"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
-
-### Ollama embeddings
-Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
-
-- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
-- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
-
-| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
-|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
-| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
-| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
-| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
-| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
-| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("ollama").create(name="nomic-embed-text")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
-
-### OpenAI embeddings
-LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
-| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
-
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
-### Instructor Embeddings
-[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
-
-If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
-
-!!! info
-    Represent the `domain` `text_type` for `task_objective`:
-
-    * `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
-    * `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
-    * `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
-
-More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
-
-| Argument | Type | Default | Description |
-|---|---|---|---|
-| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
-| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
-| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
-| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
-| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
-| `quantize` | `bool` | `False` | Whether to quantize the model |
-| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
-| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
-
-
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
-
-instructor = get_registry().get("instructor").create(
-                            source_instruction="represent the docuement for retreival",
-                            query_instruction="represent the document for retreiving the most similar documents"
-                            )
-
-class Schema(LanceModel):
-    vector: Vector(instructor.ndims()) = instructor.VectorField()
-    text: str = instructor.SourceField()
-
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-
-texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
-        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
-        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
-
-tbl.add(texts)
-```
-
-### Gemini Embeddings
-With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
-The Gemini Embedding Model API supports various task types:
-
-| Task Type               | Description                                                                                                                                                |
-|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
-| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically proided by Embeddings API |
-| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
-| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
-| "`clusering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
-
-
-Usage Example:
-
-```python
-import lancedb
-import pandas as pd
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-
-model = get_registry().get("gemini-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```
-
-### Cohere Embeddings
-Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
-You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
-
-Supported models are:
-```
-    * embed-english-v3.0
-    * embed-multilingual-v3.0
-    * embed-english-light-v3.0
-    * embed-multilingual-light-v3.0
-    * embed-english-v2.0
-    * embed-english-light-v2.0
-    * embed-multilingual-v2.0
-```
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
-| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
-| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
-
-Cohere supports following input types:
-
-| Input Type               | Description                          |
-|-------------------------|---------------------------------------|
-| "`search_document`"     | Used for embeddings stored in a vector|
-|                         | database for search use-cases.        |
-| "`search_query`"        | Used for embeddings of search queries |
-|                         | run against a vector DB               |
-| "`semantic_similarity`" | Specifies the given text will be used |
-|                         | for Semantic Textual Similarity (STS) |
-| "`classification`"      | Used for embeddings passed through a  |
-|                         | text classifier.                      |
-| "`clustering`"          | Used for the embeddings run through a |
-|                         | clustering algorithm                  |
-
-Usage Example:
-    
-    ```python
-        import lancedb
-        from lancedb.pydantic import LanceModel, Vector
-        from lancedb.embeddings import EmbeddingFunctionRegistry
-
-        cohere = EmbeddingFunctionRegistry
-            .get_instance()
-            .get("cohere")
-            .create(name="embed-multilingual-v2.0")
-
-        class TextModel(LanceModel):
-            text: str = cohere.SourceField()
-            vector: Vector(cohere.ndims()) =  cohere.VectorField()
-
-        data = [ { "text": "hello world" },
-                { "text": "goodbye world" }]
-
-        db = lancedb.connect("~/.lancedb")
-        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-        tbl.add(data)
-    ```
-
-### Jina Embeddings
-Jina embeddings are used to generate embeddings for text and image data.
-You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
-
-You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
-
-Usage Example:
-
-```python
-    import os
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    os.environ['JINA_API_KEY'] = 'jina_*'
-
-    jina_embed = EmbeddingFunctionRegistry.get_instance().get("jina").create(name="jina-embeddings-v2-base-en")
-
-
-    class TextModel(LanceModel):
-        text: str = jina_embed.SourceField()
-        vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
-
-
-    data = [{"text": "hello world"},
-            {"text": "goodbye world"}]
-
-    db = lancedb.connect("~/.lancedb-2")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```
-
-### AWS Bedrock Text Embedding Functions
-AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
-You can do so by using `awscli` and also add your session_token:
-```shell
-aws configure
-aws configure set aws_session_token "<your_session_token>"
-```
-to ensure that the credentials are set up correctly, you can run the following command:
-```shell
-aws sts get-caller-identity
-```
-
-Supported Embedding modelIDs are:
-* `amazon.titan-embed-text-v1`
-* `cohere.embed-english-v3`
-* `cohere.embed-multilingual-v3`
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
-| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
-| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
-| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
-| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
-| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
-| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
-
-Usage Example:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-model = get_registry().get("bedrock-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("tmp_path")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```
-
-# IBM watsonx.ai Embeddings
-
-Generate text embeddings using IBM's watsonx.ai platform.
-
-## Supported Models
-
-You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are:
-
-- `ibm/slate-125m-english-rtrvr`
-- `ibm/slate-30m-english-rtrvr`
-- `sentence-transformers/all-minilm-l12-v2`
-- `intfloat/multilingual-e5-large`
-
-## Parameters
-
-The following parameters can be passed to the `create` method:
-
-| Parameter  | Type     | Default Value                    | Description                                               |
-|------------|----------|----------------------------------|-----------------------------------------------------------|
-| name       | str      | "ibm/slate-125m-english-rtrvr"   | The model ID of the watsonx.ai model to use               |
-| api_key    | str      | None                             | Optional IBM Cloud API key (or set `WATSONX_API_KEY`)     |
-| project_id | str      | None                             | Optional watsonx project ID (or set `WATSONX_PROJECT_ID`) |
-| url        | str      | None                             | Optional custom URL for the watsonx.ai instance           |
-| params     | dict     | None                             | Optional additional parameters for the embedding model    |
-
-## Usage Example
-
-First, the watsonx.ai library is an optional dependency, so must be installed seperately:
-
-```
-pip install ibm-watsonx-ai
-```
-
-Optionally set environment variables (if not passing credentials to `create` directly):
-
-```sh
-export WATSONX_API_KEY="YOUR_WATSONX_API_KEY"
-export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID"
-```
-
-```python
-import os
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import EmbeddingFunctionRegistry
-
-watsonx_embed = EmbeddingFunctionRegistry
-  .get_instance()
-  .get("watsonx")
-  .create(
-    name="ibm/slate-125m-english-rtrvr",
-    # Uncomment and set these if not using environment variables
-    # api_key="your_api_key_here",
-    # project_id="your_project_id_here",
-    # url="your_watsonx_url_here",
-    # params={...},
-  )
-
-class TextModel(LanceModel):
-    text: str = watsonx_embed.SourceField()
-    vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"},
-]
-
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
-
-tbl.add(data)
-
-rs = tbl.search("hello").limit(1).to_pandas()
-print(rs)
-```
-
-## Multi-modal embedding functions
-Multi-modal embedding functions allow you to query your table using both images and text.
-
-### OpenClip embeddings
-We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"ViT-B-32"` | The name of the model. |
-| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `batch_size` | `int` | `64` | The number of images to process in a batch. |
-| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
-
-This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
-
-!!! info
-    LanceDB supports ingesting images directly from accessible links.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry.get("open-clip").create()
-
-class Images(LanceModel):
-    label: str
-    image_uri: str = func.SourceField() # image uri as the source
-    image_bytes: bytes = func.SourceField() # image bytes as the source
-    vector: Vector(func.ndims()) = func.VectorField() # vector column 
-    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
-
-table = db.create_table("images", schema=Images)
-labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
-uris = [
-    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
-    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
-    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
-    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
-    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
-    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
-]
-# get each uri as bytes
-image_bytes = [requests.get(uri).content for uri in uris]
-table.add(
-    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
-)
-```
-Now we can search using text from both the default vector column and the custom vector column
-```python
-
-# text search
-actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
-print(actual.label) # prints "dog"
-
-frombytes = (
-    table.search("man's best friend", vector_column_name="vec_from_bytes")
-    .limit(1)
-    .to_pydantic(Images)[0]
-)
-print(frombytes.label)
-
-```
-
-Because we're using a multi-modal embedding function, we can also search using images
-
-```python
-# image search
-query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
-image_bytes = requests.get(query_image_uri).content
-query_image = Image.open(io.BytesIO(image_bytes))
-actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
-print(actual.label == "dog")
-
-# image search using a custom vector column
-other = (
-    table.search(query_image, vector_column_name="vec_from_bytes")
-    .limit(1)
-    .to_pydantic(Images)[0]
-)
-print(actual.label)
-
-```
-
-### Imagebind embeddings
-We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
-
-This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"imagebind_huge"` | Name of the model. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
-
-Below is an example demonstrating how the API works:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry.get("imagebind").create()
-
-class ImageBindModel(LanceModel):
-    text: str
-    image_uri: str = func.SourceField()
-    audio_path: str
-    vector: Vector(func.ndims()) = func.VectorField()
-
-# add locally accessible image paths
-text_list=["A dog.", "A car", "A bird"]
-image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
-audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
-
-# Load data
-inputs = [
-    {"text": a, "audio_path": b, "image_uri": c}
-    for a, b, c in zip(text_list, audio_paths, image_paths)
-]
-
-#create table and add data
-table = db.create_table("img_bind", schema=ImageBindModel)
-table.add(inputs)
-```
-
-Now, we can search using any modality:
-
-#### image search
-```python
-query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
-actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "dog")
-```
-#### audio search
-
-```python
-query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
-actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "car")
-```
-#### Text search
-You can add any input query and fetch the result as follows:
-```python
-query = "an animal which flies and tweets" 
-actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "bird")
-```
-
-If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).
-
-### Jina Embeddings
-Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
-under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
-
-Usage Example:
-
-```python
-    import os
-    import requests
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-    import pandas as pd
-
-    os.environ['JINA_API_KEY'] = 'jina_*'
-
-    db = lancedb.connect("~/.lancedb")
-    func = get_registry().get("jina").create()
-
-
-    class Images(LanceModel):
-        label: str
-        image_uri: str = func.SourceField()  # image uri as the source
-        image_bytes: bytes = func.SourceField()  # image bytes as the source
-        vector: Vector(func.ndims()) = func.VectorField()  # vector column
-        vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
-
-
-    table = db.create_table("images", schema=Images)
-    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
-    uris = [
-        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
-        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
-        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
-        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
-        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
-        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
-    ]
-    # get each uri as bytes
-    image_bytes = [requests.get(uri).content for uri in uris]
-    table.add(
-      pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
-    )
-```

docs/src/embeddings/embedding_functions.md

@@ -1,206 +0,0 @@
-Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions can themselves be thought of as key part of the data processing pipeline that each request has to be passed through. The assumption here is: after initial setup, these components and the underlying methodology are not expected to change for a particular project.
-
-For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
-
-!!! Note "Embedding functions on LanceDB cloud"
-    When using embedding functions with LanceDB cloud, the embeddings will be generated on the source device and sent to the cloud. This means that the source device must have the necessary resources to generate the embeddings.
-
-!!! warning
-    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself.
-    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function
-    and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
-    table metadata and have LanceDB automatically take care of regenerating the embeddings.
-
-
-## 1. Define the embedding function
-
-=== "Python"
-    In the LanceDB python SDK, we define a global embedding function registry with
-    many different embedding models and even more coming soon.
-    Here's let's an implementation of CLIP as example.
-
-    ```python
-    from lancedb.embeddings import get_registry
-
-    registry = get_registry()
-    clip = registry.get("open-clip").create()
-    ```
-
-    You can also define your own embedding function by implementing the `EmbeddingFunction`
-    abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
-
-=== "TypeScript"
-    In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
-    embedding function is available.
-
-    ```javascript
-    import * as lancedb from '@lancedb/lancedb'
-    import { getRegistry } from '@lancedb/lancedb/embeddings'
-
-    // You need to provide an OpenAI API key
-    const apiKey = "sk-..."
-    // The embedding function will create embeddings for the 'text' column
-    const func = getRegistry().get("openai").create({apiKey})
-    ```
-=== "Rust"
-    In the Rust SDK, the choices are more limited. For now, only the OpenAI
-    embedding function is available. But unlike the Python and TypeScript SDKs, you need manually register the OpenAI embedding function.
-
-    ```toml
-    // Make sure to include the `openai` feature
-    [dependencies]
-    lancedb = {version = "*", features = ["openai"]}
-    ```
-
-    ```rust
-    --8<-- "rust/lancedb/examples/openai.rs:imports"
-    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-## 2. Define the data model or schema
-
-=== "Python"
-    The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
-
-    ```python
-    class Pets(LanceModel):
-        vector: Vector(clip.ndims()) = clip.VectorField()
-        image_uri: str = clip.SourceField()
-    ```
-
-    `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
-
-=== "TypeScript"
-
-    For the TypeScript SDK, a schema can be inferred from input data, or an explicit
-    Arrow schema can be provided.
-
-## 3. Create table and add data
-
-Now that we have chosen/defined our embedding function and the schema,
-we can create the table and ingest data without needing to explicitly generate
-the embeddings at all:
-
-=== "Python"
-    ```python
-    db = lancedb.connect("~/lancedb")
-    table = db.create_table("pets", schema=Pets)
-
-    table.add([{"image_uri": u} for u in uris])
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        --8<-- "nodejs/examples/embedding.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:embedding_function"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const db = await lancedb.connect("data/sample-lancedb");
-        const data = [
-            { text: "pepperoni"},
-            { text: "pineapple"}
-        ]
-
-        const table = await db.createTable("vectors", data, embedding)
-        ```
-
-## 4. Querying your table
-Not only can you forget about the embeddings during ingestion, you also don't
-need to worry about it when you query the table:
-
-=== "Python"
-
-    Our OpenCLIP query embedding function supports querying via both text and images:
-
-    ```python
-    results = (
-        table.search("dog")
-            .limit(10)
-            .to_pandas()
-    )
-    ```
-
-    Or we can search using an image:
-
-    ```python
-    p = Path("path/to/images/samoyed_100.jpg")
-    query_image = Image.open(p)
-    results = (
-        table.search(query_image)
-            .limit(10)
-            .to_pandas()
-    )
-    ```
-
-    Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        const results = await table.search("What's the best pizza topping?")
-            .limit(10)
-            .toArray()
-        ```
-
-    === "vectordb (deprecated)
-
-        ```ts
-        const results = await table
-            .search("What's the best pizza topping?")
-            .limit(10)
-            .execute()
-        ```
-
-    The above snippet returns an array of records with the top 10 nearest neighbors to the query.
-
----
-
-## Rate limit Handling
-`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default, the maximum retires is set to 7. You can tune it by setting it to a different number, or disable it by setting it to 0.
-
-An example of how to do this is shown below:
-
-```python
-clip = registry.get("open-clip").create() # Defaults to 7 max retries
-clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
-clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
-```
-
-!!! note
-    Embedding functions can also fail due to other errors that have nothing to do with rate limits.
-    This is why the error is also logged.
-
-## Some fun with Pydantic
-
-LanceDB is integrated with Pydantic, which was used in the example above to define the schema in Python. It's also used behind the scenes by the embedding function API to ingest useful information as table metadata.
-
-You can also use the integration for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let's define a utility function to plot the image.
-
-```python
-class Pets(LanceModel):
-    vector: Vector(clip.ndims()) = clip.VectorField()
-    image_uri: str = clip.SourceField()
-
-    @property
-    def image(self):
-        return Image.open(self.image_uri)
-```
-Now, you can covert your search results to a Pydantic model and use this property.
-
-```python
-rs = table.search(query_image).limit(3).to_pydantic(Pets)
-rs[2].image
-```
-
-![](../assets/dog_clip_output.png)
-
-Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
-let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -1,134 +0,0 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio.
-This makes them a very powerful tool for machine learning practitioners.
-However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs
-(both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
-
-LanceDB supports 3 methods of working with embeddings.
-
-1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
-2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
-3. You can define your own [custom embedding function](./custom_embedding_function.md)
-   that extends the default embedding functions.
-
-For python users, there is also a legacy [with_embeddings API](./legacy.md).
-It is retained for compatibility and will be removed in a future version.
-
-## Quickstart
-
-To get started with embeddings, you can use the built-in embedding functions.
-
-### OpenAI Embedding function
-
-LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
-
-=== "Python"
-
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-    class Words(LanceModel):
-        text: str = func.SourceField()
-        vector: Vector(func.ndims()) = func.VectorField()
-
-    table = db.create_table("words", schema=Words, mode="overwrite")
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-        )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-
-=== "TypeScript"
-
-    ```typescript
-    --8<--- "nodejs/examples/embedding.ts:imports"
-    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<--- "rust/lancedb/examples/openai.rs:imports"
-    --8<--- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-=== "Python"
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-    )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-
-=== "TypeScript"
-
-    Coming Soon!
-
-=== "Rust"
-
-    Coming Soon!
-
-### Embedding function with LanceDB cloud
-Embedding functions are now supported on LanceDB cloud. The embeddings will be generated on the source device and sent to the cloud. This means that the source device must have the necessary resources to generate the embeddings. Here's an example using the OpenAI embedding function:
-
-```python
-import os
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-os.environ['OPENAI_API_KEY'] = "..."
-
-db = lancedb.connect(
-  uri="db://....",
-  api_key="sk_...",
-  region="us-east-1"
-)
-func = get_registry().get("openai").create()
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/legacy.md

@@ -1,99 +0,0 @@
-The legacy `with_embeddings` API is for Python only and is deprecated.
-
-### Hugging Face
-
-The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
-library, which can be installed via pip.
-
-```bash
-pip install sentence-transformers
-```
-
-The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
-for a given document.
-
-```python
-from sentence_transformers import SentenceTransformer
-
-name="paraphrase-albert-small-v2"
-model = SentenceTransformer(name)
-
-# used for both training and querying
-def embed_func(batch):
-    return [model.encode(sentence) for sentence in batch]
-```
-
-
-### OpenAI
-
-Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
-
-```python
-import openai
-import os
-
-# Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
-# OR set the key here as a variable
-openai.api_key = "sk-..."
-
-client = openai.OpenAI()
-
-def embed_func(c):    
-    rs = client.embeddings.create(input=c, model="text-embedding-ada-002")
-    return [record.embedding for record in rs["data"]]
-```
-
-
-## Applying an embedding function to data
-
-Using an embedding function, you can apply it to raw data
-to generate embeddings for each record.
-
-Say you have a pandas DataFrame with a `text` column that you want embedded,
-you can use the `with_embeddings` function to generate embeddings and add them to 
-an existing table.
-
-```python
-    import pandas as pd
-    from lancedb.embeddings import with_embeddings
-
-    df = pd.DataFrame(
-        [
-            {"text": "pepperoni"},
-            {"text": "pineapple"}
-        ]
-    )
-    data = with_embeddings(embed_func, df)
-
-    # The output is used to create / append to a table
-    tbl = db.create_table("my_table", data=data)
-```
-
-If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-using the `batch_size` parameter to `with_embeddings`.
-
-LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-API call is reliable.
-
-## Querying using an embedding function
-
-!!! warning
-    At query time, you **must** use the same embedding function you used to vectorize your data.
-    If you use a different embedding function, the embeddings will not reside in the same vector
-    space and the results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     results = (
-        tbl.search(query_vector)
-        .limit(10)
-        .to_pandas()
-     )
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/examples/code_documentation_qa_bot_with_langchain.md

@@ -1,7 +0,0 @@
-# Code documentation Q&A bot with LangChain
-
-## use LanceDB's LangChain integration to build a Q&A bot for your documentation
-
-<img id="splash" width="400" alt="langchain" src="https://user-images.githubusercontent.com/917119/236580868-61a246a9-e587-4c2b-8ae5-6fe5f7b7e81e.png">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/code_qa_bot.ipynb)

docs/src/examples/examples_js.md

@@ -1,11 +0,0 @@
-# Examples: JavaScript
-
-To help you get started, we provide some examples, projects and applications that use the LanceDB JavaScript API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
-
-| Example | Scripts  |
-|-------- | ------   |
-| | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/index.js)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
-| [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js)  |

docs/src/examples/examples_python.md

@@ -1,17 +0,0 @@
-# Examples: Python
-
-To help you get started, we provide some examples, projects and applications that use the LanceDB Python API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
-
-| Example | Interactive Envs | Scripts  |
-|-------- | ---------------- | ------   |
-| | | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
-| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
-| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
-| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
-| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
-| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
-| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
-

docs/src/examples/examples_rust.md

@@ -1,3 +0,0 @@
-# Examples: Rust
-
-Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -1,165 +0,0 @@
-# How to Load Image Embeddings into LanceDB
-
-With the rise of Large Multimodal Models (LMMs) such as [GPT-4 Vision](https://blog.roboflow.com/gpt-4-vision/), the need for storing image embeddings is growing. The most effective way to store text and image embeddings is in a vector database such as LanceDB. Vector databases are a special kind of data store that enables efficient search over stored embeddings. 
-
-[CLIP](https://blog.roboflow.com/openai-clip/), a multimodal model developed by OpenAI, is commonly used to calculate image embeddings. These embeddings can then be used with a vector database to build a semantic search engine that you can query using images or text. For example, you could use LanceDB and CLIP embeddings to build a search engine for a database of folders.
-
-In this guide, we are going to show you how to use Roboflow Inference to load image embeddings into LanceDB. Without further ado, let’s get started!
-
-## Step #1: Install Roboflow Inference
-
-[Roboflow Inference](https://inference.roboflow.com) enables you to run state-of-the-art computer vision models with minimal configuration. Inference supports a range of models, from fine-tuned object detection, classification, and segmentation models to foundation models like CLIP. We will use Inference to calculate CLIP image embeddings.
-
-Inference provides a HTTP API through which you can run vision models.
-
-Inference powers the Roboflow hosted API, and is available as an open source utility. In this guide, we are going to run Inference locally, which enables you to calculate CLIP embeddings on your own hardware. We will also show you how to use the hosted Roboflow CLIP API, which is ideal if you need to scale and do not want to manage a system for calculating embeddings.
-
-To get started, first install the Inference CLI:
-
-```
-pip install inference-cli
-```
-
-Next, install Docker. Refer to the official Docker installation instructions for your operating system to get Docker set up. Once Docker is ready, you can start Inference using the following command:
-
-```
-inference server start
-```
-
-An Inference server will start running at ‘http://localhost:9001’.
-
-## Step #2: Set Up a LanceDB Vector Database
-
-Now that we have Inference running, we can set up a LanceDB vector database. You can run LanceDB in JavaScript and Python. For this guide, we will use the Python API. But, you can take the HTTP requests we make below and change them to JavaScript if required.
-
-For this guide, we are going to search the [COCO 128 dataset](https://universe.roboflow.com/team-roboflow/coco-128), which contains a wide range of objects. The variability in objects present in this dataset makes it a good dataset to demonstrate the capabilities of vector search. If you want to use this dataset, you can download [COCO 128 from Roboflow Universe](https://universe.roboflow.com/team-roboflow/coco-128). With that said, you can search whatever folder of images you want.
-
-Once you have a dataset ready, install LanceDB with the following command:
-
-```
-pip install lancedb
-```
-
-We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
-
-```
-pip install tantivy
-```
-
-Create a new Python file and add the following code:
-
-```python
-import cv2
-import supervision as sv
-import requests
-
-import lancedb
-
-db = lancedb.connect("./embeddings")
-
-IMAGE_DIR = "images/"
-API_KEY = os.environ.get("ROBOFLOW_API_KEY")
-SERVER_URL = "http://localhost:9001"
-
-results = []
-
-for i, image in enumerate(os.listdir(IMAGE_DIR)):
-    infer_clip_payload = {
-        #Images can be provided as urls or as base64 encoded strings
-        "image": {
-            "type": "base64",
-            "value": base64.b64encode(open(IMAGE_DIR + image, "rb").read()).decode("utf-8"),
-        },
-    }
-
-    res = requests.post(
-        f"{SERVER_URL}/clip/embed_image?api_key={API_KEY}",
-        json=infer_clip_payload,
-    )
-
-    embeddings = res.json()['embeddings']
-
-    print("Calculated embedding for image: ", image)
-
-    image = {"vector": embeddings[0], "name": os.path.join(IMAGE_DIR, image)}
-
-    results.append(image)
-
-tbl = db.create_table("images", data=results)
-
-tbl.create_fts_index("name")
-```
-
-To use the code above, you will need a Roboflow API key. [Learn how to retrieve a Roboflow API key](https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key). Run the following command to set up your API key in your environment:
-
-```
-export ROBOFLOW_API_KEY=""
-```
-
-Replace the `IMAGE_DIR` value with the folder in which you are storing the images for which you want to calculate embeddings. If you want to use the Roboflow CLIP API to calculate embeddings, replace the `SERVER_URL` value with `https://infer.roboflow.com`.
-
-Run the script above to create a new LanceDB database. This database will be stored on your local machine. The database will be called `embeddings` and the table will be called `images`.
-
-The script above calculates all embeddings for a folder then creates a new table. To add additional images, use the following code:
-
-```python
-def make_batches():
-    for i in range(5):
-        yield [
-                {"vector": [3.1, 4.1], "name": "image1.png"},
-                {"vector": [5.9, 26.5], "name": "image2.png"}
-            ]
-
-tbl = db.open_table("images")
-tbl.add(make_batches())
-```
-
-Replacing the `make_batches()` function with code to load embeddings for images.
-
-## Step #3: Run a Search Query
-
-We are now ready to run a search query. To run a search query, we need a text embedding that represents a text query. We can use this embedding to search our LanceDB database for an entry.
-
-Let’s calculate a text embedding for the query “cat”, then run a search query:
-
-```python
-infer_clip_payload = {
-    "text": "cat",
-}
-
-res = requests.post(
-    f"{SERVER_URL}/clip/embed_text?api_key={API_KEY}",
-    json=infer_clip_payload,
-)
-
-embeddings = res.json()['embeddings']
-
-df = tbl.search(embeddings[0]).limit(3).to_list()
-
-print("Results:")
-
-for i in df:
-    print(i["name"])
-```
-
-This code will search for the three images most closely related to the prompt “cat”. The names of the most similar three images will be printed to the console. Here are the three top results:
-
-```
-dataset/images/train/000000000650_jpg.rf.1b74ba165c5a3513a3211d4a80b69e1c.jpg
-dataset/images/train/000000000138_jpg.rf.af439ef1c55dd8a4e4b142d186b9c957.jpg
-dataset/images/train/000000000165_jpg.rf.eae14d5509bf0c9ceccddbb53a5f0c66.jpg
-```
-
-Let’s open the top image:
-
-![Cat](https://media.roboflow.com/cat_lancedb.jpg)
-
-The top image was a cat. Our search was successful.
-
-## Conclusion
-
-LanceDB is a vector database that you can use to store and efficiently search your image embeddings. You can use Roboflow Inference, a scalable computer vision inference server, to calculate CLIP embeddings that you can store in LanceDB.
-
-You can use Inference and LanceDB together to build a range of applications with image embeddings, from a media search engine to a retrieval-augmented generation pipeline for use with LMMs.
-
-To learn more about Inference and its capabilities, refer to the Inference documentation.

docs/src/examples/index.md

@@ -1,16 +0,0 @@
-# Example projects and recipes
-
-## Recipes and example code
-
-LanceDB provides language APIs, allowing you to embed a database in your language of choice.
-
-* 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](examples_js.md) examples
-* 🦀 Rust examples (coming soon)
-
-## Applications powered by LanceDB
-
-| Project Name | Description |
-| --- | --- |
-| **Ultralytics Explorer 🚀**<br>[![Ultralytics](https://img.shields.io/badge/Ultralytics-Docs-green?labelColor=0f3bc4&style=flat-square&logo=https://cdn.prod.website-files.com/646dd1f1a3703e451ba81ecc/64994922cf2a6385a4bf4489_UltralyticsYOLO_mark_blue.svg&link=https://docs.ultralytics.com/datasets/explorer/)](https://docs.ultralytics.com/datasets/explorer/)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ultralytics/ultralytics/blob/main/docs/en/datasets/explorer/explorer.ipynb) |  - 🔍 **Explore CV Datasets**: Semantic search, SQL queries, vector similarity, natural language.<br>- 🖥️ **GUI & Python API**: Seamless dataset interaction.<br>- ⚡ **Efficient & Scalable**: Leverages LanceDB for large datasets.<br>- 📊 **Detailed Analysis**: Easily analyze data patterns.<br>- 🌐 **Browser GUI Demo**: Create embeddings, search images, run queries. |
-| **Website Chatbot🤖**<br>[![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/lancedb-vercel-chatbot)<br>[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&amp;env=OPENAI_API_KEY&amp;envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&amp;project-name=lancedb-vercel-chatbot&amp;repository-name=lancedb-vercel-chatbot&amp;demo-title=LanceDB%20Chatbot%20Demo&amp;demo-description=Demo%20website%20chatbot%20with%20LanceDB.&amp;demo-url=https%3A%2F%2Flancedb.vercel.app&amp;demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png) | - 🌐 **Chatbot from Sitemap/Docs**: Create a chatbot using site or document context.<br>- 🚀 **Embed LanceDB in Next.js**: Lightweight, on-prem storage.<br>- 🧠 **AI-Powered Context Retrieval**: Efficiently access relevant data.<br>- 🔧 **Serverless & Native JS**: Seamless integration with Next.js.<br>- ⚡ **One-Click Deploy on Vercel**: Quick and easy setup.. |

docs/src/examples/modal_langchain.py

@@ -1,119 +0,0 @@
-import pickle
-import re
-import zipfile
-from pathlib import Path
-
-import requests
-from langchain.chains import RetrievalQA
-from langchain.document_loaders import UnstructuredHTMLLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import LanceDB
-from modal import Image, Secret, Stub, web_endpoint
-
-import lancedb
-
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb", "langchain", "openai", "pandas", "tiktoken", "unstructured", "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-
-
-def get_document_title(document):
-    m = str(document.metadata["source"])
-    title = re.findall("pandas.documentation(.*).html", m)
-    if title[0] is not None:
-        return title[0]
-    return ""
-
-
-def download_docs():
-    pandas_docs = requests.get(
-        "https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"
-    )
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-
-
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-
-
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path)
-    table = db.create_table(
-        "pandas_docs",
-        data=[
-            {
-                "vector": embeddings.embed_query("Hello World"),
-                "text": "Hello World",
-                "id": "1",
-            }
-        ],
-        mode="overwrite",
-    )
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(
-        llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever()
-    )
-    return qa.run(query)
-
-
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-
-
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)

docs/src/examples/multimodal_search.md

@@ -1,7 +0,0 @@
-# Image multimodal search
-
-## Search through an image dataset using natural language, full text and SQL
-
-<img id="splash" width="400" alt="multimodal search" src="https://github.com/lancedb/lancedb/assets/917119/993a7c9f-be01-449d-942e-1ce1d4ed63af">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/multimodal_search.ipynb)

docs/src/examples/python_examples/build_from_scratch.md

@@ -1,13 +0,0 @@
-# Build from Scratch with LanceDB 🚀
-
-Start building your GenAI applications from the ground up using LanceDB's efficient vector-based document retrieval capabilities! 📄
-
-#### Get Started in Minutes ⏱️
-
-These examples provide a solid foundation for building your own GenAI applications using LanceDB. Jump from idea to proof of concept quickly with applied examples. Get started and see what you can create! 💻
-
-| **Build From Scratch**                     | **Description**                                                                                              | **Links**                                                                                                                                                                                                                                                                                                                                                                                                                |
-|:-------------------------------------------|:-------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Build RAG from Scratch🚀💻**             | 📝 Create a **Retrieval-Augmented Generation** (RAG) model from scratch using LanceDB.                       | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/RAG-from-Scratch)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)]()                                                                                                                                  |
-| **Local RAG from Scratch with Llama3🔥💡** | 🐫 Build a local RAG model using **Llama3** and **LanceDB** for fast and efficient text generation.          | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Local-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Local-RAG-from-Scratch/rag.py)    |
-| **Multi-Head RAG from Scratch📚💻**        | 🤯 Develop a **Multi-Head RAG model** from scratch, enabling generation of text based on multiple documents. | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch) |

docs/src/examples/python_examples/multimodal.md

@@ -1,28 +0,0 @@
-# Multimodal Search with LanceDB 🔍💡
-
-Experience the future of search with LanceDB's multimodal capabilities. Combine text and image queries to find the most relevant results in your corpus and unlock new possibilities! 🔓💡
-
-#### Explore the Future of Search 🚀
-
-Unlock the power of multimodal search with LanceDB, enabling efficient vector-based retrieval of text and image data! 📊💻
-
-
-
-| **Multimodal**  | **Description**  | **Links**  |
-|:----------------|:-----------------|:-----------|
-| **Multimodal CLIP: DiffusionDB 🌐💥**    | Revolutionize search with Multimodal CLIP and DiffusionDB, combining text and image understanding for a new dimension of discovery! 🔓 | [![GitHub](../../assets/github.svg)][Clip_diffusionDB_github] <br>[![Open In Collab](../../assets/colab.svg)][Clip_diffusionDB_colab] <br>[![Python](../../assets/python.svg)][Clip_diffusionDB_python] <br>[![Ghost](../../assets/ghost.svg)][Clip_diffusionDB_ghost] |
-| **Multimodal CLIP: Youtube Videos 📹👀** | Search Youtube videos using Multimodal CLIP, finding relevant content with ease and accuracy! 🎯                                                                                                                                                                | [![Github](../../assets/github.svg)][Clip_youtube_github]                                                                                                    <br>[![Open In Collab](../../assets/colab.svg)][Clip_youtube_colab]                                                                                           <br> [![Python](../../assets/python.svg)][Clip_youtube_python]         <br>[![Ghost](../../assets/ghost.svg)][Clip_youtube_python] |
-| **Multimodal Image + Text Search 📸🔍**               | Discover relevant documents and images with a single query, using LanceDB's multimodal search capabilities to bridge the gap between text and visuals! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
-| **Cambrian-1: Vision-Centric Image Exploration 🔍👀** | Dive into vision-centric exploration of images with Cambrian-1, powered by LanceDB's multimodal search to uncover new insights! 🔎                        | [![GitHub](../../assets/github.svg)](https://www.kaggle.com/code/prasantdixit/cambrian-1-vision-centric-exploration-of-images/)<br>[![Open In Collab](../../assets/colab.svg)]() <br> [![Python](../../assets/python.svg)]() <br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/cambrian-1-vision-centric-exploration/)                                                                                                                                                                                                                   |
-
-
-[Clip_diffusionDB_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb
-[Clip_diffusionDB_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.ipynb
-[Clip_diffusionDB_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.py
-[Clip_diffusionDB_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/
-
-
-[Clip_youtube_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search
-[Clip_youtube_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb
-[Clip_youtube_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.py
-[Clip_youtube_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/

docs/src/examples/python_examples/rag.md

@@ -1,85 +0,0 @@
-
-**🔍💡 RAG: Revolutionize Information Retrieval with LanceDB 🔓**
-====================================================================
-
-Unlock the full potential of Retrieval-Augmented Generation (RAG) with LanceDB, the ultimate solution for efficient vector-based information retrieval 📊. Input text queries and retrieve relevant documents with lightning-fast speed ⚡️ and accuracy ✅. Generate comprehensive answers by combining retrieved information, uncovering new insights 🔍 and connections.
-
-### Experience the Future of Search 🔄
-
-Experience the future of search with RAG, transforming information retrieval and answer generation. Apply RAG to various industries, streamlining processes 📈, saving time ⏰, and resources 💰. Stay ahead of the curve with innovative technology 🔝, powered by LanceDB. Discover the power of RAG with LanceDB and transform your industry with innovative solutions 💡.
-
-
-| **RAG**                                      | **Description**                                                                                                                                                  | **Links**                  |
-|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|                                                                                                                                                     
-| **RAG with Matryoshka Embeddings and LlamaIndex** 🪆🔗  | Utilize **Matryoshka embeddings** and **LlamaIndex** to improve the efficiency and accuracy of your RAG models. 📈✨ | [![Github](../../assets/github.svg)][matryoshka_github] <br>[![Open In Collab](../../assets/colab.svg)][matryoshka_colab]  |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with Re-ranking** 📈🔄 | Enhance your RAG applications by implementing **re-ranking strategies** for more relevant document retrieval. 📚🔍 | [![Github](../../assets/github.svg)][rag_reranking_github] <br>[![Open In Collab](../../assets/colab.svg)][rag_reranking_colab] <br>[![Ghost](../../assets/ghost.svg)][rag_reranking_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Instruct-Multitask** 🧠🎯 |  Integrate the **Instruct Embedding Model** with LanceDB to streamline your embedding API, reducing redundant code and overhead. 🌐📊 | [![Github](../../assets/github.svg)][instruct_multitask_github] <br>[![Open In Collab](../../assets/colab.svg)][instruct_multitask_colab] <br>[![Python](../../assets/python.svg)][instruct_multitask_python] <br>[![Ghost](../../assets/ghost.svg)][instruct_multitask_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with HyDE** 🌌🔍 | Use **Hypothetical Document Embeddings** for efficient, accurate, and unsupervised dense retrieval. 📄🔍 | [![Github](../../assets/github.svg)][hyde_github] <br>[![Open In Collab](../../assets/colab.svg)][hyde_colab]<br>[![Ghost](../../assets/ghost.svg)][hyde_ghost]                 |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with LOTR** 🧙‍♂️📜 | Enhance RAG with **Lord of the Retriever (LOTR)** to address 'Lost in the Middle' challenges, especially in medical data. 🌟📜 | [![Github](../../assets/github.svg)][lotr_github] <br>[![Open In Collab](../../assets/colab.svg)][lotr_colab] <br>[![Ghost](../../assets/ghost.svg)][lotr_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Advanced RAG: Parent Document Retriever** 📑🔗 | Use **Parent Document & Bigger Chunk Retriever** to maintain context and relevance when generating related content. 🎵📄 | [![Github](../../assets/github.svg)][parent_doc_retriever_github] <br>[![Open In Collab](../../assets/colab.svg)][parent_doc_retriever_colab] <br>[![Ghost](../../assets/ghost.svg)][parent_doc_retriever_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Corrective RAG with Langgraph** 🔧📊 | Enhance RAG reliability with **Corrective RAG (CRAG)** by self-reflecting and fact-checking for accurate and trustworthy results. ✅🔍 |[![Github](../../assets/github.svg)][corrective_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][corrective_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][corrective_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Contextual Compression with RAG** 🗜️🧠 | Apply **contextual compression techniques** to condense large documents while retaining essential information. 📄🗜️                                               | [![Github](../../assets/github.svg)][compression_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][compression_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][compression_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with FLARE** 🔥|  Enable users to ask questions directly to academic papers, focusing on ArXiv papers, with Forward-Looking Active REtrieval augmented generation.🚀🌟                                                                       | [![Github](../../assets/github.svg)][flare_github] <br>[![Open In Collab](../../assets/colab.svg)][flare_colab] <br>[![Ghost](../../assets/ghost.svg)][flare_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Query Expansion and Reranker** 🔍🔄 | Enhance RAG with query expansion using Large Language Models and advanced **reranking methods** like Cross Encoders, ColBERT v2, and FlashRank for improved document retrieval precision and recall 🔍📈                                                         | [![Github](../../assets/github.svg)][query_github] <br>[![Open In Collab](../../assets/colab.svg)][query_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **RAG Fusion** ⚡🌐 | Revolutionize search with RAG Fusion, utilizing the **RRF algorithm** to rerank documents based on user queries, and leveraging LanceDB and OPENAI Embeddings for efficient information retrieval ⚡🌐                                                                          | [![Github](../../assets/github.svg)][fusion_github] <br>[![Open In Collab](../../assets/colab.svg)][fusion_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Agentic RAG** 🤖📚 | Unlock autonomous information retrieval with **Agentic RAG**, a framework of **intelligent agents** that collaborate to synthesize, summarize, and compare data across sources, enabling proactive and informed decision-making 🤖📚                                                              | [![Github](../../assets/github.svg)][agentic_github] <br>[![Open In Collab](../../assets/colab.svg)][agentic_colab] |
-
-
-
-
-
-
-
-
-
-
-
-
-[matryoshka_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex
-[matryoshka_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex/RAG_with_MatryoshkaEmbedding_and_Llamaindex.ipynb
-
-[rag_reranking_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking
-[rag_reranking_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking/main.ipynb
-[rag_reranking_ghost]: https://blog.lancedb.com/simplest-method-to-improve-rag-pipeline-re-ranking-cf6eaec6d544
-
-
-[instruct_multitask_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask
-[instruct_multitask_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.ipynb
-[instruct_multitask_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.py
-[instruct_multitask_ghost]: https://blog.lancedb.com/multitask-embedding-with-lancedb-be18ec397543
-
-[hyde_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE
-[hyde_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb
-[hyde_ghost]: https://blog.lancedb.com/advanced-rag-precise-zero-shot-dense-retrieval-with-hyde-0946c54dfdcb
-
-[lotr_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR
-[lotr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR/main.ipynb
-[lotr_ghost]: https://blog.lancedb.com/better-rag-with-lotr-lord-of-retriever-23c8336b9a35
-
-[parent_doc_retriever_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever
-[parent_doc_retriever_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever/main.ipynb
-[parent_doc_retriever_ghost]: https://blog.lancedb.com/modified-rag-parent-document-bigger-chunk-retriever-62b3d1e79bc6
-
-[corrective_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph
-[corrective_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb
-[corrective_rag_ghost]: https://blog.lancedb.com/implementing-corrective-rag-in-the-easiest-way-2/
-
-[compression_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG
-[compression_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG/main.ipynb
-[compression_rag_ghost]: https://blog.lancedb.com/enhance-rag-integrate-contextual-compression-and-filtering-for-precision-a29d4a810301/
-
-[flare_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR
-[flare_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb
-[flare_ghost]: https://blog.lancedb.com/better-rag-with-active-retrieval-augmented-generation-flare-3b66646e2a9f/
-
-[query_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker
-[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker/main.ipynb
-
-
-[fusion_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion
-[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion/main.ipynb
-
-[agentic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG
-[agentic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb
-
-

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -1,106 +0,0 @@
-# Serverless LanceDB
-
-## Store your data on S3 and use Lambda to compute embeddings and retrieve queries in production easily.
-
-<img id="splash" width="400" alt="s3-lambda" src="https://user-images.githubusercontent.com/917119/234653050-305a1e90-9305-40ab-b014-c823172a948c.png">
-
-This is a great option if you're wanting to scale with your use case and save effort and costs of maintenance.
-
-Let's walk through how to get a simple Lambda function that queries the SIFT dataset on S3.
-
-Before we start, you'll need to ensure you create a secure account access to AWS. We recommend using user policies, as this way AWS can share credentials securely without you having to pass around environment variables into Lambda.
-
-We'll also use a container to ship our Lambda code. This is a good option for Lambda as you don't have the space limits that you would otherwise by building a package yourself.
-
-# Initial setup: creating a LanceDB Table and storing it remotely on S3
-
-We'll use the SIFT vector dataset as an example. To make it easier, we've already made a Lance-format SIFT dataset publicly available, which we can access and use to populate our LanceDB Table. 
-
-To do this, download the Lance files locally first from:
-
-```
-s3://eto-public/datasets/sift/vec_data.lance
-```
-
-Then, we can write a quick Python script to populate our LanceDB Table:
-
-```python
-import pylance
-sift_dataset = pylance.dataset("/path/to/local/vec_data.lance")
-df = sift_dataset.to_table().to_pandas()
-
-import lancedb
-db = lancedb.connect(".")
-table = db.create_table("vector_example", df)
-```
-
-Once we've created our Table, we are free to move this data over to S3 so we can remotely host it.
-
-# Building our Lambda app: a simple event handler for vector search
-
-Now that we've got a remotely hosted LanceDB Table, we'll want to be able to query it from Lambda. To do so, let's create a new `Dockerfile` using the AWS python container base:
-
-```docker
-FROM public.ecr.aws/lambda/python:3.10
-
-RUN pip3 install --upgrade pip
-RUN pip3 install --no-cache-dir -U numpy --target "${LAMBDA_TASK_ROOT}"
-RUN pip3 install --no-cache-dir -U lancedb --target "${LAMBDA_TASK_ROOT}"
-
-COPY app.py ${LAMBDA_TASK_ROOT}
-
-CMD [ "app.handler" ]
-```
-
-Now let's make a simple Lambda function that queries the SIFT dataset in `app.py`.
-
-```python    
-import json
-import numpy as np
-import lancedb
-
-db = lancedb.connect("s3://eto-public/tables")
-table = db.open_table("vector_example")
-
-def handler(event, context):
-    status_code = 200
-
-    if event['query_vector'] is None:
-        status_code = 404
-        return {
-            "statusCode": status_code,
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body": json.dumps({
-                "Error ": "No vector to query was issued"
-            })
-        }
-    
-    # Shape of SIFT is (128,1M), d=float32
-    query_vector = np.array(event['query_vector'], dtype=np.float32)
-
-    rs = table.search(query_vector).limit(2).to_list()
-
-    return {
-        "statusCode": status_code,
-        "headers": {
-            "Content-Type": "application/json"
-        },
-        "body": json.dumps(rs)
-    }
-``` 
-
-# Deploying the container to ECR
-
-The next step is to build and push the container to ECR, where it can then be used to create a new Lambda function. 
-
-It's best to follow the official AWS documentation for how to do this, which you can view here:
-
-```
-https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-upload
-```
-
-# Final step: setting up your Lambda function
-
-Once the container is pushed, you can create a Lambda function by selecting the container. 

docs/src/examples/serverless_qa_bot_with_modal_and_langchain.md

@@ -1,166 +0,0 @@
-# Serverless QA Bot with Modal and LangChain
-
-## use LanceDB's LangChain integration with Modal to run a serverless app
-
-<img id="splash" width="400" alt="modal" src="https://github.com/lancedb/lancedb/assets/917119/7d80a40f-60d7-48a6-972f-dab05000eccf">
-
-We're going to build a QA bot for your documentation using LanceDB's LangChain integration and use Modal for deployment.
-
-Modal is an end-to-end compute platform for model inference, batch jobs, task queues, web apps and more. It's a great way to deploy your LanceDB models and apps.
-
-To get started, ensure that you have created an account and logged into [Modal](https://modal.com/). To follow along, the full source code is available on Github [here](https://github.com/lancedb/lancedb/blob/main/docs/src/examples/modal_langchain.py).
-
-### Setting up Modal
-
-We'll start by specifying our dependencies and creating a new Modal `Stub`:
-
-```python
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-```
-
-We're using Modal's Secrets injection to secure our OpenAI key. To set your own, you can access the Modal UI and enter your key.
-
-### Setting up caches for LanceDB and LangChain
-
-Next, we can setup some globals to cache our LanceDB database, as well as our LangChain docsource:
-
-```python
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-```
-
-### Downloading our dataset
-
-We're going use a pregenerated dataset, which stores HTML files of the Pandas 2.0 documentation. 
-You could switch this out for your own dataset.
-
-```python
-def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-```
-
-### Pre-processing the dataset and generating metadata
-
-Once we've downloaded it, we want to parse and pre-process them using LangChain, and then vectorize them and store it in LanceDB.
-Let's first create a function that uses LangChains `UnstructuredHTMLLoader` to parse them.
-We can then add our own metadata to it and store it alongside the data, we'll later be able to use this for filtering metadata.
-
-```python
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-```
-
-### Simple LangChain chain for a QA bot
-
-Now we can create a simple LangChain chain for our QA bot. We'll use the `RecursiveCharacterTextSplitter` to split our documents into chunks, and then use the `OpenAIEmbeddings` to vectorize them.
-
-Lastly, we'll create a LanceDB table and store the vectorized documents in it, then create a `RetrievalQA` model from the chain and return it.
-
-```python
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
-    return qa.run(query)
-```
-
-### Creating our Modal entry points
-
-Now we can create our Modal entry points for our CLI and web endpoint:
-
-```python
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-    
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)
-```
-
-# Testing it out!
-
-Testing the CLI:
-
-```bash
-modal run modal_langchain.py --query "What are the major differences in pandas 2.0?"
-```
-
-Testing the web endpoint:
-
-```bash
-modal serve modal_langchain.py
-```
-
-In the CLI, Modal will provide you a web endpoint. Copy this endpoint URI for the next step.
-Once this is served, then we can hit it with `curl`. 
-
-Note, the first time this runs, it will take a few minutes to download the dataset and vectorize it.
-An actual production example would pre-cache/load the dataset and vectorized documents prior
-
-```bash
-curl --get --data-urlencode "query=What are the major differences in pandas 2.0?" https://your-modal-endpoint-app.modal.run
-
-{"answer":" The major differences in pandas 2.0 include the ability to use any numpy numeric dtype in a Index, installing optional dependencies with pip extras, and enhancements, bug fixes, and performance improvements."}
-```
-

docs/src/examples/serverless_website_chatbot.md

@@ -1,61 +0,0 @@
-# LanceDB Chatbot - Vercel Next.js Template
-Use an AI chatbot with website context retrieved from a vector store like LanceDB. LanceDB is lightweight and can be embedded directly into Next.js, with data stored on-prem.
-
-## One click deploy on Vercel
-[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&env=OPENAI_API_KEY&envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&project-name=lancedb-vercel-chatbot&repository-name=lancedb-vercel-chatbot&demo-title=LanceDB%20Chatbot%20Demo&demo-description=Demo%20website%20chatbot%20with%20LanceDB.&demo-url=https%3A%2F%2Flancedb.vercel.app&demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png)
-
-![Demo website landing page](../assets/vercel-template.gif)
-
-## Development
-
-First, rename `.env.example` to `.env.local`, and fill out `OPENAI_API_KEY` with your OpenAI API key. You can get one [here](https://openai.com/blog/openai-api).
-
-Run the development server:
-
-```bash
-npm run dev
-# or
-yarn dev
-# or
-pnpm dev
-```
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
-
-## Learn More
-
-To learn more about LanceDB or Next.js, take a look at the following resources:
-
-- [LanceDB Documentation](https://lancedb.github.io/lancedb/) - learn about LanceDB, the developer-friendly serverless vector database.
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
-
-## LanceDB on Next.js and Vercel
-
-FYI: these configurations have been pre-implemented in this template.
-
-Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
-```js
-/** @type {import('next').NextConfig} */
-module.exports = ({
-  webpack(config) {
-    config.externals.push({ vectordb: 'vectordb' })
-    return config;
-  }
-})
-```
-
-To deploy on Vercel, we need to make sure that the NodeJS runtime static file analysis for Vercel can find the binary, since LanceDB uses dynamic imports by default. We can do this by modifying `package.json` in the `scripts` section.
-```json
-{
-  ...
-  "scripts": {
-    ...
-    "vercel-build": "sed -i 's/nativeLib = require(`@lancedb\\/vectordb-\\${currentTarget()}`);/nativeLib = require(`@lancedb\\/vectordb-linux-x64-gnu`);/' node_modules/vectordb/native.js && next build",
-    ...
-  },
-  ...
-}
-```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -1,121 +0,0 @@
-# Vector embedding search using TransformersJS
-
-## Embed and query data from LanceDB using TransformersJS
-
-<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
-
-This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
-
-
-### Setting up
-First, install the dependencies:
-```bash
-npm install vectordb
-npm i @xenova/transformers
-```
-
-We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
-
-Within our `index.js` file we will import the necessary libraries and define our model and database:
-
-```javascript
-const lancedb = require('vectordb')
-const { pipeline } = await import('@xenova/transformers')
-const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-```
-
-### Creating the embedding function
-
-Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
-
-```javascript
-// Define the function. `sourceColumn` is required for LanceDB to know
-// which column to use as input.
-const embed_fun = {}
-embed_fun.sourceColumn = 'text'
-embed_fun.embed = async function (batch) {
-    let result = []
-    // Given a batch of strings, we will use the `pipe` function to get
-    // the vector embedding of each string.
-    for (let text of batch) {
-        // 'mean' pooling and normalizing allows the embeddings to share the
-        // same length.
-        const res = await pipe(text, { pooling: 'mean', normalize: true })
-        result.push(Array.from(res['data']))
-    }
-    return (result)
-}
-```
-
-### Creating the database
-
-Now, we will create the LanceDB database and add the embedding function we defined earlier.
-
-```javascript
-// Link a folder and create a table with data
-const db = await lancedb.connect('data/sample-lancedb')
-
-// You can also import any other data, but make sure that you have a column
-// for the embedding function to use.
-const data = [
-    { id: 1, text: 'Cherry', type: 'fruit' },
-    { id: 2, text: 'Carrot', type: 'vegetable' },
-    { id: 3, text: 'Potato', type: 'vegetable' },
-    { id: 4, text: 'Apple', type: 'fruit' },
-    { id: 5, text: 'Banana', type: 'fruit' }
-]
-
-// Create the table with the embedding function
-const table = await db.createTable('food_table', data, "create", embed_fun)
-```
-
-### Performing the search
-
-Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
-
-```javascript
-// Query the table
-const results = await table
-    .search("a sweet fruit to eat")
-    .metricType("cosine")
-    .limit(2)
-    .execute()
-console.log(results.map(r => r.text))
-```
-```bash
-[ 'Banana', 'Cherry' ]
-```
-
-Output of `results`:
-```bash
-[
-  {
-    vector: Float32Array(384) [
-      -0.057455405592918396,
-      0.03617725893855095,
-      -0.0367760956287384,
-      ... 381 more items
-    ],
-    id: 5,
-    text: 'Banana',
-    type: 'fruit',
-    _distance: 0.4919965863227844
-  },
-  {
-    vector: Float32Array(384) [
-      0.0009714411571621895,
-      0.008223623037338257,
-      0.009571489877998829,
-      ... 381 more items
-    ],
-    id: 1,
-    text: 'Cherry',
-    type: 'fruit',
-    _distance: 0.5540297031402588
-  }
-]
-```
-
-### Wrapping it up
-
-In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!