Bump version: 0.26.0-beta.2 → 0.26.0

Includes fix https://github.com/lance-format/lance-namespace/pull/281

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.22.2-beta.2"
+current_version = "0.23.0-beta.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -72,3 +72,9 @@ search = "\nversion = \"{current_version}\""
 filename = "nodejs/Cargo.toml"
 replace = "\nversion = \"{new_version}\""
 search = "\nversion = \"{current_version}\""
+
+# Java documentation
+[[tool.bumpversion.files]]
+filename = "docs/src/java/java.md"
+replace = "<version>{new_version}</version>"
+search = "<version>{current_version}</version>"

.cargo/config.toml

@@ -19,7 +19,7 @@ rustflags = [
     "-Wclippy::string_add_assign",
     "-Wclippy::string_add",
     "-Wclippy::string_lit_as_bytes",
-    "-Wclippy::string_to_string",
+    "-Wclippy::implicit_clone",
     "-Wclippy::use_self",
     "-Dclippy::cargo",
     "-Dclippy::dbg_macro",

.github/ISSUE_TEMPLATE/documentation.yml

@@ -18,6 +18,6 @@ body:
       label: Link
       description: >
         Provide a link to the existing documentation, if applicable.
-      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
+      placeholder: ex. https://lancedb.com/docs/tables/...
     validations:
       required: false

.github/workflows/build_linux_wheel/action.yml

@@ -31,6 +31,7 @@ runs:
       with:
         command: build
         working-directory: python
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         target: x86_64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
@@ -45,7 +46,7 @@ runs:
       with:
         command: build
         working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         target: aarch64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}

.github/workflows/build_mac_wheel/action.yml

@@ -22,5 +22,5 @@ runs:
         command: build
         # TODO: pass through interpreter
         args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         working-directory: python

.github/workflows/build_windows_wheel/action.yml

@@ -26,7 +26,7 @@ runs:
       with:
         command: build
         args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         working-directory: python
     - uses: actions/upload-artifact@v4
       with:

.github/workflows/codex-update-lance-dependency.yml

@@ -0,0 +1,127 @@
+name: Codex Update Lance Dependency
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        description: "Tag name from Lance"
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag name from Lance"
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: read
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Show inputs
+        run: |
+          echo "tag = ${{ inputs.tag }}"
+
+      - name: Checkout Repo LanceDB
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: true
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install Codex CLI
+        run: npm install -g @openai/codex
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          components: clippy, rustfmt
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler libssl-dev
+
+      - name: Install cargo-info
+        run: cargo install cargo-info
+
+      - name: Install Python dependencies
+        run: python3 -m pip install --upgrade pip packaging
+
+      - name: Configure git user
+        run: |
+          git config user.name "lancedb automation"
+          git config user.email "robot@lancedb.com"
+
+      - name: Run Codex to update Lance dependency
+        env:
+          TAG: ${{ inputs.tag }}
+          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
+        run: |
+          set -euo pipefail
+          VERSION="${TAG#refs/tags/}"
+          VERSION="${VERSION#v}"
+          BRANCH_NAME="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
+
+          cat <<EOF >/tmp/codex-prompt.txt
+          You are running inside the lancedb repository on a GitHub Actions runner. Update the Lance dependency to version ${VERSION} and prepare a pull request for maintainers to review.
+
+          Follow these steps exactly:
+          1. Use script "ci/set_lance_version.py" to update Lance dependencies. The script already refreshes Cargo metadata, so allow it to finish even if it takes time.
+          2. Run "cargo clippy --workspace --tests --all-features -- -D warnings". If diagnostics appear, fix them yourself and rerun clippy until it exits cleanly. Do not skip any warnings.
+          3. After clippy succeeds, run "cargo fmt --all" to format the workspace.
+          4. Ensure the repository is clean except for intentional changes. Inspect "git status --short" and "git diff" to confirm the dependency update and any required fixes.
+          5. Create and switch to a new branch named "${BRANCH_NAME}" (replace any duplicated hyphens if necessary).
+          6. Stage all relevant files with "git add -A". Commit using the message "chore: update lance dependency to v${VERSION}".
+          7. Push the branch to origin. If the branch already exists, force-push your changes.
+          8. env "GH_TOKEN" is available, use "gh" tools for github related operations like creating pull request.
+          9. Create a pull request targeting "main" with title "chore: update lance dependency to v${VERSION}". In the body, summarize the dependency bump, clippy/fmt verification, and link the triggering tag (${TAG}).
+          10. After creating the PR, display the PR URL, "git status --short", and a concise summary of the commands run and their results.
+
+          Constraints:
+          - Use bash commands; avoid modifying GitHub workflow files other than through the scripted task above.
+          - Do not merge the PR.
+          - If any command fails, diagnose and fix the issue instead of aborting.
+          EOF
+
+          printenv OPENAI_API_KEY | codex login --with-api-key
+          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"
+
+      - name: Trigger sophon dependency update
+        env:
+          TAG: ${{ inputs.tag }}
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          VERSION="${TAG#refs/tags/}"
+          VERSION="${VERSION#v}"
+          LANCEDB_BRANCH="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
+
+          echo "Triggering sophon workflow with:"
+          echo "  lance_ref: ${TAG#refs/tags/}"
+          echo "  lancedb_ref: ${LANCEDB_BRANCH}"
+
+          gh workflow run codex-bump-lancedb-lance.yml \
+            --repo lancedb/sophon \
+            -f lance_ref="${TAG#refs/tags/}" \
+            -f lancedb_ref="${LANCEDB_BRANCH}"
+
+      - name: Show latest sophon workflow run
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          echo "Latest sophon workflow run:"
+          gh run list --repo lancedb/sophon --workflow codex-bump-lancedb-lance.yml --limit 1 --json databaseId,url,displayTitle

.github/workflows/docs.yml

@@ -24,6 +24,7 @@ env:
   # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
   # CI builds are faster with incremental disabled.
   CARGO_INCREMENTAL: "0"
+  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
 
 jobs:
   # Single deploy job since we're just deploying
@@ -49,8 +50,8 @@ jobs:
       - name: Build Python
         working-directory: python
         run: |
-          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .
-          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
+          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .
+          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
       - name: Set up node
         uses: actions/setup-node@v3
         with:

.github/workflows/docs_test.yml

@@ -1,61 +0,0 @@
-name: Documentation Code Testing
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-  pull_request:
-    paths:
-      - docs/**
-      - .github/workflows/docs_test.yml
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
-  RUST_BACKTRACE: "1"
-
-jobs:
-  test-python:
-    name: Test doc python code
-    runs-on: warp-ubuntu-2204-x64-8x
-    timeout-minutes: 60
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
-    - name: Install protobuf
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler
-    - name: Install dependecies needed for ubuntu
-      run: |
-        sudo apt install -y libssl-dev
-        rustup update && rustup default
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.11
-        cache: "pip"
-        cache-dependency-path: "docs/test/requirements.txt"
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
-    - name: Build Python
-      working-directory: docs/test
-      run:
-        python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r requirements.txt
-    - name: Create test files
-      run: |
-        cd docs/test
-        python md_testing.py
-    - name: Test
-      run: |
-        cd docs/test/python
-        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done

.github/workflows/java-publish.yml

@@ -1,76 +1,35 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: Build and publish Java packages
 on:
-  release:
-    types: [released]
+  push:
+    tags:
+      - "v*"
   pull_request:
     paths:
       - .github/workflows/java-publish.yml
 
 jobs:
-  macos-arm64:
-    name: Build on MacOS Arm64
-    runs-on: macos-14
-    timeout-minutes: 45
-    defaults:
-      run:
-        working-directory: ./java/core/lancedb-jni
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-      - name: Install dependencies
-        run: |
-          brew install protobuf
-      - name: Build release
-        run: |
-          cargo build --release
-      - uses: actions/upload-artifact@v4
-        with:
-          name: liblancedb_jni_darwin_aarch64.zip
-          path: target/release/liblancedb_jni.dylib
-          retention-days: 1
-          if-no-files-found: error
-  linux-arm64:
-    name: Build on Linux Arm64
-    runs-on: warp-ubuntu-2204-arm64-8x
-    timeout-minutes: 45
-    defaults:
-      run:
-        working-directory: ./java/core/lancedb-jni
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          cache-workspaces: "./java/core/lancedb-jni"
-          # Disable full debug symbol generation to speed up CI build and keep memory down
-          # "1" means line tables only, which is useful for panic tracebacks.
-          rustflags: "-C debuginfo=1"
-      - name: Install dependencies
-        run: |
-          sudo apt -y -qq update
-          sudo apt install -y protobuf-compiler libssl-dev pkg-config
-      - name: Build release
-        run: |
-          cargo build --release
-      - uses: actions/upload-artifact@v4
-        with:
-          name: liblancedb_jni_linux_aarch64.zip
-          path: target/release/liblancedb_jni.so
-          retention-days: 1
-          if-no-files-found: error
-  linux-x86:
-    runs-on: warp-ubuntu-2204-x64-8x
+  publish:
+    name: Build and Publish
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
-    needs: [macos-arm64, linux-arm64]
     defaults:
       run:
         working-directory: ./java
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
       - name: Set up Java 8
         uses: actions/setup-java@v4
         with:
@@ -82,40 +41,30 @@ jobs:
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
-      - name: Install dependencies
+      - name: Set git config
         run: |
-          sudo apt -y -qq update
-          sudo apt install -y protobuf-compiler libssl-dev pkg-config
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-      - name: Copy native libs
-        run: |
-          mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64
-          cp ../liblancedb_jni_darwin_aarch64.zip/liblancedb_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblancedb_jni.dylib
-          cp ../liblancedb_jni_linux_aarch64.zip/liblancedb_jni.so ./core/target/classes/nativelib/linux-aarch64/liblancedb_jni.so
+          git config --global user.email "dev+gha@lancedb.com"
+          git config --global user.name "LanceDB Github Runner"
       - name: Dry run
         if: github.event_name == 'pull_request'
         run: |
-          mvn --batch-mode -DskipTests -Drust.release.build=true package
-      - name: Set github
-        run: |
-          git config --global user.email "LanceDB Github Runner"
-          git config --global user.name "dev+gha@lancedb.com"
-      - name: Publish with Java 8
-        if: github.event_name == 'release'
+          ./mvnw --batch-mode -DskipTests package -pl lancedb-core -am
+      - name: Publish
+        if: startsWith(github.ref, 'refs/tags/v')
         run: |
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
+          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
+
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
-    needs: [linux-arm64, linux-x86, macos-arm64]
-    if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
+    needs: [publish]
+    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
     permissions:
       contents: read
       issues: write

.github/workflows/java.yml

@@ -1,118 +1,46 @@
-name: Build and Run Java JNI Tests
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build Java LanceDB Core
+
 on:
   push:
     branches:
       - main
     paths:
       - java/**
+      - .github/workflows/java.yml
   pull_request:
     paths:
       - java/**
-      - rust/**
       - .github/workflows/java.yml
-env:
-  # This env var is used by Swatinem/rust-cache@v2 for the cache
-  # key, so we set it to make sure it is always consistent.
-  CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
-  RUST_BACKTRACE: "1"
-  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
-  # CI builds are faster with incremental disabled.
-  CARGO_INCREMENTAL: "0"
-  CARGO_BUILD_JOBS: "1"
+
 jobs:
-  linux-build-java-11:
-    runs-on: ubuntu-22.04
-    name: ubuntu-22.04 + Java 11
+  build-java:
+    runs-on: ubuntu-24.04
+    name: Build
     defaults:
       run:
         working-directory: ./java
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: java/core/lancedb-jni
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          components: rustfmt
-      - name: Run cargo fmt
-        run: cargo fmt --check
-        working-directory: ./java/core/lancedb-jni
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install Java 11
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 11
-          cache: "maven"
-      - name: Java Style Check
-        run: mvn checkstyle:check
-      # Disable because of issues in lancedb rust core code
-      # - name: Rust Clippy
-      #   working-directory: java/core/lancedb-jni
-      #   run: cargo clippy --all-targets -- -D warnings
-      - name: Running tests with Java 11
-        run: mvn clean test
-  linux-build-java-17:
-    runs-on: ubuntu-22.04
-    name: ubuntu-22.04 + Java 17
-    defaults:
-      run:
-        working-directory: ./java
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: java/core/lancedb-jni
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          components: rustfmt
-      - name: Run cargo fmt
-        run: cargo fmt --check
-        working-directory: ./java/core/lancedb-jni
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install Java 17
+      - name: Set up Java 17
         uses: actions/setup-java@v4
         with:
           distribution: temurin
           java-version: 17
           cache: "maven"
-      - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
       - name: Java Style Check
-        run: mvn checkstyle:check
-      # Disable because of issues in lancedb rust core code
-      # - name: Rust Clippy
-      #   working-directory: java/core/lancedb-jni
-      #   run: cargo clippy --all-targets -- -D warnings
-      - name: Running tests with Java 17
-        run: |
-          export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
-          -XX:+IgnoreUnrecognizedVMOptions \
-          --add-opens=java.base/java.lang=ALL-UNNAMED \
-          --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
-          --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
-          --add-opens=java.base/java.io=ALL-UNNAMED \
-          --add-opens=java.base/java.net=ALL-UNNAMED \
-          --add-opens=java.base/java.nio=ALL-UNNAMED \
-          --add-opens=java.base/java.util=ALL-UNNAMED \
-          --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
-          --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
-          --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
-          --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
-          --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
-          --add-opens=java.base/sun.security.action=ALL-UNNAMED \
-          --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
-          --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
-          -Djdk.reflect.useDirectMethodHandle=false \
-          -Dio.netty.tryReflectionSetAccessible=true"
-          JAVA_HOME=$JAVA_17 mvn clean test
+        run: ./mvnw checkstyle:check
+      - name: Build and install
+        run: ./mvnw clean install

.github/workflows/lance-release-timer.yml

@@ -0,0 +1,62 @@
+name: Lance Release Timer
+
+on:
+  schedule:
+    - cron: "*/10 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: write
+
+concurrency:
+  group: lance-release-timer
+  cancel-in-progress: false
+
+jobs:
+  trigger-update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check for new Lance tag
+        id: check
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          python3 ci/check_lance_release.py --github-output "$GITHUB_OUTPUT"
+
+      - name: Look for existing PR
+        if: steps.check.outputs.needs_update == 'true'
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          TITLE="chore: update lance dependency to v${{ steps.check.outputs.latest_version }}"
+          COUNT=$(gh pr list --search "\"$TITLE\" in:title" --state open --limit 1 --json number --jq 'length')
+          if [ "$COUNT" -gt 0 ]; then
+            echo "Open PR already exists for $TITLE"
+            echo "pr_exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No existing PR for $TITLE"
+            echo "pr_exists=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Trigger codex update workflow
+        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          TAG=${{ steps.check.outputs.latest_tag }}
+          gh workflow run codex-update-lance-dependency.yml -f tag=refs/tags/$TAG
+
+      - name: Show latest codex workflow run
+        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
+        run: |
+          set -euo pipefail
+          gh run list --workflow codex-update-lance-dependency.yml --limit 1 --json databaseId,url,displayTitle

.github/workflows/nodejs.yml

@@ -16,9 +16,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
 
 jobs:
@@ -43,18 +40,20 @@ jobs:
         node-version: 20
         cache: 'npm'
         cache-dependency-path: nodejs/package-lock.json
-    - uses: Swatinem/rust-cache@v2
+    - uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        components: rustfmt, clippy
     - name: Install dependencies
       run: |
         sudo apt update
         sudo apt install -y protobuf-compiler libssl-dev
-    - uses: actions-rust-lang/setup-rust-toolchain@v1
-      with:
-        components: rustfmt, clippy
-    - name: Lint
+    - uses: Swatinem/rust-cache@v2
+    - name: Format Rust
+      run: cargo fmt --all -- --check
+    - name: Lint Rust
+      run: cargo clippy --profile ci --all --all-features -- -D warnings
+    - name: Lint Typescript
       run: |
-        cargo fmt --all -- --check
-        cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint-ci
     - name: Lint examples
@@ -89,8 +88,9 @@ jobs:
         npm install -g @napi-rs/cli
     - name: Build
       run: |
-        npm ci
-        npm run build
+        npm ci --include=optional
+        npm run build:debug -- --profile ci
+        npm run tsc
     - name: Setup localstack
       working-directory: .
       run: docker compose up --detach --wait
@@ -146,8 +146,9 @@ jobs:
         npm install -g @napi-rs/cli
     - name: Build
       run: |
-        npm ci
-        npm run build
+        npm ci --include=optional
+        npm run build:debug -- --profile ci
+        npm run tsc
     - name: Test
       run: |
         npm run test

.github/workflows/npm-publish.yml

@@ -97,12 +97,6 @@ jobs:
       fail-fast: false
       matrix:
         settings:
-          - target: x86_64-apple-darwin
-            host: macos-latest
-            features: ","
-            pre_build: |-
-                brew install protobuf
-                rustup target add x86_64-apple-darwin
           - target: aarch64-apple-darwin
             host: macos-latest
             features: fp16kernels

.github/workflows/pypi-publish.yml

@@ -10,6 +10,9 @@ on:
       - .github/workflows/pypi-publish.yml
       - Cargo.toml # Change in dependency frequently breaks builds
 
+env:
+  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
+
 jobs:
   linux:
     name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
@@ -61,8 +64,6 @@ jobs:
     strategy:
       matrix:
         config:
-          - target: x86_64-apple-darwin
-            runner: macos-13
           - target: aarch64-apple-darwin
             runner: warp-macos-14-arm64-6x
     env:

.github/workflows/python.yml

@@ -18,6 +18,8 @@ env:
   # Color output for pytest is off by default.
   PYTEST_ADDOPTS: "--color=yes"
   FORCE_COLOR: "1"
+  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
+  RUST_BACKTRACE: "1"
 
 jobs:
   lint:
@@ -47,8 +49,8 @@ jobs:
 
   type-check:
     name: "Type Check"
-    timeout-minutes: 30
-    runs-on: "ubuntu-22.04"
+    timeout-minutes: 60
+    runs-on: ubuntu-2404-8x-x64
     defaults:
       run:
         shell: bash
@@ -76,8 +78,8 @@ jobs:
 
   doctest:
     name: "Doctest"
-    timeout-minutes: 30
-    runs-on: "ubuntu-24.04"
+    timeout-minutes: 60
+    runs-on: ubuntu-2404-8x-x64
     defaults:
       run:
         shell: bash
@@ -96,12 +98,9 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
       - name: Install
         run: |
-          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
+          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
           pip install tantivy
           pip install mlx
       - name: Doctest
@@ -130,10 +129,9 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.${{ matrix.python-minor-version }}
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
       - uses: ./.github/workflows/build_linux_wheel
+        with:
+          args: --profile ci
       - uses: ./.github/workflows/run_tests
         with:
           integration: true
@@ -145,16 +143,9 @@ jobs:
       - name: Delete wheels
         run: rm -rf target/wheels
   platform:
-    name: "Mac: ${{ matrix.config.name }}"
+    name: "Mac"
     timeout-minutes: 30
-    strategy:
-      matrix:
-        config:
-          - name: x86
-            runner: macos-13
-          - name: Arm
-            runner: macos-14
-    runs-on: "${{ matrix.config.runner }}"
+    runs-on: macos-14
     defaults:
       run:
         shell: bash
@@ -168,10 +159,9 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
       - uses: ./.github/workflows/build_mac_wheel
+        with:
+          args: --profile ci
       - uses: ./.github/workflows/run_tests
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
@@ -198,10 +188,9 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
       - uses: ./.github/workflows/build_windows_wheel
+        with:
+          args: --profile ci
       - uses: ./.github/workflows/run_tests
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
@@ -230,7 +219,7 @@ jobs:
         run: |
           pip install "pydantic<2"
           pip install pyarrow==16
-          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
+          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests
         run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -15,7 +15,7 @@ runs:
     - name: Install lancedb
       shell: bash
       run: |
-        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
+        pip3 install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
     - name: Setup localstack for integration tests
       if: ${{ inputs.integration == 'true' }}
       shell: bash

.github/workflows/rust.yml

@@ -18,11 +18,7 @@ env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
-  CARGO_INCREMENTAL: 0
 
 jobs:
   lint:
@@ -44,8 +40,6 @@ jobs:
         with:
           components: rustfmt, clippy
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install dependencies
         run: |
           sudo apt update
@@ -53,7 +47,7 @@ jobs:
       - name: Run format
         run: cargo fmt --all -- --check
       - name: Run clippy
-        run: cargo clippy --workspace --tests --all-features -- -D warnings
+        run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
 
   build-no-lock:
     runs-on: ubuntu-24.04
@@ -80,7 +74,7 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Build all
         run: |
-          cargo build --benches --all-features --tests
+          cargo build --profile ci --benches --all-features --tests
 
   linux:
     timeout-minutes: 30
@@ -103,14 +97,8 @@ jobs:
           fetch-depth: 0
           lfs: true
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install dependencies
-        run: |
-          # This shaves 2 minutes off this step in CI. This doesn't seem to be
-          # necessary in standard runners, but it is in the 4x runners.
-          sudo rm /var/lib/man-db/auto-update
-          sudo apt install -y protobuf-compiler libssl-dev
+        run: sudo apt install -y protobuf-compiler libssl-dev
       - uses: rui314/setup-mold@v1
       - name: Make Swap
         run: |
@@ -119,22 +107,22 @@ jobs:
           sudo mkswap /swapfile
           sudo swapon /swapfile
       - name: Build
-        run: cargo build --all-features --tests --locked --examples
+        run: cargo build --profile ci --all-features --tests --locked --examples
       - name: Run feature tests
-        run: make -C ./lancedb feature-tests
+        run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
       - name: Run examples
-        run: cargo run --example simple --locked
+        run: cargo run  --profile ci --example simple --locked
       - name: Run remote tests
         # Running this requires access to secrets, so skip if this is
         # a PR from a fork.
         if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
-        run: make -C ./lancedb remote-tests
+        run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
 
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: ["macos-13", "macos-14"]
+        mac-runner: ["macos-14", "macos-15"]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
@@ -148,8 +136,6 @@ jobs:
       - name: CPU features
         run: sysctl -a | grep cpu
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install dependencies
         run: brew install protobuf
       - name: Run tests
@@ -159,7 +145,7 @@ jobs:
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
             | jq -r '.packages[] | .features | keys | .[]' \
             | grep -v s3-test | sort | uniq | paste -s -d "," -`
-          cargo test --features $ALL_FEATURES --locked
+          cargo test --profile ci --features $ALL_FEATURES --locked
 
   windows:
     runs-on: windows-2022
@@ -173,22 +159,21 @@ jobs:
         working-directory: rust/lancedb
     steps:
       - uses: actions/checkout@v4
+      - name: Set target
+        run: rustup target add ${{ matrix.target }}
       - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install Protoc v21.12
         run: choco install --no-progress protoc
       - name: Build
         run: |
-          rustup target add ${{ matrix.target }}
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build --features remote --tests --locked --target ${{ matrix.target }}
+          cargo build --profile ci --features remote --tests --locked --target ${{ matrix.target }}
       - name: Run tests
         # Can only run tests when target matches host
         if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --features remote --locked
+          cargo test --profile ci --features remote --locked
 
   msrv:
     # Check the minimum supported Rust version
@@ -213,6 +198,7 @@ jobs:
         uses: dtolnay/rust-toolchain@master
         with:
           toolchain: ${{ matrix.msrv }}
+      - uses: Swatinem/rust-cache@v2
       - name: Downgrade  dependencies
         # These packages have newer requirements for MSRV
         run: |
@@ -226,4 +212,4 @@ jobs:
           cargo update -p aws-sdk-sts --precise 1.51.0
           cargo update -p home --precise 0.5.9
       - name: cargo +${{ matrix.msrv }} check
-        run: cargo check --workspace --tests --benches --all-features
+        run: cargo check --profile ci --workspace --tests --benches --all-features

.gitignore

@@ -1,4 +1,5 @@
 .idea
+*.swp
 **/*.whl
 *.egg-info
 **/__pycache__

AGENTS.md

@@ -0,0 +1,101 @@
+LanceDB is a database designed for retrieval, including vector, full-text, and hybrid search.
+It is a wrapper around Lance. There are two backends: local (in-process like SQLite) and
+remote (against LanceDB Cloud).
+
+The core of LanceDB is written in Rust. There are bindings in Python, Typescript, and Java.
+
+Project layout:
+
+* `rust/lancedb`: The LanceDB core Rust implementation.
+* `python`: The Python bindings, using PyO3.
+* `nodejs`: The Typescript bindings, using napi-rs
+* `java`: The Java bindings
+
+Common commands:
+
+* Check for compiler errors: `cargo check --quiet --features remote --tests --examples`
+* Run tests: `cargo test --quiet --features remote --tests`
+* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
+* Lint: `cargo clippy --quiet --features remote --tests --examples`
+* Format: `cargo fmt --all`
+
+Before committing changes, run formatting.
+
+## Coding tips
+
+* When writing Rust doctests for things that require a connection or table reference,
+  write them as a function instead of a fully executable test. This allows type checking
+  to run but avoids needing a full test environment. For example:
+    ```rust
+    /// ```
+    /// use lance_index::scalar::FullTextSearchQuery;
+    /// use lancedb::query::{QueryBase, ExecutableQuery};
+    ///
+    /// # use lancedb::Table;
+    /// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let results = table.query()
+    ///     .full_text_search(FullTextSearchQuery::new("hello world".into()))
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ```
+
+## Example plan: adding a new method on Table
+
+Adding a new method involves first adding it to the Rust core, then exposing it
+in the Python and TypeScript bindings. There are both local and remote tables.
+Remote tables are implemented via a HTTP API and require the `remote` cargo
+feature flag to be enabled. Python has both sync and async methods.
+
+Rust core changes:
+
+1. Add method on `Table` struct in `rust/lancedb/src/table.rs` (calls `BaseTable` trait).
+2. Add method to `BaseTable` trait in `rust/lancedb/src/table.rs`.
+3. Implement new trait method on `NativeTable` in `rust/lancedb/src/table.rs`.
+    * Test with unit test in `rust/lancedb/src/table.rs`.
+4. Implement new trait method on `RemoteTable` in `rust/lancedb/src/remote/table.rs`.
+    * Test with unit test in `rust/lancedb/src/remote/table.rs` against mocked endpoint.
+
+Python bindings changes:
+
+1. Add PyO3 method binding in `python/src/table.rs`. Run `make develop` to compile bindings.
+2. Add types for PyO3 method in `python/python/lancedb/_lancedb.pyi`.
+3. Add method to `AsyncTable` class in `python/python/lancedb/table.py`.
+4. Add abstract method to `Table` abstract base class in `python/python/lancedb/table.py`.
+5. Add concrete sync method to `LanceTable` class in `python/python/lancedb/table.py`.
+    * Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
+6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
+7. Add unit test in `python/tests/test_table.py`.
+
+TypeScript bindings changes:
+
+1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
+2. Run `npm run build` to generate TypeScript definitions.
+3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
+4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
+    * Note: despite the name, this class is also used for remote tables.
+5. Add test in `nodejs/__test__/table.test.ts`.
+6. Run `npm run docs` to generate TypeScript documentation.
+
+## Review Guidelines
+
+Please consider the following when reviewing code contributions.
+
+### Rust API design
+* Design public APIs so they can be evolved easily in the future without breaking
+  changes. Often this means using builder patterns or options structs instead of
+  long argument lists.
+* For public APIs, prefer inputs that use `Into<T>` or `AsRef<T>` traits to allow
+  more flexible inputs. For example, use `name: Into<String>` instead of `name: String`,
+  so we don't have to write `func("my_string".to_string())`.
+
+### Testing
+* Ensure all new public APIs have documentation and examples.
+* Ensure that all bugfixes and features have corresponding tests. **We do not merge
+  code without tests.**
+
+### Documentation
+* New features must include updates to the rust documentation comments. Link to
+  relevant structs and methods to increase the value of documentation.

CLAUDE.md

@@ -1,80 +0,0 @@
-LanceDB is a database designed for retrieval, including vector, full-text, and hybrid search.
-It is a wrapper around Lance. There are two backends: local (in-process like SQLite) and
-remote (against LanceDB Cloud).
-
-The core of LanceDB is written in Rust. There are bindings in Python, Typescript, and Java.
-
-Project layout:
-
-* `rust/lancedb`: The LanceDB core Rust implementation.
-* `python`: The Python bindings, using PyO3.
-* `nodejs`: The Typescript bindings, using napi-rs
-* `java`: The Java bindings
-
-Common commands:
-
-* Check for compiler errors: `cargo check --quiet --features remote --tests --examples`
-* Run tests: `cargo test --quiet --features remote --tests`
-* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
-* Lint: `cargo clippy --quiet --features remote --tests --examples`
-* Format: `cargo fmt --all`
-
-Before committing changes, run formatting.
-
-## Coding tips
-
-* When writing Rust doctests for things that require a connection or table reference,
-  write them as a function instead of a fully executable test. This allows type checking
-  to run but avoids needing a full test environment. For example:
-    ```rust
-    /// ```
-    /// use lance_index::scalar::FullTextSearchQuery;
-    /// use lancedb::query::{QueryBase, ExecutableQuery};
-    ///
-    /// # use lancedb::Table;
-    /// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
-    /// let results = table.query()
-    ///     .full_text_search(FullTextSearchQuery::new("hello world".into()))
-    ///     .execute()
-    ///     .await?;
-    /// # Ok(())
-    /// # }
-    /// ```
-    ```
-
-## Example plan: adding a new method on Table
-
-Adding a new method involves first adding it to the Rust core, then exposing it
-in the Python and TypeScript bindings. There are both local and remote tables.
-Remote tables are implemented via a HTTP API and require the `remote` cargo
-feature flag to be enabled. Python has both sync and async methods.
-
-Rust core changes:
-
-1. Add method on `Table` struct in `rust/lancedb/src/table.rs` (calls `BaseTable` trait).
-2. Add method to `BaseTable` trait in `rust/lancedb/src/table.rs`.
-3. Implement new trait method on `NativeTable` in `rust/lancedb/src/table.rs`.
-    * Test with unit test in `rust/lancedb/src/table.rs`.
-4. Implement new trait method on `RemoteTable` in `rust/lancedb/src/remote/table.rs`.
-    * Test with unit test in `rust/lancedb/src/remote/table.rs` against mocked endpoint.
-
-Python bindings changes:
-
-1. Add PyO3 method binding in `python/src/table.rs`. Run `make develop` to compile bindings.
-2. Add types for PyO3 method in `python/python/lancedb/_lancedb.pyi`.
-3. Add method to `AsyncTable` class in `python/python/lancedb/table.py`.
-4. Add abstract method to `Table` abstract base class in `python/python/lancedb/table.py`.
-5. Add concrete sync method to `LanceTable` class in `python/python/lancedb/table.py`.
-    * Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
-6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
-7. Add unit test in `python/tests/test_table.py`.
-
-TypeScript bindings changes:
-
-1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
-2. Run `npm run build` to generate TypeScript definitions.
-3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
-4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
-    * Note: despite the name, this class is also used for remote tables.
-5. Add test in `nodejs/__test__/table.test.ts`.
-6. Run `npm run docs` to generate TypeScript documentation.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["rust/lancedb", "nodejs", "python", "java/core/lancedb-jni"]
+members = ["rust/lancedb", "nodejs", "python"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -15,15 +15,21 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.38.2", default-features = false, "features" = ["dynamodb"] }
-lance-io = { "version" = "=0.38.2", default-features = false }
-lance-index = "=0.38.2"
-lance-linalg = "=0.38.2"
-lance-table = "=0.38.2"
-lance-testing = "=0.38.2"
-lance-datafusion = "=0.38.2"
-lance-encoding = "=0.38.2"
-lance-namespace = "0.0.18"
+lance = { "version" = "=1.0.0", default-features = false, "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=1.0.0", default-features = false, "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=1.0.0", default-features = false, "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=1.0.0", "tag" = "v1.0.0", "git" = "https://github.com/lance-format/lance.git" }
+ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "56.2", optional = false }
 arrow-array = "56.2"
@@ -31,6 +37,7 @@ arrow-data = "56.2"
 arrow-ipc = "56.2"
 arrow-ord = "56.2"
 arrow-schema = "56.2"
+arrow-select = "56.2"
 arrow-cast = "56.2"
 async-trait = "0"
 datafusion = { version = "50.1", default-features = false }
@@ -48,27 +55,25 @@ log = "0.4"
 moka = { version = "0.12", features = ["future"] }
 object_store = "0.12.0"
 pin-project = "1.0.7"
+rand = "0.9"
 snafu = "0.8"
 url = "2"
 num-traits = "0.2"
 regex = "1.10"
 lazy_static = "1"
 semver = "1.0.25"
-crunchy = "0.2.4"
-# Temporary pins to work around downstream issues
-# https://github.com/apache/arrow-rs/commit/2fddf85afcd20110ce783ed5b4cdeb82293da30b
-chrono = "=0.4.41"
-# Workaround for: https://github.com/Lokathor/bytemuck/issues/306
-bytemuck_derive = ">=1.8.1, <1.9.0"
+chrono = "0.4"
 
-# This is only needed when we reference preview releases of lance
-# [patch.crates-io]
-# # Force to use the same lance version as the rest of the project to avoid duplicate dependencies
-# lance = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-io = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-index = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-linalg = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-table = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-testing = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-datafusion = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
-# lance-encoding = { "version" = "=0.38.0", "tag" = "v0.38.0", "git" = "https://github.com/lancedb/lance.git" }
+[profile.ci]
+debug = "line-tables-only"
+inherits = "dev"
+incremental = false
+
+# This rule applies to every package except workspace members (dependencies
+# such as `arrow` and `tokio`). It disables debug info and related features on
+# dependencies so their binaries stay smaller, improving cache reuse.
+[profile.ci.package."*"]
+debug = false
+debug-assertions = false
+strip = "debuginfo"
+incremental = false

README.md

@@ -15,7 +15,7 @@
 
 # **The Multimodal AI Lakehouse**
 
-[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.github.io/lancedb/) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
+[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
 
 **The ultimate multimodal data platform for AI/ML applications.** 
 
@@ -57,7 +57,7 @@ LanceDB is a central location where developers can build, train and analyze thei
 
 ## **How to Install**:
 
-Follow the [Quickstart](https://lancedb.github.io/lancedb/basic/) doc to set up LanceDB locally. 
+Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally. 
 
 **API & SDK:** We also support Python, Typescript and Rust SDKs
 

ci/check_lance_release.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Determine whether a newer Lance tag exists and expose results for CI."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, List, Sequence, Tuple, Union
+
+try:  # Python >=3.11
+    import tomllib  # type: ignore
+except ModuleNotFoundError:  # pragma: no cover - fallback for older Python
+    import tomli as tomllib  # type: ignore
+
+LANCE_REPO = "lance-format/lance"
+
+SEMVER_RE = re.compile(
+    r"^\s*(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
+    r"(?:-(?P<prerelease>[0-9A-Za-z.-]+))?"
+    r"(?:\+[0-9A-Za-z.-]+)?\s*$"
+)
+
+
+@dataclass(frozen=True)
+class SemVer:
+    major: int
+    minor: int
+    patch: int
+    prerelease: Tuple[Union[int, str], ...]
+
+    def __lt__(self, other: "SemVer") -> bool:  # pragma: no cover - simple comparison
+        if (self.major, self.minor, self.patch) != (other.major, other.minor, other.patch):
+            return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
+        if self.prerelease == other.prerelease:
+            return False
+        if not self.prerelease:
+            return False  # release > anything else
+        if not other.prerelease:
+            return True
+        for left, right in zip(self.prerelease, other.prerelease):
+            if left == right:
+                continue
+            if isinstance(left, int) and isinstance(right, int):
+                return left < right
+            if isinstance(left, int):
+                return True
+            if isinstance(right, int):
+                return False
+            return str(left) < str(right)
+        return len(self.prerelease) < len(other.prerelease)
+
+    def __eq__(self, other: object) -> bool:  # pragma: no cover - trivial
+        if not isinstance(other, SemVer):
+            return NotImplemented
+        return (
+            self.major == other.major
+            and self.minor == other.minor
+            and self.patch == other.patch
+            and self.prerelease == other.prerelease
+        )
+
+
+def parse_semver(raw: str) -> SemVer:
+    match = SEMVER_RE.match(raw)
+    if not match:
+        raise ValueError(f"Unsupported version format: {raw}")
+    prerelease = match.group("prerelease")
+    parts: Tuple[Union[int, str], ...] = ()
+    if prerelease:
+        parsed: List[Union[int, str]] = []
+        for piece in prerelease.split("."):
+            if piece.isdigit():
+                parsed.append(int(piece))
+            else:
+                parsed.append(piece)
+        parts = tuple(parsed)
+    return SemVer(
+        major=int(match.group("major")),
+        minor=int(match.group("minor")),
+        patch=int(match.group("patch")),
+        prerelease=parts,
+    )
+
+
+@dataclass
+class TagInfo:
+    tag: str  # e.g. v1.0.0-beta.2
+    version: str  # e.g. 1.0.0-beta.2
+    semver: SemVer
+
+
+def run_command(cmd: Sequence[str]) -> str:
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"Command {' '.join(cmd)} failed with {result.returncode}: {result.stderr.strip()}"
+        )
+    return result.stdout.strip()
+
+
+def fetch_remote_tags() -> List[TagInfo]:
+    output = run_command(
+        [
+            "gh",
+            "api",
+            "-X",
+            "GET",
+            f"repos/{LANCE_REPO}/git/refs/tags",
+            "--paginate",
+            "--jq",
+            ".[].ref",
+        ]
+    )
+    tags: List[TagInfo] = []
+    for line in output.splitlines():
+        ref = line.strip()
+        if not ref.startswith("refs/tags/v"):
+            continue
+        tag = ref.split("refs/tags/")[-1]
+        version = tag.lstrip("v")
+        try:
+            tags.append(TagInfo(tag=tag, version=version, semver=parse_semver(version)))
+        except ValueError:
+            continue
+    if not tags:
+        raise RuntimeError("No Lance tags could be parsed from GitHub API output")
+    return tags
+
+
+def read_current_version(repo_root: Path) -> str:
+    cargo_path = repo_root / "Cargo.toml"
+    with cargo_path.open("rb") as fh:
+        data = tomllib.load(fh)
+    try:
+        deps = data["workspace"]["dependencies"]
+        entry = deps["lance"]
+    except KeyError as exc:  # pragma: no cover - configuration guard
+        raise RuntimeError("Failed to locate workspace.dependencies.lance in Cargo.toml") from exc
+
+    if isinstance(entry, str):
+        raw_version = entry
+    elif isinstance(entry, dict):
+        raw_version = entry.get("version", "")
+    else:  # pragma: no cover - defensive
+        raise RuntimeError("Unexpected lance dependency format")
+
+    raw_version = raw_version.strip()
+    if not raw_version:
+        raise RuntimeError("lance dependency does not declare a version")
+    return raw_version.lstrip("=")
+
+
+def determine_latest_tag(tags: Iterable[TagInfo]) -> TagInfo:
+    return max(tags, key=lambda tag: tag.semver)
+
+
+def write_outputs(args: argparse.Namespace, payload: dict) -> None:
+    target = getattr(args, "github_output", None)
+    if not target:
+        return
+    with open(target, "a", encoding="utf-8") as handle:
+        for key, value in payload.items():
+            handle.write(f"{key}={value}\n")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--repo-root",
+        default=Path(__file__).resolve().parents[1],
+        type=Path,
+        help="Path to the lancedb repository root",
+    )
+    parser.add_argument(
+        "--github-output",
+        default=os.environ.get("GITHUB_OUTPUT"),
+        help="Optional file path for writing GitHub Action outputs",
+    )
+    args = parser.parse_args(argv)
+
+    repo_root = Path(args.repo_root)
+    current_version = read_current_version(repo_root)
+    current_semver = parse_semver(current_version)
+
+    tags = fetch_remote_tags()
+    latest = determine_latest_tag(tags)
+    needs_update = latest.semver > current_semver
+
+    payload = {
+        "current_version": current_version,
+        "current_tag": f"v{current_version}",
+        "latest_version": latest.version,
+        "latest_tag": latest.tag,
+        "needs_update": "true" if needs_update else "false",
+    }
+
+    print(json.dumps(payload))
+    write_outputs(args, payload)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/set_lance_version.py

@@ -3,6 +3,8 @@ import re
 import sys
 import json
 
+LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
+
 
 def run_command(command: str) -> str:
     """
@@ -29,7 +31,7 @@ def get_latest_stable_version() -> str:
 
 def get_latest_preview_version() -> str:
     lance_tags = run_command(
-        "git ls-remote --tags https://github.com/lancedb/lance.git | grep 'refs/tags/v[0-9beta.-]\\+$'"
+        f"git ls-remote --tags {LANCE_GIT_URL} | grep 'refs/tags/v[0-9beta.-]\\+$'"
     ).splitlines()
     lance_tags = (
         tag.split("refs/tags/")[1]
@@ -55,7 +57,7 @@ def extract_features(line: str) -> list:
     match = re.search(r'"features"\s*=\s*\[\s*(.*?)\s*\]', line, re.DOTALL)
     if match:
         features_str = match.group(1)
-        return [f.strip('"') for f in features_str.split(",") if len(f) > 0]
+        return [f.strip().strip('"') for f in features_str.split(",") if f.strip()]
     return []
 
 
@@ -117,7 +119,7 @@ def update_cargo_toml(line_updater):
     lance_line = ""
     is_parsing_lance_line = False
     for line in lines:
-        if line.startswith("lance") and not line.startswith("lance-namespace"):
+        if line.startswith("lance"):
             # Check if this is a single-line or multi-line entry
             # Single-line entries either:
             # 1. End with } (complete inline table)
@@ -176,17 +178,15 @@ def set_stable_version(version: str):
 def set_preview_version(version: str):
     """
     Sets lines to
-    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.29.0-beta.2", "git" = "https://github.com/lancedb/lance.git" }
-    lance-io = { "version" = "=0.29.0", default-features = false, "tag" = "v0.29.0-beta.2", "git" = "https://github.com/lancedb/lance.git" }
+    lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.29.0-beta.2", "git" = LANCE_GIT_URL }
+    lance-io = { "version" = "=0.29.0", default-features = false, "tag" = "v0.29.0-beta.2", "git" = LANCE_GIT_URL }
     ...
     """
 
     def line_updater(line: str) -> str:
         package_name = line.split("=", maxsplit=1)[0].strip()
-        base_version = version.split("-")[0]  # Get the base version without beta suffix
-
         # Build config in desired order: version, default-features, features, tag, git
-        config = {"version": f"={base_version}"}
+        config = {"version": f"={version}"}
 
         if extract_default_features(line):
             config["default-features"] = False
@@ -196,7 +196,7 @@ def set_preview_version(version: str):
             config["features"] = features
 
         config["tag"] = f"v{version}"
-        config["git"] = "https://github.com/lancedb/lance.git"
+        config["git"] = LANCE_GIT_URL
 
         return dict_to_toml_line(package_name, config)
 
@@ -229,6 +229,29 @@ def set_local_version():
     update_cargo_toml(line_updater)
 
 
+def update_lockfiles(version: str, fallback_to_git: bool = False):
+    """
+    Update Cargo metadata and optionally fall back to using the git tag if the
+    requested crates.io version is unavailable.
+    """
+    try:
+        print("Updating lockfiles...", file=sys.stderr, end="")
+        run_command("cargo metadata > /dev/null")
+        print(" done.", file=sys.stderr)
+    except Exception as e:
+        if fallback_to_git and "failed to select a version" in str(e):
+            print(
+                f" failed for crates.io v{version}, retrying with git tag...",
+                file=sys.stderr,
+            )
+            set_preview_version(version)
+            print("Updating lockfiles...", file=sys.stderr, end="")
+            run_command("cargo metadata > /dev/null")
+            print(" done.", file=sys.stderr)
+        else:
+            raise
+
+
 parser = argparse.ArgumentParser(description="Set the version of the Lance package.")
 parser.add_argument(
     "version",
@@ -244,6 +267,7 @@ if args.version == "stable":
         file=sys.stderr,
     )
     set_stable_version(latest_stable_version)
+    update_lockfiles(latest_stable_version)
 elif args.version == "preview":
     latest_preview_version = get_latest_preview_version()
     print(
@@ -251,8 +275,10 @@ elif args.version == "preview":
         file=sys.stderr,
     )
     set_preview_version(latest_preview_version)
+    update_lockfiles(latest_preview_version)
 elif args.version == "local":
     set_local_version()
+    update_lockfiles("local")
 else:
     # Parse the version number.
     version = args.version
@@ -262,9 +288,7 @@ else:
 
     if "beta" in version:
         set_preview_version(version)
+        update_lockfiles(version)
     else:
         set_stable_version(version)
-
-print("Updating lockfiles...", file=sys.stderr, end="")
-run_command("cargo metadata > /dev/null")
-print(" done.", file=sys.stderr)
+        update_lockfiles(version, fallback_to_git=True)

docs/README.md

@@ -1,8 +1,8 @@
 # LanceDB Documentation
 
-LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
+LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs).
 
-Docs is built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
+The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
 unreleased features.
 

docs/mkdocs.yml

@@ -41,7 +41,6 @@ theme:
   icon:
     repo: fontawesome/brands/github
     annotation: material/arrow-right-circle
-  custom_dir: overrides
 
 plugins:
   - search
@@ -49,7 +48,9 @@ plugins:
   - mkdocstrings:
       handlers:
         python:
-          paths: [../python]
+          # Ensure the handler points to the real package root
+          # so it reads local sources at python/python/lancedb
+          paths: [../python/python]
           options:
             docstring_style: numpy
             heading_level: 3
@@ -65,26 +66,24 @@ plugins:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
-            - https://lancedb.github.io/lance/objects.inv
             - https://docs.pydantic.dev/latest/objects.inv
-  - mkdocs-jupyter
   - render_swagger:
       allow_arbitrary_locations: true
-  - redirects:
-      redirect_maps:
-        # Redirect the home page and other top-level markdown files. This enables maximum SEO benefit
-        # other sub-pages are handled by the ingected js in overrides/partials/header.html
-        'index.md': 'https://lancedb.com/docs/'
-        'guides/tables.md': 'https://lancedb.com/docs/tables/'
-        'ann_indexes.md': 'https://lancedb.com/docs/indexing/'
-        'basic.md': 'https://lancedb.com/docs/quickstart/'
-        'faq.md': 'https://lancedb.com/docs/faq/'
-        'embeddings/understanding_embeddings.md': 'https://lancedb.com/docs/embedding/'
-        'integrations.md': 'https://lancedb.com/docs/integrations/'
-        'examples.md': 'https://lancedb.com/docs/tutorials/'
-        'concepts/vector_search.md': 'https://lancedb.com/docs/search/vector-search/'
-        'troubleshooting.md': 'https://lancedb.com/docs/troubleshooting/'
-        'guides/storage.md': 'https://lancedb.com/docs/storage/integrations'
+#   - redirects:
+#       redirect_maps:
+#         # Redirect the home page and other top-level markdown files. This enables maximum SEO benefit
+#         # other sub-pages are handled by the ingected js in overrides/partials/header.html
+#         'index.md': 'https://lancedb.com/docs/'
+#         'guides/tables.md': 'https://lancedb.com/docs/tables/'
+#         'ann_indexes.md': 'https://lancedb.com/docs/indexing/'
+#         'basic.md': 'https://lancedb.com/docs/quickstart/'
+#         'faq.md': 'https://lancedb.com/docs/faq/'
+#         'embeddings/understanding_embeddings.md': 'https://lancedb.com/docs/embedding/'
+#         'integrations.md': 'https://lancedb.com/docs/integrations/'
+#         'examples.md': 'https://lancedb.com/docs/tutorials/'
+#         'concepts/vector_search.md': 'https://lancedb.com/docs/search/vector-search/'
+#         'troubleshooting.md': 'https://lancedb.com/docs/troubleshooting/'
+#         'guides/storage.md': 'https://lancedb.com/docs/storage/integrations'
 
 
 
@@ -120,269 +119,11 @@ markdown_extensions:
       permalink: ""
 
 nav:
-  - Home:
-      - LanceDB: index.md
-      - 🏃🏼‍♂️ Quick start: basic.md
-      - 📚 Concepts:
-          - Vector search: concepts/vector_search.md
-          - Indexing:
-              - IVFPQ: concepts/index_ivfpq.md
-              - HNSW: concepts/index_hnsw.md
-          - Storage: concepts/storage.md
-          - Data management: concepts/data_management.md
-      - 🔨 Guides:
-          - Working with tables: guides/tables.md
-          - Building a vector index: ann_indexes.md
-          - Vector Search: search.md
-          - Full-text search (native): fts.md
-          - Full-text search (tantivy-based): fts_tantivy.md
-          - Building a scalar index: guides/scalar_index.md
-          - Hybrid search:
-              - Overview: hybrid_search/hybrid_search.md
-              - Comparing Rerankers: hybrid_search/eval.md
-              - Airbnb financial data example: notebooks/hybrid_search.ipynb
-          - Late interaction with MultiVector search:
-              - Overview: guides/multi-vector.md
-              - Example: notebooks/Multivector_on_LanceDB.ipynb
-          - RAG:
-              - Vanilla RAG: rag/vanilla_rag.md
-              - Multi-head RAG: rag/multi_head_rag.md
-              - Corrective RAG: rag/corrective_rag.md
-              - Agentic RAG: rag/agentic_rag.md
-              - Graph RAG: rag/graph_rag.md
-              - Self RAG: rag/self_rag.md
-              - Adaptive RAG: rag/adaptive_rag.md
-              - SFR RAG: rag/sfr_rag.md
-              - Advanced Techniques:
-                  - HyDE: rag/advanced_techniques/hyde.md
-                  - FLARE: rag/advanced_techniques/flare.md
-          - Reranking:
-              - Quickstart: reranking/index.md
-              - Cohere Reranker: reranking/cohere.md
-              - Linear Combination Reranker: reranking/linear_combination.md
-              - Reciprocal Rank Fusion Reranker: reranking/rrf.md
-              - Cross Encoder Reranker: reranking/cross_encoder.md
-              - ColBERT Reranker: reranking/colbert.md
-              - Jina Reranker: reranking/jina.md
-              - OpenAI Reranker: reranking/openai.md
-              - AnswerDotAi Rerankers: reranking/answerdotai.md
-              - Voyage AI Rerankers: reranking/voyageai.md
-              - Building Custom Rerankers: reranking/custom_reranker.md
-              - Example: notebooks/lancedb_reranking.ipynb
-          - Filtering: sql.md
-          - Versioning & Reproducibility:
-              - sync API: notebooks/reproducibility.ipynb
-              - async API: notebooks/reproducibility_async.ipynb
-          - Configuring Storage: guides/storage.md
-          - Migration Guide: migration.md
-          - Tuning retrieval performance:
-              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
-              - Reranking: guides/tuning_retrievers/2_reranking.md
-              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
-      - 🧬 Managing embeddings:
-          - Understand Embeddings: embeddings/understanding_embeddings.md
-          - Get Started: embeddings/index.md
-          - Embedding functions: embeddings/embedding_functions.md
-          - Available models:
-              - Overview: embeddings/default_embedding_functions.md
-              - Text Embedding Functions:
-                  - Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
-                  - Huggingface Embedding Models: embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md
-                  - Ollama Embeddings: embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md
-                  - OpenAI Embeddings: embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
-                  - Instructor Embeddings: embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md
-                  - Gemini Embeddings: embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md
-                  - Cohere Embeddings: embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md
-                  - Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
-                  - AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
-                  - IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
-                  - Voyage AI Embeddings: embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
-              - Multimodal Embedding Functions:
-                  - OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
-                  - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
-                  - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
-          - User-defined embedding functions: embeddings/custom_embedding_function.md
-          - Variables and secrets: embeddings/variables_and_secrets.md
-          - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
-          - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
-      - 🔌 Integrations:
-          - Tools and data formats: integrations/index.md
-          - Pandas and PyArrow: python/pandas_and_pyarrow.md
-          - Polars: python/polars_arrow.md
-          - DuckDB: python/duckdb.md
-          - Datafusion: python/datafusion.md
-          - LangChain:
-              - LangChain 🔗: integrations/langchain.md
-              - LangChain demo: notebooks/langchain_demo.ipynb
-              - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-          - LlamaIndex 🦙:
-              - LlamaIndex docs: integrations/llamaIndex.md
-              - LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
-          - Pydantic: python/pydantic.md
-          - Voxel51: integrations/voxel51.md
-          - PromptTools: integrations/prompttools.md
-          - dlt: integrations/dlt.md
-          - phidata: integrations/phidata.md
-          - Genkit: integrations/genkit.md
-      - 🎯 Examples:
-          - Overview: examples/index.md
-          - 🐍 Python:
-              - Overview: examples/examples_python.md
-              - Build From Scratch: examples/python_examples/build_from_scratch.md
-              - Multimodal: examples/python_examples/multimodal.md
-              - Rag: examples/python_examples/rag.md
-              - Vector Search: examples/python_examples/vector_search.md
-              - Chatbot: examples/python_examples/chatbot.md
-              - Evaluation: examples/python_examples/evaluations.md
-              - AI Agent: examples/python_examples/aiagent.md
-              - Recommender System: examples/python_examples/recommendersystem.md
-              - Miscellaneous:
-                  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-                  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-          - 👾 JavaScript:
-              - Overview: examples/examples_js.md
-              - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-              - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-              - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-          - 🦀 Rust:
-              - Overview: examples/examples_rust.md
-      - 📓 Studies:
-          - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
-      - 💭 FAQs: faq.md
-      - 🔍 Troubleshooting: troubleshooting.md
-      - ⚙️ API reference:
-          - 🐍 Python: python/python.md
-          - 👾 JavaScript (vectordb): javascript/modules.md
-          - 👾 JavaScript (lancedb): js/globals.md
-          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
-
-  - Quick start: basic.md
-  - Concepts:
-      - Vector search: concepts/vector_search.md
-      - Indexing:
-          - IVFPQ: concepts/index_ivfpq.md
-          - HNSW: concepts/index_hnsw.md
-      - Storage: concepts/storage.md
-      - Data management: concepts/data_management.md
-  - Guides:
-      - Working with tables: guides/tables.md
-      - Working with SQL: guides/sql_querying.md
-      - Building an ANN index: ann_indexes.md
-      - Vector Search: search.md
-      - Full-text search (native): fts.md
-      - Full-text search (tantivy-based): fts_tantivy.md
-      - Building a scalar index: guides/scalar_index.md
-      - Hybrid search:
-          - Overview: hybrid_search/hybrid_search.md
-          - Comparing Rerankers: hybrid_search/eval.md
-          - Airbnb financial data example: notebooks/hybrid_search.ipynb
-      - Late interaction with MultiVector search:
-          - Overview: guides/multi-vector.md
-          - Document search Example: notebooks/Multivector_on_LanceDB.ipynb
-      - RAG:
-          - Vanilla RAG: rag/vanilla_rag.md
-          - Multi-head RAG: rag/multi_head_rag.md
-          - Corrective RAG: rag/corrective_rag.md
-          - Agentic RAG: rag/agentic_rag.md
-          - Graph RAG: rag/graph_rag.md
-          - Self RAG: rag/self_rag.md
-          - Adaptive RAG: rag/adaptive_rag.md
-          - SFR RAG: rag/sfr_rag.md
-          - Advanced Techniques:
-              - HyDE: rag/advanced_techniques/hyde.md
-              - FLARE: rag/advanced_techniques/flare.md
-      - Reranking:
-          - Quickstart: reranking/index.md
-          - Cohere Reranker: reranking/cohere.md
-          - Linear Combination Reranker: reranking/linear_combination.md
-          - Reciprocal Rank Fusion Reranker: reranking/rrf.md
-          - Cross Encoder Reranker: reranking/cross_encoder.md
-          - ColBERT Reranker: reranking/colbert.md
-          - Jina Reranker: reranking/jina.md
-          - OpenAI Reranker: reranking/openai.md
-          - AnswerDotAi Rerankers: reranking/answerdotai.md
-          - Building Custom Rerankers: reranking/custom_reranker.md
-          - Example: notebooks/lancedb_reranking.ipynb
-      - Filtering: sql.md
-      - Versioning & Reproducibility:
-          - sync API: notebooks/reproducibility.ipynb
-          - async API: notebooks/reproducibility_async.ipynb
-      - Configuring Storage: guides/storage.md
-      - Migration Guide: migration.md
-      - Tuning retrieval performance:
-          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
-          - Reranking: guides/tuning_retrievers/2_reranking.md
-          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
-  - Managing Embeddings:
-      - Understand Embeddings: embeddings/understanding_embeddings.md
-      - Get Started: embeddings/index.md
-      - Embedding functions: embeddings/embedding_functions.md
-      - Available models:
-          - Overview: embeddings/default_embedding_functions.md
-          - Text Embedding Functions:
-              - Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
-              - Huggingface Embedding Models: embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md
-              - Ollama Embeddings: embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md
-              - OpenAI Embeddings: embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
-              - Instructor Embeddings: embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md
-              - Gemini Embeddings: embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md
-              - Cohere Embeddings: embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md
-              - Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
-              - AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
-              - IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
-          - Multimodal Embedding Functions:
-              - OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
-              - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
-              - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
-      - User-defined embedding functions: embeddings/custom_embedding_function.md
-      - Variables and secrets: embeddings/variables_and_secrets.md
-      - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
-      - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
-  - Integrations:
-      - Overview: integrations/index.md
-      - Pandas and PyArrow: python/pandas_and_pyarrow.md
-      - Polars: python/polars_arrow.md
-      - DuckDB: python/duckdb.md
-      - Datafusion: python/datafusion.md
-      - LangChain 🦜️🔗↗: integrations/langchain.md
-      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-      - LlamaIndex 🦙↗: integrations/llamaIndex.md
-      - Pydantic: python/pydantic.md
-      - Voxel51: integrations/voxel51.md
-      - PromptTools: integrations/prompttools.md
-      - dlt: integrations/dlt.md
-      - phidata: integrations/phidata.md
-      - Genkit: integrations/genkit.md
-  - Examples:
-      - examples/index.md
-      - 🐍 Python:
-          - Overview: examples/examples_python.md
-          - Build From Scratch: examples/python_examples/build_from_scratch.md
-          - Multimodal: examples/python_examples/multimodal.md
-          - Rag: examples/python_examples/rag.md
-          - Vector Search: examples/python_examples/vector_search.md
-          - Chatbot: examples/python_examples/chatbot.md
-          - Evaluation: examples/python_examples/evaluations.md
-          - AI Agent: examples/python_examples/aiagent.md
-          - Recommender System: examples/python_examples/recommendersystem.md
-          - Miscellaneous:
-              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-      - 👾 JavaScript:
-          - Overview: examples/examples_js.md
-          - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-          - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-          - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-      - 🦀 Rust:
-          - Overview: examples/examples_rust.md
-  - Studies:
-      - studies/overview.md
-      - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
   - API reference:
-      - Overview: api_reference.md
+      - Overview: index.md
       - Python: python/python.md
-      - Javascript (vectordb): javascript/modules.md
-      - Javascript (lancedb): js/globals.md
+      - Javascript/TypeScript: js/globals.md
+      - Java: java/java.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
 
 extra_css:
@@ -390,7 +131,6 @@ extra_css:
   - styles/extra.css
 
 extra_javascript:
-  - "extra_js/init_ask_ai_widget.js"
   - "extra_js/reo.js"
 
 extra:

docs/overrides/partials/header.html

@@ -1,255 +0,0 @@
-<!--
-  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to
-  deal in the Software without restriction, including without limitation the
-  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-  sell copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-  IN THE SOFTWARE.
--->
-<div id="deprecation-banner" style="background-color: #f8d7da; color: #721c24; padding: 1em; text-align: center;">                                                           
-    <p style="margin: 0; font-size: 1.1em;">                                                                                                           
-        <strong>This documentation site is deprecated.</strong>                                                                           
-        Please visit our new documentation site at <a href="https://lancedb.com/docs" style="color: #721c24; text-decoration: underline;">
-          lancedb.com/docs</a> for the latest information.                                                                                        
-    </p>                                                                                                                                               
-  </div>  
-{% set class = "md-header" %}
-{% if "navigation.tabs.sticky" in features %}
-  {% set class = class ~ " md-header--shadow md-header--lifted" %}
-{% elif "navigation.tabs" not in features %}
-  {% set class = class ~ " md-header--shadow" %}
-{% endif %}
-
-<!-- Header -->
-<header class="{{ class }}" data-md-component="header">
-  <nav
-    class="md-header__inner md-grid"
-    aria-label="{{ lang.t('header') }}"
-  >
-
-    <!-- Link to home -->
-    <a
-      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
-      title="{{ config.site_name | e }}"
-      class="md-header__button md-logo"
-      aria-label="{{ config.site_name }}"
-      data-md-component="logo"
-    >
-      {% include "partials/logo.html" %}
-    </a>
-
-    <!-- Button to open drawer -->
-    <label class="md-header__button md-icon" for="__drawer">
-      {% include ".icons/material/menu" ~ ".svg" %}
-    </label>
-
-    <!-- Header title -->
-    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
-      <div class="md-header__ellipsis">
-        <div class="md-header__topic">
-          <span class="md-ellipsis">
-            {{ config.site_name }}
-          </span>
-        </div>
-        <div class="md-header__topic" data-md-component="header-topic">
-          <span class="md-ellipsis">
-            {% if page.meta and page.meta.title %}
-              {{ page.meta.title }}
-            {% else %}
-              {{ page.title }}
-            {% endif %}
-          </span>
-        </div>
-      </div>
-    </div>
-
-    <!-- Color palette -->
-    {% if config.theme.palette %}
-      {% if not config.theme.palette is mapping %}
-        <form class="md-header__option" data-md-component="palette">
-          {% for option in config.theme.palette %}
-            {% set scheme  = option.scheme  | d("default", true) %}
-            {% set primary = option.primary | d("indigo", true) %}
-            {% set accent  = option.accent  | d("indigo", true) %}
-            <input
-              class="md-option"
-              data-md-color-media="{{ option.media }}"
-              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
-              data-md-color-primary="{{ primary | replace(' ', '-') }}"
-              data-md-color-accent="{{ accent | replace(' ', '-') }}"
-              {% if option.toggle %}
-                aria-label="{{ option.toggle.name }}"
-              {% else  %}
-                aria-hidden="true"
-              {% endif %}
-              type="radio"
-              name="__palette"
-              id="__palette_{{ loop.index }}"
-            />
-            {% if option.toggle %}
-              <label
-                class="md-header__button md-icon"
-                title="{{ option.toggle.name }}"
-                for="__palette_{{ loop.index0 or loop.length }}"
-                hidden
-              >
-                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
-              </label>
-            {% endif %}
-          {% endfor %}
-        </form>
-      {% endif %}
-    {% endif %}
-
-    <!-- Site language selector -->
-    {% if config.extra.alternate %}
-      <div class="md-header__option">
-        <div class="md-select">
-          {% set icon = config.theme.icon.alternate or "material/translate" %}
-          <button
-            class="md-header__button md-icon"
-            aria-label="{{ lang.t('select.language') }}"
-          >
-            {% include ".icons/" ~ icon ~ ".svg" %}
-          </button>
-          <div class="md-select__inner">
-            <ul class="md-select__list">
-              {% for alt in config.extra.alternate %}
-                <li class="md-select__item">
-                  <a
-                    href="{{ alt.link | url }}"
-                    hreflang="{{ alt.lang }}"
-                    class="md-select__link"
-                  >
-                    {{ alt.name }}
-                  </a>
-                </li>
-              {% endfor %}
-            </ul>
-          </div>
-        </div>
-      </div>
-    {% endif %}
-
-    <!-- Button to open search modal -->
-    {% if "material/search" in config.plugins %}
-      <label class="md-header__button md-icon" for="__search">
-        {% include ".icons/material/magnify.svg" %}
-      </label>
-
-      <!-- Search interface -->
-      {% include "partials/search.html" %}
-    {% endif %}
-
-    <div style="margin-left: 10px; margin-right: 5px;">
-        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
-          <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
-      </a>
-  </div>
-    <div style="margin-left: 5px; margin-right: 5px;">
-        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
-            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
-        </a>
-    </div>
-
-    <!-- Repository information -->
-    {% if config.repo_url %}
-      <div class="md-header__source" style="margin-left: -5px !important;">
-        {% include "partials/source.html" %}
-      </div>
-    {% endif %}
-  </nav>
-
-  <!-- Navigation tabs (sticky) -->
-  {% if "navigation.tabs.sticky" in features %}
-    {% if "navigation.tabs" in features %}
-      {% include "partials/tabs.html" %}
-    {% endif %}
-  {% endif %}
-</header>
-
-<script>
-  (function() {
-    function checkPathAndRedirect() {
-      var banner = document.getElementById('deprecation-banner');
-
-      if (document.querySelector('meta[http-equiv="refresh"]')) {
-        return; // The redirects plugin is already handling this page.
-      }
-
-      var currentPath = window.location.pathname;
-
-      var cleanPath = currentPath.endsWith('/') && currentPath.length > 1
-        ? currentPath.slice(0, -1)
-        : currentPath;
-
-      // These are the ONLY paths that should remain on the old site
-      var apiPaths = [
-        '/lancedb/python', 
-        '/lancedb/javascript', 
-        '/lancedb/js',
-        '/lancedb/api_reference'
-      ];
-      
-      var isApiPage = apiPaths.some(function(apiPath) {
-        return cleanPath.startsWith(apiPath);
-      });
-
-      if (isApiPage) {
-        if (banner) {
-          banner.style.display = 'none';
-        }
-      } else {
-        if (banner) {
-          banner.style.display = 'block';
-        }
-        
-        // Add noindex meta tag to prevent indexing of old docs for seo
-        var noindexMeta = document.createElement('meta');
-        noindexMeta.setAttribute('name', 'robots');
-        noindexMeta.setAttribute('content', 'noindex, follow');
-        document.head.appendChild(noindexMeta);
-
-        // Add canonical link to point to the new docs to reward new site for seo
-        var canonicalLink = document.createElement('link');
-        canonicalLink.setAttribute('rel', 'canonical');
-        canonicalLink.setAttribute('href', 'https://lancedb.com/docs');
-        document.head.appendChild(canonicalLink);
-        
-        window.location.replace('https://lancedb.com/docs');
-      }
-    }
-
-    // Run the check only if doc is ready. This makes sure we catch the initial load
-    // and redirect.
-    if (document.readyState === 'loading') {
-      document.addEventListener('DOMContentLoaded', checkPathAndRedirect);
-    } else {
-      checkPathAndRedirect();
-    }
-
-    // Use an interval to handle subsequent navigation clicks.
-    var lastPath = window.location.pathname;
-    setInterval(function() {
-      if (window.location.pathname !== lastPath) {
-        lastPath = window.location.pathname;
-        checkPathAndRedirect();
-      }
-    }, 2000); // keeping it 2 second to make it easy for user to understand
-              // what's happening
-
-  })();
-</script>

docs/overrides/partials/main.html

@@ -1,5 +0,0 @@
-{% extends "base.html" %}
-
-{% block announce %}
-  📚 Starting June 1st, 2025, please use <a href="https://lancedb.github.io/documentation" target="_blank" rel="noopener noreferrer">lancedb.github.io/documentation</a> for the latest docs.
-{% endblock %}

docs/requirements.txt

@@ -1,8 +1,9 @@
 mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
+mkdocs-autorefs<=1.0
 mkdocstrings[python]==0.25.2
 griffe
 mkdocs-render-swagger-plugin
 pydantic
-mkdocs-redirects
+mkdocs-redirects

docs/src/ann_indexes.md

@@ -1,307 +0,0 @@
-# Approximate Nearest Neighbor (ANN) Indexes
-
-An ANN or a vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity via the chosen distance metric.
-By constructing a vector index, the search space is effectively narrowed down, avoiding the need
-for brute-force scanning of the entire vector space.
-A vector index is faster but less accurate than exhaustive search (kNN or flat search).
-LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
-
-## Disk-based Index
-
-Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
-the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
-See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
-
-## Creating an IVF_PQ Index
-
-Lance supports `IVF_PQ` index type by default.
-
-=== "Python"
-    === "Sync API"
-
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index"
-        ```
-    === "Async API"
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-ivfpq"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index_async"
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
-
-        ```typescript
-        --8<--- "nodejs/examples/ann_indexes.test.ts:import"
-
-        --8<-- "nodejs/examples/ann_indexes.test.ts:ingest"
-        ```
-
-    === "vectordb (deprecated)"
-
-        Creating indexes is done via the [lancedb.Table.createIndex](../javascript/interfaces/Table.md/#createIndex) method.
-
-        ```typescript
-        --8<--- "docs/src/ann_indexes.ts:import"
-
-        --8<-- "docs/src/ann_indexes.ts:ingest"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
-    ```
-
-    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
-
-The following IVF_PQ paramters can be specified:
-
-- **distance_type**: The distance metric to use. By default it uses euclidean distance "`l2`".
-  We also support "cosine" and "dot" distance as well.
-- **num_partitions**: The number of partitions in the index. The default is the square root
-  of the number of rows.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
-
-- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
-  a single PQ code. The default is the dimension of the vector divided by 16.
-- **num_bits**: The number of bits used to encode each sub-vector. Only 4 and 8 are supported. The higher the number of bits, the higher the accuracy of the index, also the slower search. The default is 8.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
-
-<figure markdown>
-  ![IVF PQ](./assets/ivf_pq.png)
-  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
-</figure>
-
-### Use GPU to build vector index
-
-Lance Python SDK has experimental GPU support for creating IVF index.
-Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
-
-You can specify the GPU device to train IVF partitions via
-
-- **accelerator**: Specify to `cuda` or `mps` (on Apple Silicon) to enable GPU training.
-
-=== "Linux"
-
-    <!-- skip-test -->
-    ``` { .python .copy }
-    # Create index using CUDA on Nvidia GPUs.
-    tbl.create_index(
-        num_partitions=256,
-        num_sub_vectors=96,
-        accelerator="cuda"
-    )
-    ```
-
-=== "MacOS"
-
-    <!-- skip-test -->
-    ```python
-    # Create index using MPS on Apple Silicon.
-    tbl.create_index(
-        num_partitions=256,
-        num_sub_vectors=96,
-        accelerator="mps"
-    )
-    ```
-!!! note
-    GPU based indexing is not yet supported with our asynchronous client.
-    
-Troubleshooting:
-
-If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
-PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
-
-## Querying an ANN Index
-
-Querying vector indexes is done via the [search](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.search) function.
-
-There are a couple of parameters that can be used to fine-tune the search:
-
-- **limit** (default: 10): The amount of results that will be returned
-- **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
-    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
-
-- **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
-  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
-!!! note
-    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
-
-
-=== "Python"
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async"
-        ```
-
-    ```text
-                                              vector       item       _distance
-    0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-    1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search1"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/ann_indexes.ts:search1"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
-    ```
-
-    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
-
-The search will return the data requested in addition to the distance of each item.
-
-### Filtering (where clause)
-
-You can further filter the elements returned by a search using a where clause.
-
-=== "Python"
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_filter"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_filter"
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search2"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```javascript
-        --8<-- "docs/src/ann_indexes.ts:search2"
-        ```
-
-### Projections (select clause)
-
-You can select the columns returned by the query using a select clause.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_select"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_select"
-        ```
-
-    ```text
-                                                vector _distance
-    0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-    1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-    ...
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search3"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/ann_indexes.ts:search3"
-        ```
-
-## FAQ
-
-### Why do I need to manually create an index?
-
-Currently, LanceDB does _not_ automatically create the ANN index.
-LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
-datasets of the order of ~100K vectors don't require index creation. If you can live with up to
-100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
-
-### When is it necessary to create an ANN vector index?
-
-`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
-In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
-We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
-vector indices are usually not necessary.
-
-For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
-
-### How big is my index, and how many memory will it take?
-
-In LanceDB, all vector indices are **disk-based**, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
-
-For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
-Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
-
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index?
-
-`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
-Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
-On `SIFT-1M` dataset, our benchmark shows that keeping each partition 4K-8K rows lead to a good latency / recall.
-
-`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. The number should be a factor of the vector dimension. Because
-PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
-less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
-
-!!! note
-    if `num_sub_vectors` is set to be greater than the vector dimension, you will see errors like `attempt to divide by zero`
-
-### How to choose `m` and `ef_construction` for `IVF_HNSW_*` index?
-
-`m` determines the number of connections a new node establishes with its closest neighbors upon entering the graph. Typically, `m` falls within the range of 5 to 48. Lower `m` values are suitable for low-dimensional data or scenarios where recall is less critical. Conversely, higher `m` values are beneficial for high-dimensional data or when high recall is required. In essence, a larger `m` results in a denser graph with increased connectivity, but at the expense of higher memory consumption.
-
-`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase

docs/src/ann_indexes.ts

@@ -1,54 +0,0 @@
-// --8<-- [start:import]
-import * as vectordb from "vectordb";
-// --8<-- [end:import]
-
-(async () => {
-  console.log("ann_indexes.ts: start");
-  // --8<-- [start:ingest]
-  const db = await vectordb.connect("data/sample-lancedb");
-
-  let data = [];
-  for (let i = 0; i < 10_000; i++) {
-    data.push({
-      vector: Array(1536).fill(i),
-      id: `${i}`,
-      content: "",
-      longId: `${i}`,
-    });
-  }
-  const table = await db.createTable("my_vectors", data);
-  await table.createIndex({
-    type: "ivf_pq",
-    column: "vector",
-    num_partitions: 16,
-    num_sub_vectors: 48,
-  });
-  // --8<-- [end:ingest]
-
-  // --8<-- [start:search1]
-  const results_1 = await table
-    .search(Array(1536).fill(1.2))
-    .limit(2)
-    .nprobes(20)
-    .refineFactor(10)
-    .execute();
-  // --8<-- [end:search1]
-
-  // --8<-- [start:search2]
-  const results_2 = await table
-    .search(Array(1536).fill(1.2))
-    .where("id != '1141'")
-    .limit(2)
-    .execute();
-  // --8<-- [end:search2]
-
-  // --8<-- [start:search3]
-  const results_3 = await table
-    .search(Array(1536).fill(1.2))
-    .select(["id"])
-    .limit(2)
-    .execute();
-  // --8<-- [end:search3]
-
-  console.log("ann_indexes.ts: done");
-})();

docs/src/api_reference.md

@@ -1,8 +0,0 @@
-# API Reference
-
-The API reference for the LanceDB client SDKs are available at the following locations:
-
-- [Python](python/python.md)
-- [JavaScript (legacy vectordb package)](javascript/modules.md)
-- [JavaScript (newer @lancedb/lancedb package)](js/globals.md)
-- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -1,655 +0,0 @@
-# Quick start
-
-!!! info "LanceDB can be run in a number of ways:"
-
-    * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Directly from a client application like a Jupyter notebook for analytical workloads
-    * Deployed as a remote serverless database
-
-![](assets/lancedb_embedded_explanation.png)
-
-## Installation
-
-=== "Python"
-
-      ```shell
-      pip install lancedb
-      ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```shell
-        npm install @lancedb/lancedb
-        ```
-        !!! note "Bundling `@lancedb/lancedb` apps with Webpack"
-
-            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-            ```javascript
-            /** @type {import('next').NextConfig} */
-            module.exports = ({
-            webpack(config) {
-                config.externals.push({ '@lancedb/lancedb': '@lancedb/lancedb' })
-                return config;
-            }
-            })
-            ```
-
-        !!! note "Yarn users"
-
-            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
-
-            ```shell
-            yarn add apache-arrow
-            ```
-
-    === "vectordb (deprecated)"
-
-        ```shell
-        npm install vectordb
-        ```
-        !!! note "Bundling `vectordb` apps with Webpack"
-
-            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-            ```javascript
-            /** @type {import('next').NextConfig} */
-            module.exports = ({
-            webpack(config) {
-                config.externals.push({ vectordb: 'vectordb' })
-                return config;
-            }
-            })
-            ```
-
-        !!! note "Yarn users"
-
-            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
-
-            ```shell
-            yarn add apache-arrow
-            ```
-
-=== "Rust"
-
-    ```shell
-    cargo add lancedb
-    ```
-
-    !!! info "To use the lancedb create, you first need to install protobuf."
-
-    === "macOS"
-
-        ```shell
-        brew install protobuf
-        ```
-
-    === "Ubuntu/Debian"
-
-        ```shell
-        sudo apt install -y protobuf-compiler libssl-dev
-        ```
-
-    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
-
-### Preview releases
-
-Stable releases are created about every 2 weeks. For the latest features and bug
-fixes, you can install the preview release. These releases receive the same
-level of testing as stable releases, but are not guaranteed to be available for
-more than 6 months after they are released. Once your application is stable, we
-recommend switching to stable releases.
-
-=== "Python"
-
-      ```shell
-      pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
-      ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```shell
-        npm install @lancedb/lancedb@preview
-        ```
-    === "vectordb (deprecated)"
-
-        ```shell
-        npm install vectordb@preview
-        ```
-
-=== "Rust"
-
-    We don't push preview releases to crates.io, but you can referent the tag
-    in GitHub within your Cargo dependencies:
-
-    ```toml
-    [dependencies]
-    lancedb = { git = "https://github.com/lancedb/lancedb.git", tag = "vX.Y.Z-beta.N" }
-    ```
-
-## Connect to a database
-
-=== "Python"
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:imports"
-
-        --8<-- "python/python/tests/docs/test_basic.py:set_uri"
-        --8<-- "python/python/tests/docs/test_basic.py:connect"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:imports"
-
-        --8<-- "python/python/tests/docs/test_basic.py:set_uri"
-        --8<-- "python/python/tests/docs/test_basic.py:connect_async"
-        ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        import * as lancedb from "@lancedb/lancedb";
-        import * as arrow from "apache-arrow";
-
-        --8<-- "nodejs/examples/basic.test.ts:connect"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:open_db"
-        ```
-
-=== "Rust"
-
-    ```rust
-    #[tokio::main]
-    async fn main() -> Result<()> {
-        --8<-- "rust/lancedb/examples/simple.rs:connect"
-    }
-    ```
-
-    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
-
-LanceDB will create the directory if it doesn't exist (including parent directories).
-
-If you need a reminder of the uri, you can call `db.uri()`.
-
-## Create a table
-
-### Create a table from initial data
-
-If you have data to insert into the table at creation time, you can simultaneously create a
-table and insert the data into it. The schema of the data will be used as the schema of the
-table.
-
-=== "Python"
-
-    If the table already exists, LanceDB will raise an error by default.
-    If you want to overwrite the table, you can pass in `mode="overwrite"`
-    to the `create_table` method.
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table"
-        ```
-
-        You can also pass in a pandas DataFrame directly:
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
-        ```
-
-        You can also pass in a pandas DataFrame directly:
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
-        ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:create_table"
-        ```
-
-        If the table already exists, LanceDB will raise an error by default.
-        If you want to overwrite the table, you can pass in `mode:"overwrite"`
-        to the `createTable` function.
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_table"
-    ```
-
-    If the table already exists, LanceDB will raise an error by default.  See
-    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
-    for details on how to overwrite (or open) existing tables instead.
-
-    !!! Providing table records in Rust
-
-        The Rust SDK currently expects data to be provided as an Arrow
-        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
-        Support for additional formats (such as serde or polars) is on the roadmap.
-
-!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
-
-!!! info "Automatic embedding generation with Embedding API"
-    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
-
-### Create an empty table
-
-Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema, so that you can add
-data to the table at a later time (as long as it conforms to the schema). This is
-similar to a `CREATE TABLE` statement in SQL.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
-        ```
-
-    !!! note "You can define schema in Pydantic"
-        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
-    ```
-
-## Open an existing table
-
-Once created, you can open a table as follows:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:open_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
-        ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:open_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        const tbl = await db.openTable("myTable");
-        ```
-
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
-    ```
-
-If you forget the name of your table, you can always get a listing of all table names:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:table_names"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
-        ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:table_names"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        console.log(await db.tableNames());
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:list_names"
-    ```
-
-## Add data to a table
-
-After a table has been created, you can always add more data to it as follows:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_data"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
-        ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:add_data"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:add"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:add"
-    ```
-
-## Search for nearest neighbors
-
-Once you've embedded the query, you can find its nearest neighbors as follows:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:vector_search"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
-        ```
-
-    This returns a pandas DataFrame with the results.
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:vector_search"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:search"
-        ```
-
-=== "Rust"
-
-    ```rust
-    use futures::TryStreamExt;
-
-    --8<-- "rust/lancedb/examples/simple.rs:search"
-    ```
-
-    !!! Query vectors in Rust
-        Rust does not yet support automatic execution of embedding functions.  You will need to
-        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
-        https://github.com/lancedb/lancedb/issues/994
-
-        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
-        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
-
-By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
-For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
-LanceDB allows you to create an ANN index on a table as follows:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
-        ```
-
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_index"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```{.typescript .ignore}
-        --8<-- "docs/src/basic_legacy.ts:create_index"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_index"
-    ```
-
-!!! note "Why do I need to create an index manually?"
-LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
-for really fast retrievals via a disk-based index, and the second is that data and query workloads can
-be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
-to fine-tune index size, query latency and accuracy. See the section on
-[ANN indexes](ann_indexes.md) for more details.
-
-## Delete rows from a table
-
-Use the `delete()` method on tables to delete rows from a table. To choose
-which rows to delete, provide a filter that matches on the metadata columns.
-This can delete any number of rows that match the filter.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
-        ```
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:delete_rows"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:delete"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:delete"
-    ```
-
-The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause (`only_if()` in Rust) on a search. They can be as
-simple or complex as needed. To see what expressions are supported, see the
-[SQL filters](sql.md) section.
-
-=== "Python"
-
-    === "Sync API"
-        Read more: [lancedb.table.Table.delete][]
-    === "Async API"
-        Read more: [lancedb.table.AsyncTable.delete][]
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        Read more: [lancedb.Table.delete](javascript/interfaces/Table.md#delete)
-
-    === "vectordb (deprecated)"
-
-        Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
-
-=== "Rust"
-
-      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
-
-## Drop a table
-
-Use the `drop_table()` method on the database to remove a table.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-        ```
-
-    This permanently removes the table and is not recoverable, unlike deleting rows.
-    By default, if the table does not exist an exception is raised. To suppress this,
-    you can pass in `ignore_missing=True`.
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:drop_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:drop_table"
-        ```
-
-        This permanently removes the table and is not recoverable, unlike deleting rows.
-        If the table does not exist an exception is raised.
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
-    ```
-
-
-## Using the Embedding API
-You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
-
-        --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
-        ```
-    === "Async API"
-
-        Coming soon to the async API.
-        https://github.com/lancedb/lancedb/issues/1938
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.test.ts:openai_embeddings"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/openai.rs:imports"
-    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/index.md).
-
-
-## What's next
-
-This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
-
-If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.
-
-[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/basic_legacy.ts

@@ -1,126 +0,0 @@
-// --8<-- [start:import]
-import * as lancedb from "vectordb";
-import {
-  Schema,
-  Field,
-  Float32,
-  FixedSizeList,
-  Int32,
-  Float16,
-} from "apache-arrow";
-import * as arrow from "apache-arrow";
-// --8<-- [end:import]
-import * as fs from "fs";
-import { Table as ArrowTable, Utf8 } from "apache-arrow";
-
-const example = async () => {
-  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
-  // --8<-- [start:open_db]
-  const lancedb = require("vectordb");
-  const uri = "data/sample-lancedb";
-  const db = await lancedb.connect(uri);
-  // --8<-- [end:open_db]
-
-  // --8<-- [start:create_table]
-  const tbl = await db.createTable(
-    "myTable",
-    [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ],
-    { writeMode: lancedb.WriteMode.Overwrite },
-  );
-  // --8<-- [end:create_table]
-  {
-    // --8<-- [start:create_table_with_schema]
-    const schema = new arrow.Schema([
-      new arrow.Field(
-        "vector",
-        new arrow.FixedSizeList(
-          2,
-          new arrow.Field("item", new arrow.Float32(), true),
-        ),
-      ),
-      new arrow.Field("item", new arrow.Utf8(), true),
-      new arrow.Field("price", new arrow.Float32(), true),
-    ]);
-    const data = [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ];
-    const tbl = await db.createTable({
-      name: "myTableWithSchema",
-      data,
-      schema,
-    });
-    // --8<-- [end:create_table_with_schema]
-  }
-
-  // --8<-- [start:add]
-  const newData = Array.from({ length: 500 }, (_, i) => ({
-    vector: [i, i + 1],
-    item: "fizz",
-    price: i * 0.1,
-  }));
-  await tbl.add(newData);
-  // --8<-- [end:add]
-
-  // --8<-- [start:create_index]
-  await tbl.createIndex({
-    type: "ivf_pq",
-    num_partitions: 2,
-    num_sub_vectors: 2,
-  });
-  // --8<-- [end:create_index]
-
-  // --8<-- [start:create_empty_table]
-  const schema = new arrow.Schema([
-    new arrow.Field("id", new arrow.Int32()),
-    new arrow.Field("name", new arrow.Utf8()),
-  ]);
-
-  const empty_tbl = await db.createTable({ name: "empty_table", schema });
-  // --8<-- [end:create_empty_table]
-  {
-    // --8<-- [start:create_f16_table]
-    const dim = 16;
-    const total = 10;
-    const schema = new Schema([
-      new Field("id", new Int32()),
-      new Field(
-        "vector",
-        new FixedSizeList(dim, new Field("item", new Float16(), true)),
-        false,
-      ),
-    ]);
-    const data = lancedb.makeArrowTable(
-      Array.from(Array(total), (_, i) => ({
-        id: i,
-        vector: Array.from(Array(dim), Math.random),
-      })),
-      { schema },
-    );
-    const table = await db.createTable("f16_tbl", data);
-    // --8<-- [end:create_f16_table]
-  }
-
-  // --8<-- [start:search]
-  const query = await tbl.search([100, 100]).limit(2).execute();
-  // --8<-- [end:search]
-
-  // --8<-- [start:delete]
-  await tbl.delete('item = "fizz"');
-  // --8<-- [end:delete]
-
-  // --8<-- [start:drop_table]
-  await db.dropTable("myTable");
-  // --8<-- [end:drop_table]
-};
-
-async function main() {
-  console.log("basic_legacy.ts: start");
-  await example();
-  console.log("basic_legacy.ts: done");
-}
-
-main();

docs/src/cloud/cloud_faq.md

@@ -1,34 +0,0 @@
-This section provides answers to the most common questions asked about LanceDB Cloud. By following these guidelines, you can ensure a smooth, performant experience with LanceDB Cloud.
-
-### Should I reuse the database connection?
-Yes! It is recommended to establish a single database connection and maintain it throughout your interaction with the tables within. 
-
-LanceDB uses HTTP connections to communicate with the servers. By re-using the Connection object, you avoid the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
-
-### Should I re-use the `Table` object?
-`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the **latest version** of the data. 
-
-### What should I do if I need to search for rows by `id`?
-LanceDB Cloud currently does not support an ID or primary key column. You are recommended to add a 
-user-defined ID column. To significantly improve the query performance with SQL causes, a scalar BITMAP/BTREE index should be created on this column. 
-
-### What are the vector indexing types supported by LanceDB Cloud?
-We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff between query latency and query quality.
-
-### When I add new rows to a table, do I need to manually update the index?
-No! LanceDB Cloud triggers an asynchronous background job to index the new vectors.
-
-Even though indexing is asynchronous, your vectors will still be immediately searchable. LanceDB uses brute-force search to search over unindexed rows. This makes you new data is immediately available, but does increase latency temporarily. To disable the brute-force part of search, set the `fast_search` flag in your query to `true`.
-
-### Do I need to reindex the whole dataset if only a small portion of the data is deleted or updated?
-No! Similar to adding data to the table, LanceDB Cloud triggers an asynchronous background job to update the existing indices. Therefore, no action is needed from users and there is absolutely no 
-downtime expected.
-
-### How do I know whether an index has been created?
-While index creation in LanceDB Cloud is generally fast, querying immediately after a `create_index` call may result in errors. It's recommended to use `list_indices` to verify index creation before querying.
-
-### Why is my query latency higher than expected?
-Multiple factors can impact query latency. To reduce query latency, consider the following:
-- Send pre-warm queries: send a few queries to warm up the cache before an actual user query.
-- Check network latency: LanceDB Cloud is hosted in AWS `us-east-1` region. It is recommended to run queries from an EC2 instance that is in the same region.
-- Create scalar indices: If you are filtering on metadata, it is recommended to create scalar indices on those columns. This will speedup searches with metadata filtering. See [here](../guides/scalar_index.md) for more details on creating a scalar index.

docs/src/cloud/index.md

@@ -1,17 +0,0 @@
-# About LanceDB Cloud
-
-LanceDB Cloud is a SaaS (software-as-a-service) solution that runs serverless in the cloud, clearly separating storage from compute. It's designed to be highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
-
-[Try out LanceDB Cloud (Public Beta)](https://cloud.lancedb.com){ .md-button .md-button--primary }
-
-## Architecture
-
-LanceDB Cloud provides the same underlying fast vector store that powers the OSS version, but without the need to maintain your own infrastructure. Because it's serverless, you only pay for the storage you use, and you can scale compute up and down as needed depending on the size of your data and its associated index.
-
-![](../assets/lancedb_cloud.png)
-
-## Transitioning from the OSS to the Cloud version
-
-The OSS version of LanceDB is designed to be embedded in your application, and it runs in-process. This makes it incredibly simple to self-host your own AI retrieval workflows for RAG and more and build and test out your concepts on your own infrastructure. The OSS version is forever free, and you can continue to build and integrate LanceDB into your existing backend applications without any added costs.
-
-Should you decide that you need a managed deployment in production, it's possible to seamlessly transition from the OSS to the cloud version by changing the connection string to point to a remote database instead of a local one. With LanceDB Cloud, you can take your AI application from development to production without major code changes or infrastructure burden.

docs/src/cloud/rest.md

@@ -1 +0,0 @@
-!!swagger ../../openapi.yml!!

docs/src/concepts/data_management.md

@@ -1,62 +0,0 @@
-# Data management
-
-This section covers concepts related to managing your data over time in LanceDB.
-
-## A primer on Lance
-
-Because LanceDB is built on top of the [Lance](https://lancedb.github.io/lance/) data format, it helps to understand some of its core ideas. Just like Apache Arrow, Lance is a fast columnar data format, but it has the added benefit of being versionable, query and train ML models on. Lance is designed to be used with simple and complex data types, like tabular data, images, videos audio, 3D point clouds (which are deeply nested) and more.
-
-The following concepts are important to keep in mind:
-
-- Data storage is columnar and is interoperable with other columnar formats (such as Parquet) via Arrow
-- Data is divided into fragments that represent a subset of the data
-- Data is versioned, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata
-
-!!! note
-    1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.
-    2. Second, these versions exist to keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be in the middle of querying the old version. It's important to retain older versions for as long as they might be queried.
-
-## What are fragments?
-
-Fragments are chunks of data in a Lance dataset. Each fragment includes multiple files that contain several columns in the chunk of data that it represents.
-
-## Compaction
-
-As you insert more data, your dataset will grow and you'll need to perform *compaction* to maintain query throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments together to reduce the amount of metadata that needs to be managed, and to reduce the number of files that need to be opened while scanning the dataset.
-
-### How does compaction improve performance?
-
-Compaction performs the following tasks in the background:
-
-- Removes deleted rows from fragments
-- Removes dropped columns from fragments
-- Merges small fragments into larger ones
-
-Depending on the use case and dataset, optimal compaction will have different requirements. As a rule of thumb:
-
-- It’s always better to use *batch* inserts rather than adding 1 row at a time (to avoid too small fragments). If single-row inserts are unavoidable, run compaction on a regular basis to merge them into larger fragments.
-- Keep the number of fragments under 100, which is suitable for most use cases (for *really* large datasets of >500M rows, more fragments might be needed)
-
-!!! note
-
-    LanceDB Cloud/Enterprise supports [auto-compaction](https://docs.lancedb.com/enterprise/architecture/architecture#write-path) which automatically optimizes fragments in the background as data changes.
-
-## Deletion
-
-Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever deleted from that fragment, it will not have a deletion file). This is important to keep in mind because it means that the data is still there, and can be recovered if needed, as long as that version still exists based on your backup policy.
-
-## Reindexing
-
-Reindexing is the process of updating the index to account for new data, keeping good performance for queries. This applies to either a full-text search (FTS) index or a vector index. For ANN search, new data will always be included in query results, but queries on tables with unindexed data will fallback to slower search methods for the new parts of the table. This is another important operation to run periodically as your data grows, as it also improves performance. This is especially important if you're appending large amounts of data to an existing dataset.
-
-!!! tip
-    When adding new data to a dataset that has an existing index (either FTS or vector), LanceDB doesn't immediately update the index until a reindex operation is complete.
-
-Both LanceDB OSS and Cloud support reindexing, but the process (at least for now) is different for each, depending on the type of index.
-
-In LanceDB OSS, re-indexing happens synchronously when you call either `create_index` or `optimize` on a table. In LanceDB Cloud, re-indexing happens asynchronously as you add and update data in your table.
-
-By default, queries will search new data even if it has yet to be indexed. This is done using brute-force methods, such as kNN for vector search, and combined with the fast index search results. This is done to ensure that you're always searching over all your data, but it does come at a performance cost. Without reindexing, adding more data to a table will make queries slower and more expensive. This behavior can be disabled by setting the [fast_search](https://lancedb.github.io/lancedb/python/python/#lancedb.query.AsyncQuery.fast_search) parameter which will instruct the query to ignore un-indexed data.
-
-* LanceDB Cloud/Enterprise supports [automatic incremental reindexing](https://docs.lancedb.com/core#vector-index) for vector, scalar, and FTS indices, where a background process will trigger a new index build for you automatically when new data is added or modified in a dataset
-* LanceDB OSS requires you to manually trigger a reindex operation -- we are working on adding incremental reindexing to LanceDB OSS as well

docs/src/concepts/index_hnsw.md

@@ -1,99 +0,0 @@
-
-# Understanding HNSW index
-
-Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. HNSW is one of the most accurate and fastest Approximate Nearest Neighbour search algorithms, It’s beneficial in high-dimensional spaces where finding the same nearest neighbor would be too slow and costly
-
-[Jump to usage](#usage)
-There are three main types of ANN search algorithms:
-
-* **Tree-based search algorithms**: Use a tree structure to organize and store data points.
-* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
-* **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. 
-
-HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below.  
-HNSW also combines this with the ideas behind a classic 1-dimensional search data structure: the skip list.
-
-## k-Nearest Neighbor Graphs and k-approximate Nearest neighbor Graphs
-The k-nearest neighbor graph actually predates its use for ANN search. Its construction is quite simple:
-
-* Each vector in the dataset is given an associated vertex.
-* Each vertex has outgoing edges to its k nearest neighbors. That is, the k closest other vertices by Euclidean distance between the two corresponding vectors. This can be thought of as a "friend list" for the vertex.
-* For some applications (including nearest-neighbor search), the incoming edges are also added.
-
-Eventually, it was realized that the following greedy search method over such a graph typically results in good approximate nearest neighbors:
-
-* Given a query vector, start at some fixed "entry point" vertex (e.g. the approximate center node).
-* Look at that vertex's neighbors. If any of them are closer to the query vector than the current vertex, then move to that vertex.
-* Repeat until a local optimum is found.
-
-The above algorithm also generalizes to e.g. top 10 approximate nearest neighbors.
-
-Computing a k-nearest neighbor graph is actually quite slow, taking quadratic time in the dataset size. It was quickly realized that near-identical performance can be achieved using a k-approximate nearest neighbor graph. That is, instead of obtaining the k-nearest neighbors for each vertex, an approximate nearest neighbor search data structure is used to build much faster.  
-In fact, another data structure is not needed: This can be done "incrementally".
-That is, if you start with a k-ANN graph for n-1 vertices, you can extend it to a k-ANN graph for n vertices as well by using the graph to obtain the k-ANN for the new vertex.
-
-One downside of k-NN and k-ANN graphs alone is that one must typically build them with a large value of k to get decent results, resulting in a large index.
-
-
-## HNSW: Hierarchical Navigable Small Worlds
-
-HNSW builds on k-ANN in two main ways:
-
-* Instead of getting the k-approximate nearest neighbors for a large value of k, it sparsifies the k-ANN graph using a carefully chosen "edge pruning" heuristic, allowing for the number of edges per vertex to be limited to a relatively small constant.
-* The "entry point" vertex is chosen dynamically using a recursively constructed data structure on a subset of the data, similarly to a skip list.
-
-This recursive structure can be thought of as separating into layers:
-
-* At the bottom-most layer, an k-ANN graph on the whole dataset is present.
-* At the second layer, a k-ANN graph on a fraction of the dataset (e.g. 10%) is present.
-* At the Lth layer, a k-ANN graph is present. It is over a (constant) fraction (e.g. 10%) of the vectors/vertices present in the L-1th layer.
-
-Then the greedy search routine operates as follows:
-
-* At the top layer (using an arbitrary vertex as an entry point), use the greedy local search routine on the k-ANN graph to get an approximate nearest neighbor at that layer.
-* Using the approximate nearest neighbor found in the previous layer as an entry point, find an approximate nearest neighbor in the next layer with the same method.
-* Repeat until the bottom-most layer is reached. Then use the entry point to find multiple nearest neighbors (e.g. top 10).
-
-
-## Usage
-
-There are three key parameters to set when constructing an HNSW index:
-
-* `metric`: Use an `l2` euclidean distance metric. We also support `dot` and `cosine` distance.
-* `m`: The number of neighbors to select for each vector in the HNSW graph.
-* `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph.
-
-
-We can combine the above concepts to understand how to build and query an HNSW index in LanceDB.
-
-### Construct index
-
-```python
-import lancedb
-import numpy as np
-uri = "/tmp/lancedb"
-db = lancedb.connect(uri)
-
-# Create 10,000 sample vectors
-data = [
-    {"vector": row, "item": f"item {i}"}
-    for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))
-]
-
-# Add the vectors to a table
-tbl = db.create_table("my_vectors", data=data)
-
-# Create and train the HNSW index for a 1536-dimensional vector
-# Make sure you have enough data in the table for an effective training step
-tbl.create_index(index_type=IVF_HNSW_SQ)
-
-```
-
-### Query the index
-
-```python
-# Search using a random 1536-dimensional embedding
-tbl.search(np.random.random((1536))) \
-    .limit(2) \
-    .to_pandas()
-```

docs/src/concepts/index_ivfpq.md

@@ -1,86 +0,0 @@
-# Understanding LanceDB's IVF-PQ index
-
-An ANN (Approximate Nearest Neighbors) index is a data structure that represents data in a way that makes it more efficient to search and retrieve. Using an ANN index is faster, but less accurate than kNN or brute force search because, in essence, the index is a lossy representation of the data.
-
-LanceDB is fundamentally different from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy.
-
-## IVF-PQ
-
-IVF-PQ is a composite index that combines inverted file index (IVF) and product quantization (PQ). The implementation in LanceDB provides several parameters to fine-tune the index's size, query throughput, latency and recall, which are described later in this section.
-
-### Product quantization
-
-Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search.
-
-Product quantization (PQ) works by dividing a large, high-dimensional vector of size into equally sized subvectors. Each subvector is assigned a "reproduction value" that maps to the nearest centroid of points for that subvector. The reproduction values are then assigned to a codebook using unique IDs, which can be used to reconstruct the original vector.
-
-![](../assets/ivfpq_pq_desc.png)
-
-It's important to remember that quantization is a *lossy process*, i.e., the reconstructed vector is not identical to the original vector. This results in a trade-off between the size of the index and the accuracy of the search results.
-
-As an example, consider starting with 128-dimensional vector consisting of 32-bit floats. Quantizing it to an 8-bit integer vector with 4 dimensions as in the image above, we can significantly reduce memory requirements.
-
-!!! example "Effect of quantization"
-
-    Original: `128 × 32 = 4096` bits
-    Quantized: `4 × 8 = 32` bits
-
-    Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial.
-
-### Inverted file index
-
-While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
-
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
-
-![](../assets/ivfpq_ivf_desc.webp)
-
-During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query.
-
-![](../assets/ivfpq_query_vector.webp)
-
-## Putting it all together
-
-We can combine the above concepts to understand how to build and query an IVF-PQ index in LanceDB.
-
-### Construct index
-
-There are three key parameters to set when constructing an IVF-PQ index:
-
-* `metric`: Use an `l2` euclidean distance metric. We also support `dot` and `cosine` distance.
-* `num_partitions`: The number of partitions in the IVF portion of the index.
-* `num_sub_vectors`: The number of sub-vectors that will be created during Product Quantization (PQ).
-
-In Python, the index can be created as follows:
-
-```python
-# Create and train the index for a 1536-dimensional vector
-# Make sure you have enough data in the table for an effective training step
-tbl.create_index(metric="l2", num_partitions=256, num_sub_vectors=96)
-```
-!!! note
-    `num_partitions`=256 and `num_sub_vectors`=96 does not work for every dataset. Those values needs to be adjusted for your particular dataset.
-
-The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See [here](../ann_indexes.md/#how-to-choose-num_partitions-and-num_sub_vectors-for-ivf_pq-index) for best practices on choosing these parameters.
-
-
-### Query the index
-
-```python
-# Search using a random 1536-dimensional embedding
-tbl.search(np.random.random((1536))) \
-    .limit(2) \
-    .nprobes(20) \
-    .refine_factor(10) \
-    .to_pandas()
-```
-
-The above query will perform a search on the table `tbl` using the given query vector, with the following parameters:
-
-* `limit`: The number of results to return
-* `nprobes`: The number of probes determines the distribution of vector space. While a higher number enhances search accuracy, it also results in slower performance. Typically, setting `nprobes` to cover 5–10% of the dataset proves effective in achieving high recall with minimal latency.
-* `refine_factor`: Refine the results by reading extra elements and re-ranking them in memory. A higher number makes the search more accurate but also slower (see the [FAQ](../faq.md#do-i-need-to-set-a-refine-factor-when-using-an-index) page for more details on this).
-* `to_pandas()`: Convert the results to a pandas DataFrame
-
-And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
-To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.

docs/src/concepts/storage.md

@@ -1,80 +0,0 @@
-# Storage
-
-LanceDB is among the only vector databases built on top of multiple modular components designed from the ground-up to be efficient on disk. This gives it the unique benefit of being flexible enough to support multiple storage backends, including local NVMe, EBS, EFS and many other third-party APIs that connect to the cloud.
-
-It is important to understand the tradeoffs between cost and latency for your specific application and use case. This section will help you understand the tradeoffs between the different storage backends.
-
-## Storage options
-
-We've prepared a simple diagram to showcase the thought process that goes into choosing a storage backend when using LanceDB OSS, Cloud or Enterprise.
-
-![](../assets/lancedb_storage_tradeoffs.png)
-
-When architecting your system, you'd typically ask yourself the following questions to decide on a storage option:
-
-1. **Latency**: How fast do I need results? What do the p50 and also p95 look like?
-2. **Scalability**: Can I scale up the amount of data and QPS easily?
-3. **Cost**: To serve my application, what’s the all-in cost of *both* storage and serving infra?
-4. **Reliability/Availability**: How does replication work? Is disaster recovery addressed?
-
-## Tradeoffs
-
-This section reviews the characteristics of each storage option in four dimensions: latency, scalability, cost and reliability.
-
-**We begin with the lowest cost option, and end with the lowest latency option.**
-
-### 1. S3 / GCS / Azure Blob Storage
-
-!!! tip "Lowest cost, highest latency"
-
-    - **Latency** ⇒ Has the highest latency. p95 latency is also substantially worse than p50. In general you get results in the order of several hundred milliseconds
-    - **Scalability** ⇒ Infinite on storage, however, QPS will be limited by S3 concurrency limits
-    - **Cost** ⇒ Lowest (order of magnitude cheaper than other options)
-    - **Reliability/Availability** ⇒ Highly available, as blob storage like S3 are critical infrastructure that form the backbone of the internet.
-
-Another important point to note is that LanceDB is designed to separate storage from compute, and the underlying Lance format stores the data in numerous immutable fragments. Due to these factors, LanceDB is a great storage option that addresses the _N + 1_ query problem. i.e., when a high query throughput is required, query processes can run in a stateless manner and be scaled up and down as needed.
-
-### 2. EFS / GCS Filestore / Azure File Storage
-
-!!! info "Moderately low cost, moderately low latency (<100ms)"
-
-    - **Latency** ⇒ Much better than object/blob storage but not as good as EBS/Local disk; < 100ms p95 achievable
-    - **Scalability** ⇒ High, but the bottleneck will be the IOPs limit, but when scaling you can provision multiple EFS volumes
-    - **Cost** ⇒ Significantly more expensive than S3 but still very cost effective compared to in-memory dbs. Inactive data in EFS is also automatically tiered to S3-level costs.
-    - **Reliability/Availability** ⇒ Highly available, as query nodes can go down without affecting EFS.  However, EFS does not provide replication / backup - this must be managed manually.
-
-A recommended best practice is to keep a copy of the data on S3 for disaster recovery scenarios. If any downtime is unacceptable, then you would need another EFS with a copy of the data. This is still much cheaper than EC2 instances holding multiple copies of the data.
-
-### 3. Third-party storage solutions
-
-Solutions like [MinIO](https://blog.min.io/lancedb-trusted-steed-against-data-complexity/), WekaFS, etc. that deliver S3 compatible API with much better performance than S3.
-
-!!! info "Moderately low cost, moderately low latency (<100ms)"
-
-    - **Latency** ⇒ Should be similar latency to EFS, better than S3 (<100ms)
-    - **Scalability** ⇒ Up to the solutions architect, who can add as many nodes to their MinIO or other third-party provider's cluster as needed
-    - **Cost** ⇒ Definitely higher than S3. The cost can be marginally higher than EFS until you get to maybe >10TB scale with high utilization
-    - **Reliability/Availability** ⇒ These are all shareable by lots of nodes, quality/cost of replication/backup depends on the vendor
-
-
-### 4. EBS / GCP Persistent Disk / Azure Managed Disk
-
-!!! info "Very low latency (<30ms), higher cost"
-
-    - **Latency** ⇒ Very good, pretty close to local disk. You’re looking at <30ms latency in most cases
-    - **Scalability** ⇒ EBS is not shareable between instances. If deployed via k8s, it can be shared between pods that live on the same instance, but beyond that you would need to shard data or make an additional copy
-    - **Cost** ⇒ Higher than EFS. There are some hidden costs to EBS as well if you’re paying for IO.
-    - **Reliability/Availability** ⇒ Not shareable between instances but can be shared between pods on the same instance. Survives instance termination. No automatic backups.
-
-Just like EFS, an EBS or persistent disk setup requires more manual work to manage data sharding, backups and capacity.
-
-### 5. Local disk (SSD/NVMe)
-
-!!! danger "Lowest latency (<10ms), highest cost"
-
-    - **Latency** ⇒ Lowest latency with modern NVMe drives, <10ms p95
-    - **Scalability** ⇒ Difficult to scale on cloud. Also need additional copies / sharding if QPS needs to be higher
-    - **Cost** ⇒ Highest cost; the main issue with keeping your application and storage tightly integrated is that it’s just not really possible to scale this up in cloud environments
-    - **Reliability/Availability** ⇒ If the instance goes down, so does your data. You have to be _very_ diligent about backing up your data
-
-As a rule of thumb, local disk should be your storage option if you require absolutely *crazy low* latency and you’re willing to do a bunch of data management work to make it happen.

docs/src/concepts/vector_search.md

@@ -1,36 +0,0 @@
-# Vector search
-
-Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search.
-
-Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a vector database like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups.
-
-![](../assets/vector-db-basics.png)
-
-## Embeddings
-
-Modern machine learning models can be trained to convert raw data into embeddings, represented as arrays (or vectors) of floating point numbers of fixed dimensionality. What makes embeddings useful in practice is that the position of an embedding in vector space captures some of the semantics of the data, depending on the type of model and how it was trained. Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar.
-
-Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model. Projecting the vectors' principal components in 2D space results in groups of vectors that represent similar concepts clustering together, as shown below.
-
-![](../assets/embedding_intro.png)
-
-## Indexes
-
-Embeddings for a given dataset are made searchable via an **index**. The index is constructed by using data structures that store the embeddings such that it's very efficient to perform scans and lookups on them. A key distinguishing feature of LanceDB is it uses a disk-based index: IVF-PQ, which is a variant of the Inverted File Index (IVF) that uses Product Quantization (PQ) to compress the embeddings.
-
-See the [IVF-PQ](./index_ivfpq.md) page for more details on how it works.
-
-## Brute force search
-
-The simplest way to perform vector search is to perform a brute force search, without an index, where the distance between the query vector and all the vectors in the database are computed, with the top-k closest vectors returned. This is equivalent to a k-nearest neighbours (kNN) search in vector space.
-
-![](../assets/knn_search.png)
-
-As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in.
-
-## Approximate nearest neighbour (ANN) search
-
-Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases.
-
-
-

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md

@@ -1,67 +0,0 @@
-# Imagebind embeddings
-We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
-
-This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"imagebind_huge"` | Name of the model. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
-
-Below is an example demonstrating how the API works:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry().get("imagebind").create()
-
-class ImageBindModel(LanceModel):
-    text: str
-    image_uri: str = func.SourceField()
-    audio_path: str
-    vector: Vector(func.ndims()) = func.VectorField()
-
-# add locally accessible image paths
-text_list=["A dog.", "A car", "A bird"]
-image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
-audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
-
-# Load data
-inputs = [
-    {"text": a, "audio_path": b, "image_uri": c}
-    for a, b, c in zip(text_list, audio_paths, image_paths)
-]
-
-#create table and add data
-table = db.create_table("img_bind", schema=ImageBindModel)
-table.add(inputs)
-```
-
-Now, we can search using any modality:
-
-#### image search
-```python
-query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
-actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "dog")
-```
-#### audio search
-
-```python
-query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
-actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "car")
-```
-#### Text search
-You can add any input query and fetch the result as follows:
-```python
-query = "an animal which flies and tweets" 
-actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "bird")
-```
-
-If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md

@@ -1,51 +0,0 @@
-# Jina Embeddings : Multimodal
-
-Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
-under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
-
-Usage Example:
-
-```python
-    import os
-    import requests
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-    import pandas as pd
-
-    os.environ['JINA_API_KEY'] = 'jina_*'
-
-    db = lancedb.connect("~/.lancedb")
-    func = get_registry().get("jina").create()
-
-
-    class Images(LanceModel):
-        label: str
-        image_uri: str = func.SourceField()  # image uri as the source
-        image_bytes: bytes = func.SourceField()  # image bytes as the source
-        vector: Vector(func.ndims()) = func.VectorField()  # vector column
-        vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
-
-
-    table = db.create_table("images", schema=Images)
-    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
-    uris = [
-        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
-        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
-        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
-        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
-        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
-        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
-    ]
-    # get each uri as bytes
-    image_bytes = [requests.get(uri).content for uri in uris]
-    table.add(
-      pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
-    )
-```

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md

@@ -1,82 +0,0 @@
-# OpenClip embeddings
-We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"ViT-B-32"` | The name of the model. |
-| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `batch_size` | `int` | `64` | The number of images to process in a batch. |
-| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
-
-This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
-
-!!! info
-    LanceDB supports ingesting images directly from accessible links.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry().get("open-clip").create()
-
-class Images(LanceModel):
-    label: str
-    image_uri: str = func.SourceField() # image uri as the source
-    image_bytes: bytes = func.SourceField() # image bytes as the source
-    vector: Vector(func.ndims()) = func.VectorField() # vector column 
-    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
-
-table = db.create_table("images", schema=Images)
-labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
-uris = [
-    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
-    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
-    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
-    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
-    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
-    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
-]
-# get each uri as bytes
-image_bytes = [requests.get(uri).content for uri in uris]
-table.add(
-    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
-)
-```
-Now we can search using text from both the default vector column and the custom vector column
-```python
-
-# text search
-actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
-print(actual.label) # prints "dog"
-
-frombytes = (
-    table.search("man's best friend", vector_column_name="vec_from_bytes")
-    .limit(1)
-    .to_pydantic(Images)[0]
-)
-print(frombytes.label)
-
-```
-
-Because we're using a multi-modal embedding function, we can also search using images
-
-```python
-# image search
-query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
-image_bytes = requests.get(query_image_uri).content
-query_image = Image.open(io.BytesIO(image_bytes))
-actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
-print(actual.label == "dog")
-
-# image search using a custom vector column
-other = (
-    table.search(query_image, vector_column_name="vec_from_bytes")
-    .limit(1)
-    .to_pydantic(Images)[0]
-)
-print(actual.label)
-
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md

@@ -1,51 +0,0 @@
-# AWS Bedrock Text Embedding Functions
-
-AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
-You can do so by using `awscli` and also add your session_token:
-```shell
-aws configure
-aws configure set aws_session_token "<your_session_token>"
-```
-to ensure that the credentials are set up correctly, you can run the following command:
-```shell
-aws sts get-caller-identity
-```
-
-Supported Embedding modelIDs are:
-* `amazon.titan-embed-text-v1`
-* `cohere.embed-english-v3`
-* `cohere.embed-multilingual-v3`
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
-| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
-| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
-| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
-| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
-| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
-| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
-
-Usage Example:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-import pandas as pd
-
-model = get_registry().get("bedrock-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("tmp_path")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md

@@ -1,63 +0,0 @@
-# Cohere Embeddings
-
-Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
-You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
-
-Supported models are:
-
-- embed-english-v3.0
-- embed-multilingual-v3.0
-- embed-english-light-v3.0
-- embed-multilingual-light-v3.0
-- embed-english-v2.0
-- embed-english-light-v2.0
-- embed-multilingual-v2.0
-
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|--------|---------|
-| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
-| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
-| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
-
-Cohere supports following input types:
-
-| Input Type               | Description                          |
-|-------------------------|---------------------------------------|
-| "`search_document`"     | Used for embeddings stored in a vector|
-|                         | database for search use-cases.        |
-| "`search_query`"        | Used for embeddings of search queries |
-|                         | run against a vector DB               |
-| "`semantic_similarity`" | Specifies the given text will be used |
-|                         | for Semantic Textual Similarity (STS) |
-| "`classification`"      | Used for embeddings passed through a  |
-|                         | text classifier.                      |
-| "`clustering`"          | Used for the embeddings run through a |
-|                         | clustering algorithm                  |
-
-Usage Example:
-    
-```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    cohere = EmbeddingFunctionRegistry
-        .get_instance()
-        .get("cohere")
-        .create(name="embed-multilingual-v2.0")
-
-    class TextModel(LanceModel):
-        text: str = cohere.SourceField()
-        vector: Vector(cohere.ndims()) =  cohere.VectorField()
-
-    data = [ { "text": "hello world" },
-            { "text": "goodbye world" }]
-
-    db = lancedb.connect("~/.lancedb")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md

@@ -1,35 +0,0 @@
-# Gemini Embeddings
-With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
-The Gemini Embedding Model API supports various task types:
-
-| Task Type               | Description                                                                                                                                                |
-|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
-| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically proided by Embeddings API |
-| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
-| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
-| "`clusering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
-
-
-Usage Example:
-
-```python
-import lancedb
-import pandas as pd
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-
-model = get_registry().get("gemini-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md

@@ -1,24 +0,0 @@
-# Huggingface embedding models
-We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. 
-
-Example usage - 
-```python
-import lancedb
-import pandas as pd
-
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-model = get_registry().get("huggingface").create(name='facebook/bart-base')
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
-table = db.create_table("greets", schema=Words)
-table.add(df)
-query = "old greeting"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md

@@ -1,75 +0,0 @@
-# IBM watsonx.ai Embeddings
-
-Generate text embeddings using IBM's watsonx.ai platform.
-
-## Supported Models
-
-You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are:
-
-- `ibm/slate-125m-english-rtrvr`
-- `ibm/slate-30m-english-rtrvr`
-- `sentence-transformers/all-minilm-l12-v2`
-- `intfloat/multilingual-e5-large`
-
-## Parameters
-
-The following parameters can be passed to the `create` method:
-
-| Parameter  | Type     | Default Value                    | Description                                               |
-|------------|----------|----------------------------------|-----------------------------------------------------------|
-| name       | str      | "ibm/slate-125m-english-rtrvr"   | The model ID of the watsonx.ai model to use               |
-| api_key    | str      | None                             | Optional IBM Cloud API key (or set `WATSONX_API_KEY`)     |
-| project_id | str      | None                             | Optional watsonx project ID (or set `WATSONX_PROJECT_ID`) |
-| url        | str      | None                             | Optional custom URL for the watsonx.ai instance           |
-| params     | dict     | None                             | Optional additional parameters for the embedding model    |
-
-## Usage Example
-
-First, the watsonx.ai library is an optional dependency, so must be installed seperately:
-
-```
-pip install ibm-watsonx-ai
-```
-
-Optionally set environment variables (if not passing credentials to `create` directly):
-
-```sh
-export WATSONX_API_KEY="YOUR_WATSONX_API_KEY"
-export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID"
-```
-
-```python
-import os
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import EmbeddingFunctionRegistry
-
-watsonx_embed = EmbeddingFunctionRegistry
-  .get_instance()
-  .get("watsonx")
-  .create(
-    name="ibm/slate-125m-english-rtrvr",
-    # Uncomment and set these if not using environment variables
-    # api_key="your_api_key_here",
-    # project_id="your_project_id_here",
-    # url="your_watsonx_url_here",
-    # params={...},
-  )
-
-class TextModel(LanceModel):
-    text: str = watsonx_embed.SourceField()
-    vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"},
-]
-
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
-
-tbl.add(data)
-
-rs = tbl.search("hello").limit(1).to_pandas()
-print(rs)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md

@@ -1,50 +0,0 @@
-# Instructor Embeddings
-[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
-
-If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
-
-!!! info
-    Represent the `domain` `text_type` for `task_objective`:
-
-    * `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
-    * `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
-    * `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
-
-More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
-
-| Argument | Type | Default | Description |
-|---|---|---|---|
-| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
-| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
-| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
-| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
-| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
-| `quantize` | `bool` | `False` | Whether to quantize the model |
-| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
-| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
-
-
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
-
-instructor = get_registry().get("instructor").create(
-                            source_instruction="represent the docuement for retreival",
-                            query_instruction="represent the document for retreiving the most similar documents"
-                            )
-
-class Schema(LanceModel):
-    vector: Vector(instructor.ndims()) = instructor.VectorField()
-    text: str = instructor.SourceField()
-
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-
-texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
-        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
-        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
-
-tbl.add(texts)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md

@@ -1,39 +0,0 @@
-# Jina Embeddings
-
-Jina embeddings are used to generate embeddings for text and image data.
-You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
-
-You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
-
-Usage Example:
-
-```python
-    import os
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    os.environ['JINA_API_KEY'] = 'jina_*'
-
-    jina_embed = EmbeddingFunctionRegistry.get_instance().get("jina").create(name="jina-embeddings-v2-base-en")
-
-
-    class TextModel(LanceModel):
-        text: str = jina_embed.SourceField()
-        vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
-
-
-    data = [{"text": "hello world"},
-            {"text": "goodbye world"}]
-
-    db = lancedb.connect("~/.lancedb-2")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md

@@ -1,37 +0,0 @@
-# Ollama embeddings
-
-Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
-
-- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
-- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
-
-| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
-|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
-| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
-| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
-| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
-| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
-| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("ollama").create(name="nomic-embed-text")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md

@@ -1,35 +0,0 @@
-# OpenAI embeddings
-
-LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
-| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
-| `use_azure` | bool | `False` | Set true to use Azure OpenAPI SDK |
-
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md

@@ -1,174 +0,0 @@
-# Sentence transformers
-Allows you to set parameters when registering a `sentence-transformers` object.
-
-!!! info
-    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
-| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
-| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
-| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository |
-
-
-??? "Check out available sentence-transformer models here!"
-    ```markdown
-    - sentence-transformers/all-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-mpnet-base-v2
-    - sentence-transformers/gtr-t5-base
-    - sentence-transformers/LaBSE
-    - sentence-transformers/all-MiniLM-L6-v2
-    - sentence-transformers/bert-base-nli-max-tokens
-    - sentence-transformers/bert-base-nli-mean-tokens
-    - sentence-transformers/bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
-    - sentence-transformers/bert-large-nli-cls-token
-    - sentence-transformers/bert-large-nli-max-tokens
-    - sentence-transformers/bert-large-nli-mean-tokens
-    - sentence-transformers/bert-large-nli-stsb-mean-tokens
-    - sentence-transformers/distilbert-base-nli-max-tokens
-    - sentence-transformers/distilbert-base-nli-mean-tokens
-    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
-    - sentence-transformers/distilroberta-base-msmarco-v1
-    - sentence-transformers/distilroberta-base-msmarco-v2
-    - sentence-transformers/nli-bert-base-cls-pooling
-    - sentence-transformers/nli-bert-base-max-pooling
-    - sentence-transformers/nli-bert-base
-    - sentence-transformers/nli-bert-large-cls-pooling
-    - sentence-transformers/nli-bert-large-max-pooling
-    - sentence-transformers/nli-bert-large
-    - sentence-transformers/nli-distilbert-base-max-pooling
-    - sentence-transformers/nli-distilbert-base
-    - sentence-transformers/nli-roberta-base
-    - sentence-transformers/nli-roberta-large
-    - sentence-transformers/roberta-base-nli-mean-tokens
-    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
-    - sentence-transformers/roberta-large-nli-mean-tokens
-    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
-    - sentence-transformers/stsb-bert-base
-    - sentence-transformers/stsb-bert-large
-    - sentence-transformers/stsb-distilbert-base
-    - sentence-transformers/stsb-roberta-base
-    - sentence-transformers/stsb-roberta-large
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
-    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
-    - sentence-transformers/bert-base-nli-cls-token
-    - sentence-transformers/all-distilroberta-v1
-    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
-    - sentence-transformers/multi-qa-distilbert-cos-v1
-    - sentence-transformers/multi-qa-distilbert-dot-v1
-    - sentence-transformers/multi-qa-mpnet-base-cos-v1
-    - sentence-transformers/multi-qa-mpnet-base-dot-v1
-    - sentence-transformers/nli-distilroberta-base-v2
-    - sentence-transformers/all-MiniLM-L6-v1
-    - sentence-transformers/all-mpnet-base-v1
-    - sentence-transformers/all-mpnet-base-v2
-    - sentence-transformers/all-roberta-large-v1
-    - sentence-transformers/allenai-specter
-    - sentence-transformers/average_word_embeddings_glove.6B.300d
-    - sentence-transformers/average_word_embeddings_glove.840B.300d
-    - sentence-transformers/average_word_embeddings_komninos
-    - sentence-transformers/average_word_embeddings_levy_dependency
-    - sentence-transformers/clip-ViT-B-32-multilingual-v1
-    - sentence-transformers/clip-ViT-B-32
-    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
-    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
-    - sentence-transformers/distilroberta-base-paraphrase-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v2
-    - sentence-transformers/distiluse-base-multilingual-cased
-    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
-    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
-    - sentence-transformers/gtr-t5-large
-    - sentence-transformers/gtr-t5-xl
-    - sentence-transformers/gtr-t5-xxl
-    - sentence-transformers/msmarco-MiniLM-L-12-v3
-    - sentence-transformers/msmarco-MiniLM-L-6-v3
-    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
-    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
-    - sentence-transformers/msmarco-bert-base-dot-v5
-    - sentence-transformers/msmarco-bert-co-condensor
-    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
-    - sentence-transformers/msmarco-distilbert-base-tas-b
-    - sentence-transformers/msmarco-distilbert-base-v2
-    - sentence-transformers/msmarco-distilbert-base-v3
-    - sentence-transformers/msmarco-distilbert-base-v4
-    - sentence-transformers/msmarco-distilbert-cos-v5
-    - sentence-transformers/msmarco-distilbert-dot-v5
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
-    - sentence-transformers/msmarco-distilroberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-ance-firstp
-    - sentence-transformers/msmarco-roberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-v3
-    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-    - sentence-transformers/nli-mpnet-base-v2
-    - sentence-transformers/nli-roberta-base-v2
-    - sentence-transformers/nq-distilbert-base-v1
-    - sentence-transformers/paraphrase-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-MiniLM-L3-v2
-    - sentence-transformers/paraphrase-MiniLM-L6-v2
-    - sentence-transformers/paraphrase-TinyBERT-L6-v2
-    - sentence-transformers/paraphrase-albert-base-v2
-    - sentence-transformers/paraphrase-albert-small-v2
-    - sentence-transformers/paraphrase-distilroberta-base-v1
-    - sentence-transformers/paraphrase-distilroberta-base-v2
-    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
-    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
-    - sentence-transformers/quora-distilbert-base
-    - sentence-transformers/quora-distilbert-multilingual
-    - sentence-transformers/sentence-t5-base
-    - sentence-transformers/sentence-t5-large
-    - sentence-transformers/sentence-t5-xxl
-    - sentence-transformers/sentence-t5-xl
-    - sentence-transformers/stsb-distilroberta-base-v2
-    - sentence-transformers/stsb-mpnet-base-v2
-    - sentence-transformers/stsb-roberta-base-v2
-    - sentence-transformers/stsb-xlm-r-multilingual
-    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
-    - sentence-transformers/clip-ViT-L-14
-    - sentence-transformers/clip-ViT-B-16
-    - sentence-transformers/use-cmlm-multilingual
-    - sentence-transformers/all-MiniLM-L12-v1
-    ```
-
-!!! info
-    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
-    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
-
-!!! note "BAAI Embeddings example"
-    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-    )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
-

docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md

@@ -1,51 +0,0 @@
-# VoyageAI Embeddings
-
-Voyage AI provides cutting-edge embedding and rerankers.
-
-
-Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
-You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API.
-
-Supported models are:
-
-- voyage-3
-- voyage-3-lite
-- voyage-finance-2
-- voyage-multilingual-2
-- voyage-law-2
-- voyage-code-2
-
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|--------|---------|
-| `name` | `str` | `None` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
-| `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. |
-| `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. |
-
-
-Usage Example:
-    
-```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    voyageai = EmbeddingFunctionRegistry
-        .get_instance()
-        .get("voyageai")
-        .create(name="voyage-3")
-
-    class TextModel(LanceModel):
-        text: str = voyageai.SourceField()
-        vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
-
-    data = [ { "text": "hello world" },
-            { "text": "goodbye world" }]
-
-    db = lancedb.connect("~/.lancedb")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```

docs/src/embeddings/custom_embedding_function.md

@@ -1,248 +0,0 @@
-To use your own custom embedding function, you can follow these 2 simple steps:
-
-1. Create your embedding function by implementing the `EmbeddingFunction` interface
-2. Register your embedding function in the global `EmbeddingFunctionRegistry`.
-
-Let us see how this looks like in action.
-
-![](../assets/embeddings_api.png)
-
-`EmbeddingFunction` and `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embedding function, you don't have to worry about the finer details - simply focus on setting up the model and leave the rest to LanceDB.
-
-## `TextEmbeddingFunction` interface
-
-There is another optional layer of abstraction available: `TextEmbeddingFunction`. You can use this abstraction if your model isn't multi-modal in nature and only needs to operate on text. In such cases, both the source and vector fields will have the same work for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
-
-Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
-
-
-=== "Python"
-
-    ```python
-    from lancedb.embeddings import register
-    from lancedb.util import attempt_import_or_raise
-
-    @register("sentence-transformers")
-    class SentenceTransformerEmbeddings(TextEmbeddingFunction):
-        name: str = "all-MiniLM-L6-v2"
-        # set more default instance vars like device, etc.
-
-        def __init__(self, **kwargs):
-            super().__init__(**kwargs)
-            self._ndims = None
-
-        def generate_embeddings(self, texts):
-            return self._embedding_model().encode(list(texts), ...).tolist()
-
-        def ndims(self):
-            if self._ndims is None:
-                self._ndims = len(self.generate_embeddings("foo")[0])
-            return self._ndims
-
-        @cached(cache={})
-        def _embedding_model(self):
-            return sentence_transformers.SentenceTransformer(name)
-    ```
-
-=== "TypeScript"
-
-    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:imports"
-
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:embedding_impl"
-    ```
-
-
-This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and default settings.
-
-!!! danger "Use sensitive keys to prevent leaking secrets"
-    To prevent leaking secrets, such as API keys, you should add any sensitive
-    parameters of an embedding function to the output of the
-    [sensitive_keys()][lancedb.embeddings.base.EmbeddingFunction.sensitive_keys] /
-    [getSensitiveKeys()](../../js/namespaces/embedding/classes/EmbeddingFunction/#getsensitivekeys)
-    method. This prevents users from accidentally instantiating the embedding
-    function with hard-coded secrets.
-
-Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
-
-=== "Python"
-
-    ```python
-    from lancedb.pydantic import LanceModel, Vector
-
-    registry = EmbeddingFunctionRegistry.get_instance()
-    stransformer = registry.get("sentence-transformers").create()
-
-    class TextModelSchema(LanceModel):
-        vector: Vector(stransformer.ndims) = stransformer.VectorField()
-        text: str = stransformer.SourceField()
-
-    tbl = db.create_table("table", schema=TextModelSchema)
-
-    tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
-    result = tbl.search("world").limit(5)
-    ```
-
-=== "TypeScript"
-
-    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:call_custom_function"
-    ```
-
-!!! note
-
-    You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
-
-## Multi-modal embedding function example
-You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support.
-
-=== "Python"
-
-    LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
-
-    ```python
-    @register("open-clip")
-    class OpenClipEmbeddings(EmbeddingFunction):
-        name: str = "ViT-B-32"
-        pretrained: str = "laion2b_s34b_b79k"
-        device: str = "cpu"
-        batch_size: int = 64
-        normalize: bool = True
-        _model = PrivateAttr()
-        _preprocess = PrivateAttr()
-        _tokenizer = PrivateAttr()
-
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
-            model, _, preprocess = open_clip.create_model_and_transforms(
-                self.name, pretrained=self.pretrained
-            )
-            model.to(self.device)
-            self._model, self._preprocess = model, preprocess
-            self._tokenizer = open_clip.get_tokenizer(self.name)
-            self._ndims = None
-
-        def ndims(self):
-            if self._ndims is None:
-                self._ndims = self.generate_text_embeddings("foo").shape[0]
-            return self._ndims
-
-        def compute_query_embeddings(
-            self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
-        ) -> List[np.ndarray]:
-            """
-            Compute the embeddings for a given user query
-
-            Parameters
-            ----------
-            query : Union[str, PIL.Image.Image]
-                The query to embed. A query can be either text or an image.
-            """
-            if isinstance(query, str):
-                return [self.generate_text_embeddings(query)]
-            else:
-                PIL = attempt_import_or_raise("PIL", "pillow")
-                if isinstance(query, PIL.Image.Image):
-                    return [self.generate_image_embedding(query)]
-                else:
-                    raise TypeError("OpenClip supports str or PIL Image as query")
-
-        def generate_text_embeddings(self, text: str) -> np.ndarray:
-            torch = attempt_import_or_raise("torch")
-            text = self.sanitize_input(text)
-            text = self._tokenizer(text)
-            text.to(self.device)
-            with torch.no_grad():
-                text_features = self._model.encode_text(text.to(self.device))
-                if self.normalize:
-                    text_features /= text_features.norm(dim=-1, keepdim=True)
-                return text_features.cpu().numpy().squeeze()
-
-        def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-            """
-            Sanitize the input to the embedding function.
-            """
-            if isinstance(images, (str, bytes)):
-                images = [images]
-            elif isinstance(images, pa.Array):
-                images = images.to_pylist()
-            elif isinstance(images, pa.ChunkedArray):
-                images = images.combine_chunks().to_pylist()
-            return images
-
-        def compute_source_embeddings(
-            self, images: IMAGES, *args, **kwargs
-        ) -> List[np.array]:
-            """
-            Get the embeddings for the given images
-            """
-            images = self.sanitize_input(images)
-            embeddings = []
-            for i in range(0, len(images), self.batch_size):
-                j = min(i + self.batch_size, len(images))
-                batch = images[i:j]
-                embeddings.extend(self._parallel_get(batch))
-            return embeddings
-
-        def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
-            """
-            Issue concurrent requests to retrieve the image data
-            """
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                futures = [
-                    executor.submit(self.generate_image_embedding, image)
-                    for image in images
-                ]
-                return [future.result() for future in futures]
-
-        def generate_image_embedding(
-            self, image: Union[str, bytes, "PIL.Image.Image"]
-        ) -> np.ndarray:
-            """
-            Generate the embedding for a single image
-
-            Parameters
-            ----------
-            image : Union[str, bytes, PIL.Image.Image]
-                The image to embed. If the image is a str, it is treated as a uri.
-                If the image is bytes, it is treated as the raw image bytes.
-            """
-            torch = attempt_import_or_raise("torch")
-            # TODO handle retry and errors for https
-            image = self._to_pil(image)
-            image = self._preprocess(image).unsqueeze(0)
-            with torch.no_grad():
-                return self._encode_and_normalize_image(image)
-
-        def _to_pil(self, image: Union[str, bytes]):
-            PIL = attempt_import_or_raise("PIL", "pillow")
-            if isinstance(image, bytes):
-                return PIL.Image.open(io.BytesIO(image))
-            if isinstance(image, PIL.Image.Image):
-                return image
-            elif isinstance(image, str):
-                parsed = urlparse.urlparse(image)
-                # TODO handle drive letter on windows.
-                if parsed.scheme == "file":
-                    return PIL.Image.open(parsed.path)
-                elif parsed.scheme == "":
-                    return PIL.Image.open(image if os.name == "nt" else parsed.path)
-                elif parsed.scheme.startswith("http"):
-                    return PIL.Image.open(io.BytesIO(url_retrieve(image)))
-                else:
-                    raise NotImplementedError("Only local and http(s) urls are supported")
-
-        def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
-            """
-            encode a single image tensor and optionally normalize the output
-            """
-            image_features = self._model.encode_image(image_tensor)
-            if self.normalize:
-                image_features /= image_features.norm(dim=-1, keepdim=True)
-            return image_features.cpu().numpy().squeeze()
-    ```
-
-=== "TypeScript"
-
-    Coming Soon! See this [issue](https://github.com/lancedb/lancedb/issues/1482) to track the status!

docs/src/embeddings/default_embedding_functions.md

@@ -1,86 +0,0 @@
-# 📚 Available Embedding Models
-
-There are various embedding functions available out of the box with LanceDB to manage your embeddings implicitly. We're actively working on adding other popular embedding APIs and models. 🚀
-
-Before jumping on the list of available models, let's understand how to get an embedding model initialized and configured to use in our code: 
-
-!!! example "Example usage"
-    ```python
-    model = get_registry()
-              .get("openai")
-              .create(name="text-embedding-ada-002")
-    ```
-
-Now let's understand the above syntax: 
-```python
-model = get_registry().get("model_id").create(...params)
-```
-**This👆 line effectively creates a configured instance of an `embedding function` with `model` of choice that is ready for use.**
-
-- `get_registry()` :  This function call returns an instance of a `EmbeddingFunctionRegistry` object. This registry manages the registration and retrieval of embedding functions.
-
-- `.get("model_id")` : This method call on the registry object and retrieves the **embedding models functions** associated with the `"model_id"` (1) .
-    { .annotate }
-
-    1.  Hover over the names in table below to find out the `model_id` of different embedding functions.
-
-- `.create(...params)` : This method call is on the object returned by the `get` method. It instantiates an embedding model function using the **specified parameters**. 
-
-??? question "What parameters does the `.create(...params)` method accepts?"
-    **Checkout the documentation of specific embedding models (links in the table below👇) to know what parameters it takes**.
-
-!!! tip "Moving on"
-    Now that we know how to get the **desired embedding model** and use it in our code, let's explore the comprehensive **list** of embedding models **supported by LanceDB**, in the tables below.
-
-## Text Embedding Functions 📝 
-These functions are registered by default to handle text embeddings.
-
-- 🔄 **Embedding functions** have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with **exponential backoff**. 
-
-- 🌕 Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7. 
-
-🌟 **Available Text Embeddings**
-
-| **Embedding** :material-information-outline:{ title="Hover over the name to find out the model_id" } | **Description** | **Documentation** |
-|-----------|-------------|---------------|
-| [**Sentence Transformers**](available_embedding_models/text_embedding_functions/sentence_transformers.md "sentence-transformers")  | 🧠 **SentenceTransformers** is a Python framework for state-of-the-art sentence, text, and image embeddings. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/sbert_2.png" alt="Sentence Transformers Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/sentence_transformers.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Huggingface Models**](available_embedding_models/text_embedding_functions/huggingface_embedding.md "huggingface") |🤗 We offer support for all **Huggingface** models. The default model is `colbert-ir/colbertv2.0`. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/hugging_face.png" alt="Huggingface Icon" width="130" height="35">](available_embedding_models/text_embedding_functions/huggingface_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Ollama Embeddings**](available_embedding_models/text_embedding_functions/ollama_embedding.md "ollama") | 🔍 Generate embeddings via the **Ollama** python library. Ollama supports embedding models, making it possible to build RAG apps. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/Ollama.png" alt="Ollama Icon" width="110" height="35">](available_embedding_models/text_embedding_functions/ollama_embedding.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**OpenAI Embeddings**](available_embedding_models/text_embedding_functions/openai_embedding.md "openai")| 🔑 **OpenAI’s** text embeddings measure the relatedness of text strings. **LanceDB** supports state-of-the-art embeddings from OpenAI. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/openai.png" alt="OpenAI Icon" width="100" height="35">](available_embedding_models/text_embedding_functions/openai_embedding.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Instructor Embeddings**](available_embedding_models/text_embedding_functions/instructor_embedding.md "instructor") | 📚 **Instructor**: An instruction-finetuned text embedding model that can generate text embeddings tailored to any task and domains by simply providing the task instruction, without any finetuning. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/instructor_embedding.png" alt="Instructor Embedding Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/instructor_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Gemini Embeddings**](available_embedding_models/text_embedding_functions/gemini_embedding.md "gemini-text") | 🌌 Google’s Gemini API generates state-of-the-art embeddings for words, phrases, and sentences. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/gemini.png" alt="Gemini Icon" width="95" height="35">](available_embedding_models/text_embedding_functions/gemini_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Cohere Embeddings**](available_embedding_models/text_embedding_functions/cohere_embedding.md "cohere") | 💬 This will help you get started with **Cohere** embedding models using LanceDB. Using cohere API requires cohere package. Install it via `pip`. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/cohere.png" alt="Cohere Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/cohere_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Jina Embeddings**](available_embedding_models/text_embedding_functions/jina_embedding.md "jina") | 🔗 World-class embedding models to improve your search and RAG systems. You will need **jina api key**. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="Jina Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/jina_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [ **AWS Bedrock Functions**](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md "bedrock-text") | ☁️ AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/aws_bedrock.png" alt="AWS Bedrock Icon" width="120" height="35">](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**IBM Watsonx.ai**](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md "watsonx") | 💡 Generate text embeddings using IBM's watsonx.ai platform. **Note**: watsonx.ai library is an optional dependency. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/watsonx.png" alt="Watsonx Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md) |
-| [**VoyageAI Embeddings**](available_embedding_models/text_embedding_functions/voyageai_embedding.md "voyageai") | 🌕 Voyage AI provides cutting-edge embedding and rerankers. This will help you get started with **VoyageAI** embedding models using LanceDB. Using voyageai API requires voyageai package. Install it via `pip`. | [<img src="https://www.voyageai.com/logo.svg" alt="VoyageAI Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/voyageai_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-
-
-
-[st-key]: "sentence-transformers"
-[hf-key]: "huggingface"
-[ollama-key]: "ollama"
-[openai-key]: "openai"
-[instructor-key]: "instructor"
-[gemini-key]: "gemini-text"
-[cohere-key]: "cohere"
-[jina-key]: "jina"
-[aws-key]: "bedrock-text"
-[watsonx-key]: "watsonx"
-[voyageai-key]: "voyageai"
-
-
-## Multi-modal Embedding Functions🖼️ 
-
-Multi-modal embedding functions allow you to query your table using both images and text. 💬🖼️
-
-🌐 **Available Multi-modal Embeddings**
-
-| Embedding :material-information-outline:{ title="Hover over the name to find out the model_id" }  | Description | Documentation  |
-|-----------|-------------|---------------|
-| [**OpenClip Embeddings**](available_embedding_models/multimodal_embedding_functions/openclip_embedding.md "open-clip") | 🎨 We support CLIP model embeddings using the open source alternative, **open-clip** which supports various customizations. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/openclip_github.png" alt="openclip Icon" width="150" height="35">](available_embedding_models/multimodal_embedding_functions/openclip_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    
-| [**Imagebind Embeddings**](available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md "imageind") | 🌌  We have support for **imagebind model embeddings**. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/imagebind_meta.png" alt="imagebind Icon" width="150" height="35">](available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    
-| [**Jina Multi-modal Embeddings**](available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md "jina") | 🔗 **Jina embeddings** can also be used to embed both **text** and **image** data, only some of the models support image data and you can check the detailed documentation. 👉 | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="jina Icon" width="90" height="35">](available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md) |
-
-!!! note
-    If you'd like to request support for additional **embedding functions**, please feel free to open an issue on our LanceDB [GitHub issue page](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/embedding_functions.md

@@ -1,206 +0,0 @@
-Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions can themselves be thought of as key part of the data processing pipeline that each request has to be passed through. The assumption here is: after initial setup, these components and the underlying methodology are not expected to change for a particular project.
-
-For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
-
-!!! Note "Embedding functions on LanceDB cloud"
-    When using embedding functions with LanceDB cloud, the embeddings will be generated on the source device and sent to the cloud. This means that the source device must have the necessary resources to generate the embeddings.
-
-!!! warning
-    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself.
-    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function
-    and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
-    table metadata and have LanceDB automatically take care of regenerating the embeddings.
-
-
-## 1. Define the embedding function
-
-=== "Python"
-    In the LanceDB python SDK, we define a global embedding function registry with
-    many different embedding models and even more coming soon.
-    Here's let's an implementation of CLIP as example.
-
-    ```python
-    from lancedb.embeddings import get_registry
-
-    registry = get_registry()
-    clip = registry.get("open-clip").create()
-    ```
-
-    You can also define your own embedding function by implementing the `EmbeddingFunction`
-    abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
-
-=== "TypeScript"
-    In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
-    embedding function is available.
-
-    ```javascript
-    import * as lancedb from '@lancedb/lancedb'
-    import { getRegistry } from '@lancedb/lancedb/embeddings'
-
-    // You need to provide an OpenAI API key
-    const apiKey = "sk-..."
-    // The embedding function will create embeddings for the 'text' column
-    const func = getRegistry().get("openai").create({apiKey})
-    ```
-=== "Rust"
-    In the Rust SDK, the choices are more limited. For now, only the OpenAI
-    embedding function is available. But unlike the Python and TypeScript SDKs, you need manually register the OpenAI embedding function.
-
-    ```toml
-    // Make sure to include the `openai` feature
-    [dependencies]
-    lancedb = {version = "*", features = ["openai"]}
-    ```
-
-    ```rust
-    --8<-- "rust/lancedb/examples/openai.rs:imports"
-    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-## 2. Define the data model or schema
-
-=== "Python"
-    The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
-
-    ```python
-    class Pets(LanceModel):
-        vector: Vector(clip.ndims()) = clip.VectorField()
-        image_uri: str = clip.SourceField()
-    ```
-
-    `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
-
-=== "TypeScript"
-
-    For the TypeScript SDK, a schema can be inferred from input data, or an explicit
-    Arrow schema can be provided.
-
-## 3. Create table and add data
-
-Now that we have chosen/defined our embedding function and the schema,
-we can create the table and ingest data without needing to explicitly generate
-the embeddings at all:
-
-=== "Python"
-    ```python
-    db = lancedb.connect("~/lancedb")
-    table = db.create_table("pets", schema=Pets)
-
-    table.add([{"image_uri": u} for u in uris])
-    ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.test.ts:embedding_function"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const db = await lancedb.connect("data/sample-lancedb");
-        const data = [
-            { text: "pepperoni"},
-            { text: "pineapple"}
-        ]
-
-        const table = await db.createTable("vectors", data, embedding)
-        ```
-
-## 4. Querying your table
-Not only can you forget about the embeddings during ingestion, you also don't
-need to worry about it when you query the table:
-
-=== "Python"
-
-    Our OpenCLIP query embedding function supports querying via both text and images:
-
-    ```python
-    results = (
-        table.search("dog")
-            .limit(10)
-            .to_pandas()
-    )
-    ```
-
-    Or we can search using an image:
-
-    ```python
-    p = Path("path/to/images/samoyed_100.jpg")
-    query_image = Image.open(p)
-    results = (
-        table.search(query_image)
-            .limit(10)
-            .to_pandas()
-    )
-    ```
-
-    Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        const results = await table.search("What's the best pizza topping?")
-            .limit(10)
-            .toArray()
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const results = await table
-            .search("What's the best pizza topping?")
-            .limit(10)
-            .execute()
-        ```
-
-    The above snippet returns an array of records with the top 10 nearest neighbors to the query.
-
----
-
-## Rate limit Handling
-`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default, the maximum retires is set to 7. You can tune it by setting it to a different number, or disable it by setting it to 0.
-
-An example of how to do this is shown below:
-
-```python
-clip = registry.get("open-clip").create() # Defaults to 7 max retries
-clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
-clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
-```
-
-!!! note
-    Embedding functions can also fail due to other errors that have nothing to do with rate limits.
-    This is why the error is also logged.
-
-## Some fun with Pydantic
-
-LanceDB is integrated with Pydantic, which was used in the example above to define the schema in Python. It's also used behind the scenes by the embedding function API to ingest useful information as table metadata.
-
-You can also use the integration for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let's define a utility function to plot the image.
-
-```python
-class Pets(LanceModel):
-    vector: Vector(clip.ndims()) = clip.VectorField()
-    image_uri: str = clip.SourceField()
-
-    @property
-    def image(self):
-        return Image.open(self.image_uri)
-```
-Now, you can covert your search results to a Pydantic model and use this property.
-
-```python
-rs = table.search(query_image).limit(3).to_pydantic(Pets)
-rs[2].image
-```
-
-![](../assets/dog_clip_output.png)
-
-Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
-let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -1,132 +0,0 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio.
-This makes them a very powerful tool for machine learning practitioners.
-However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs
-(both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
-
-LanceDB supports 3 methods of working with embeddings.
-
-1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
-2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
-3. You can define your own [custom embedding function](./custom_embedding_function.md)
-   that extends the default embedding functions.
-
-For python users, there is also a legacy [with_embeddings API](./legacy.md).
-It is retained for compatibility and will be removed in a future version.
-
-## Quickstart
-
-To get started with embeddings, you can use the built-in embedding functions.
-
-### OpenAI Embedding function
-
-LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
-
-=== "Python"
-
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-    class Words(LanceModel):
-        text: str = func.SourceField()
-        vector: Vector(func.ndims()) = func.VectorField()
-
-    table = db.create_table("words", schema=Words, mode="overwrite")
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-        )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-
-=== "TypeScript"
-
-    ```typescript
-    --8<--- "nodejs/examples/embedding.test.ts:imports"
-    --8<--- "nodejs/examples/embedding.test.ts:openai_embeddings"
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<--- "rust/lancedb/examples/openai.rs:imports"
-    --8<--- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-=== "Python"
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-    )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-
-=== "TypeScript"
-
-    Coming Soon!
-
-=== "Rust"
-
-    Coming Soon!
-
-### Embedding function with LanceDB cloud
-Embedding functions are now supported on LanceDB cloud. The embeddings will be generated on the source device and sent to the cloud. This means that the source device must have the necessary resources to generate the embeddings. Here's an example using the OpenAI embedding function:
-
-```python
-import os
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-os.environ['OPENAI_API_KEY'] = "..."
-
-db = lancedb.connect(
-  uri="db://....",
-  api_key="sk_...",
-  region="us-east-1"
-)
-func = get_registry().get("openai").create()
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words)
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/legacy.md

@@ -1,99 +0,0 @@
-The legacy `with_embeddings` API is for Python only and is deprecated.
-
-### Hugging Face
-
-The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
-library, which can be installed via pip.
-
-```bash
-pip install sentence-transformers
-```
-
-The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
-for a given document.
-
-```python
-from sentence_transformers import SentenceTransformer
-
-name="paraphrase-albert-small-v2"
-model = SentenceTransformer(name)
-
-# used for both training and querying
-def embed_func(batch):
-    return [model.encode(sentence) for sentence in batch]
-```
-
-
-### OpenAI
-
-Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
-
-```python
-import openai
-import os
-
-# Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
-# OR set the key here as a variable
-openai.api_key = "sk-..."
-
-client = openai.OpenAI()
-
-def embed_func(c):    
-    rs = client.embeddings.create(input=c, model="text-embedding-ada-002")
-    return [record.embedding for record in rs["data"]]
-```
-
-
-## Applying an embedding function to data
-
-Using an embedding function, you can apply it to raw data
-to generate embeddings for each record.
-
-Say you have a pandas DataFrame with a `text` column that you want embedded,
-you can use the `with_embeddings` function to generate embeddings and add them to 
-an existing table.
-
-```python
-    import pandas as pd
-    from lancedb.embeddings import with_embeddings
-
-    df = pd.DataFrame(
-        [
-            {"text": "pepperoni"},
-            {"text": "pineapple"}
-        ]
-    )
-    data = with_embeddings(embed_func, df)
-
-    # The output is used to create / append to a table
-    tbl = db.create_table("my_table", data=data)
-```
-
-If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-using the `batch_size` parameter to `with_embeddings`.
-
-LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-API call is reliable.
-
-## Querying using an embedding function
-
-!!! warning
-    At query time, you **must** use the same embedding function you used to vectorize your data.
-    If you use a different embedding function, the embeddings will not reside in the same vector
-    space and the results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     results = (
-        tbl.search(query_vector)
-        .limit(10)
-        .to_pandas()
-     )
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/embeddings/understanding_embeddings.md

@@ -1,133 +0,0 @@
-# Understand Embeddings
-
-The term **dimension** is a synonym for the number of elements in a feature vector. Each feature can be thought of as a different axis in a geometric space. 
-
-High-dimensional data means there are many features(or attributes) in the data.
-
-!!! example
-     1. An image is a data point and it might have thousands of dimensions because each pixel could be considered as a feature. 
-
-     2. Text data, when represented by each word or character, can also lead to high dimensions, especially when considering all possible words in a language.
-
-Embedding captures **meaning and relationships** within data by mapping high-dimensional data into a lower-dimensional space. It captures it by placing inputs that are more **similar in meaning** closer together in the **embedding space**. 
-
-## What are Vector Embeddings?
-
-Vector embeddings is a way to convert complex data, like text, images, or audio into numerical coordinates (called vectors) that can be plotted in an n-dimensional space(embedding space). 
-
-The closer these data points are related in the real world, the closer their corresponding numerical coordinates (vectors) will be to each other in the embedding space. This proximity in the embedding space reflects their semantic similarities, allowing machines to intuitively understand and process the data in a way that mirrors human perception of relationships and meaning.
-
-In a way, it captures the most important aspects of the data while ignoring the less important ones. As a result, tasks like searching for related content or identifying patterns become more efficient and accurate, as the embeddings make it possible to quantify how **closely related** different **data points** are and **reduce** the **computational complexity**.
-
-??? question "Are vectors and embeddings the same thing?"
-
-    When we say “vectors” we mean - **list of numbers** that **represents the data**. 
-    When we say “embeddings” we mean - **list of numbers** that **capture important details and relationships**.
-
-    Although the terms are often used interchangeably, “embeddings” highlight how the data is represented with meaning and structure, while “vector” simply refers to the numerical form of that representation.
-
-## Embedding vs Indexing
-
-We already saw that creating **embeddings** on data is a method of creating **vectors** for a **n-dimensional embedding space** that captures the meaning and relationships inherent in the data.
-
-Once we have these **vectors**, indexing comes into play. Indexing is a method of organizing these vector embeddings, that allows us to quickly and efficiently locate and retrieve them from the entire dataset of vector embeddings.
-
-## What types of data/objects can be embedded?
-
-The following are common types of data that can be embedded:
-
-1. **Text**: Text data includes sentences, paragraphs, documents, or any written content.
-2. **Images**:  Image data encompasses photographs, illustrations, or any visual content.
-3. **Audio**: Audio data includes sounds, music, speech, or any auditory content.
-4. **Video**:  Video data consists of moving images and sound, which can convey complex information.
-
-Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model.
-
-!!! tip "LanceDB vs Other traditional Vector DBs"
-    While many vector databases primarily focus on the storage and retrieval of vector embeddings, **LanceDB** uses **Lance file format** (operates on a disk-based architecture), which allows for the storage and management of not just embeddings but also **raw file data (bytes)**. This capability means that users can integrate various types of data, including images and text, alongside their vector embeddings in a unified system.
-
-    With the ability to store both vectors and associated file data, LanceDB enhances the querying process. Users can perform semantic searches that not only retrieve similar embeddings but also access related files and metadata, thus streamlining the workflow.
-
-## How does embedding works?
-
-As mentioned, after creating embedding, each data point is represented as a vector in a n-dimensional space (embedding space). The dimensionality of this space can vary depending on the complexity of the data and the specific embedding technique used.
-
-Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar. To quantify this closeness, we use distance as a metric which can be measured in the  following way - 
-
-1. **Euclidean Distance (l2)**: It calculates the straight-line distance between two points (vectors) in a multidimensional space.
-2. **Cosine Similarity**: It measures the cosine of the angle between two vectors, providing a normalized measure of similarity based on their direction.
-3. **Dot product**: It is calculated as the sum of the products of their corresponding components. To measure relatedness it considers both the magnitude and direction of the vectors.
-
-## How do you create and store vector embeddings for your data?
-
-1. **Creating embeddings**: Choose an embedding model, it can be a pre-trained model (open-source or commercial) or you can train a custom embedding model for your scenario. Then feed your preprocessed data into the chosen model to obtain embeddings.
-
-??? question "Popular choices for embedding models"
-    For text data, popular choices are OpenAI’s text-embedding models, Google Gemini text-embedding models, Cohere’s Embed models, and SentenceTransformers, etc.
-
-    For image data, popular choices are CLIP (Contrastive Language–Image Pretraining), Imagebind embeddings by meta (supports audio, video, and image), and Jina multi-modal embeddings, etc.
-
-2. **Storing vector embeddings**: This effectively requires **specialized databases** that can handle the complexity of vector data, as traditional databases often struggle with this task. Vector databases are designed specifically for storing and querying vector embeddings. They optimize for efficient nearest-neighbor searches and provide built-in indexing mechanisms.
-
-!!! tip "Why LanceDB"
-    LanceDB **automates** the entire process of creating and storing embeddings for your data. LanceDB allows you to define and use **embedding functions**, which can be **pre-trained models** or **custom models**. 
-    
-    This enables you to **generate** embeddings tailored to the nature of your data (e.g., text, images) and **store** both the **original data** and **embeddings** in a **structured schema** thus providing efficient querying capabilities for similarity searches.
-
-Let's quickly [get started](./index.md) and learn how to manage embeddings in LanceDB. 
-
-## Bonus: As a developer, what you can create using embeddings?
-
-As a developer, you can create a variety of innovative applications using vector embeddings. Check out the following - 
-
-<div class="grid cards" markdown>
-
--   __Chatbots__
-
-    ---
-
-    Develop chatbots that utilize embeddings to retrieve relevant context and generate coherent, contextually aware responses to user queries.
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/chatbot.md)
-
--   __Recommendation Systems__
-
-    ---
-
-    Develop systems that recommend content (such as articles, movies, or products) based on the similarity of keywords and descriptions, enhancing user experience.
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/recommendersystem.md)
-
--   __Vector Search__
-
-    ---
-
-    Build powerful applications that harness the full potential of semantic search, enabling them to retrieve relevant data quickly and effectively. 
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/vector_search.md)
-
--   __RAG Applications__
-
-    ---
-
-    Combine the strengths of large language models (LLMs) with retrieval-based approaches to create more useful applications.
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/rag.md)
-
--   __Many more examples__
-
-    ---
-
-    Explore applied examples available as Colab notebooks or Python scripts to integrate into your applications.
-
-    [:octicons-arrow-right-24: More](../examples/examples_python.md)
-
-</div>
-
-
-
-
-
-
-
-

docs/src/embeddings/variables_and_secrets.md

@@ -1,53 +0,0 @@
-# Variable and Secrets
-
-Most embedding configuration options are saved in the table's metadata. However,
-this isn't always appropriate. For example, API keys should never be stored in the
-metadata. Additionally, other configuration options might be best set at runtime,
-such as the `device` configuration that controls whether to use GPU or CPU for
-inference. If you hardcoded this to GPU, you wouldn't be able to run the code on
-a server without one.
-
-To handle these cases, you can set variables on the embedding registry and
-reference them in the embedding configuration. These variables will be available
-during the runtime of your program, but not saved in the table's metadata. When
-the table is loaded from a different process, the variables must be set again.
-
-To set a variable, use the `set_var()` / `setVar()` method on the embedding registry.
-To reference a variable, use the syntax `$env:VARIABLE_NAME`. If there is a default
-value, you can use the syntax `$env:VARIABLE_NAME:DEFAULT_VALUE`.
-
-## Using variables to set secrets
-
-Sensitive configuration, such as API keys, must either be set as environment
-variables or using variables on the embedding registry. If you pass in a hardcoded
-value, LanceDB will raise an error. Instead, if you want to set an API key via
-configuration, use a variable:
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:register_secret"
-    ```
-
-=== "Typescript"
-
-    ```typescript
-    --8<-- "nodejs/examples/embedding.test.ts:register_secret"
-    ```
-
-## Using variables to set the device parameter
-
-Many embedding functions that run locally have a `device` parameter that controls
-whether to use GPU or CPU for inference. Because not all computers have a GPU,
-it's helpful to be able to set the `device` parameter at runtime, rather than
-have it hard coded in the embedding configuration. To make it work even if the
-variable isn't set, you could provide a default value of `cpu` in the embedding
-configuration.
-
-Some embedding libraries even have a method to detect which devices are available,
-which could be used to dynamically set the device at runtime. For example, in Python
-you can check if a CUDA GPU is available using `torch.cuda.is_available()`.
-
-```python
---8<-- "python/python/tests/docs/test_embeddings_optional.py:register_device"
-```

docs/src/examples/code_documentation_qa_bot_with_langchain.md

@@ -1,7 +0,0 @@
-# Code documentation Q&A bot with LangChain
-
-## use LanceDB's LangChain integration to build a Q&A bot for your documentation
-
-<img id="splash" width="400" alt="langchain" src="https://user-images.githubusercontent.com/917119/236580868-61a246a9-e587-4c2b-8ae5-6fe5f7b7e81e.png">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/code_qa_bot.ipynb)

docs/src/examples/examples_js.md

@@ -1,11 +0,0 @@
-# Examples: JavaScript
-
-To help you get started, we provide some examples, projects and applications that use the LanceDB JavaScript API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
-
-| Example | Scripts  |
-|-------- | ------   |
-| | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/index.js)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
-| [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js)  |

docs/src/examples/examples_python.md

@@ -1,22 +0,0 @@
-# Overview : Python Examples 
-
-To help you get started, we provide some examples, projects, and applications that use the LanceDB Python API. These examples are designed to get you right into the code with minimal introduction, enabling you to move from an idea to a proof of concept in minutes. 
-
-You can find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository. 
-
-**Introduction**
-
-Explore applied examples available as Colab notebooks or Python scripts to integrate into your applications. You can also checkout our blog posts related to the particular example for deeper understanding.
-
-| Explore                                         | Description                                                                                                                                                      |
-|----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [**Build from Scratch with LanceDB** 🛠️🚀](python_examples/build_from_scratch.md)               | Start building your **GenAI applications** from the **ground up** using **LanceDB's** efficient vector-based document retrieval capabilities! Get started quickly with a solid foundation.      |
-| [**Multimodal Search with LanceDB** 🤹‍♂️🔍](python_examples/multimodal.md)               | Combine **text** and **image queries** to find the most relevant results using **LanceDB’s multimodal** capabilities. Leverage the efficient vector-based similarity search.   |
-| [**RAG (Retrieval-Augmented Generation) with LanceDB** 🔓🧐](python_examples/rag.md) | Build RAG (Retrieval-Augmented Generation) with **LanceDB** for efficient **vector-based information retrieval** and more accurate responses from AI.                    |
-| [**Vector Search: Efficient Retrieval** 🔓👀](python_examples/vector_search.md) | Use **LanceDB's** vector search capabilities to perform efficient and accurate **similarity searches**, enabling rapid discovery and retrieval of relevant documents in Large datasets.                                                 |
-| [**Chatbot applications with LanceDB** 🤖](python_examples/chatbot.md)                | Create **chatbots** that retrieves relevant context for **coherent and context-aware replies**, enhancing user experience through advanced conversational AI. |
-| [**Evaluation: Assessing Text Performance with Precision** 📊💡](python_examples/evaluations.md) | Develop **evaluation** applications that allows you to input reference and candidate texts to **measure** their performance across various metrics.                                                      |
-| [**AI Agents: Intelligent Collaboration** 🤖](python_examples/aiagent.md)              | Enable **AI agents** to communicate and collaborate efficiently through dense vector representations, achieving shared goals seamlessly.                                         |
-| [**Recommender Systems: Personalized Discovery** 🍿📺](python_examples/recommendersystem.md)      | Deliver **personalized experiences** by efficiently storing and querying item embeddings with **LanceDB's** powerful vector database capabilities.                        |
-| **Miscellaneous Examples🌟** | Find other **unique examples** and **creative solutions** using **LanceDB**, showcasing the flexibility and broad applicability of the platform. |
-

docs/src/examples/examples_rust.md

@@ -1,3 +0,0 @@
-# Examples: Rust
-
-Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -1,165 +0,0 @@
-# How to Load Image Embeddings into LanceDB
-
-With the rise of Large Multimodal Models (LMMs) such as [GPT-4 Vision](https://blog.roboflow.com/gpt-4-vision/), the need for storing image embeddings is growing. The most effective way to store text and image embeddings is in a vector database such as LanceDB. Vector databases are a special kind of data store that enables efficient search over stored embeddings. 
-
-[CLIP](https://blog.roboflow.com/openai-clip/), a multimodal model developed by OpenAI, is commonly used to calculate image embeddings. These embeddings can then be used with a vector database to build a semantic search engine that you can query using images or text. For example, you could use LanceDB and CLIP embeddings to build a search engine for a database of folders.
-
-In this guide, we are going to show you how to use Roboflow Inference to load image embeddings into LanceDB. Without further ado, let’s get started!
-
-## Step #1: Install Roboflow Inference
-
-[Roboflow Inference](https://inference.roboflow.com) enables you to run state-of-the-art computer vision models with minimal configuration. Inference supports a range of models, from fine-tuned object detection, classification, and segmentation models to foundation models like CLIP. We will use Inference to calculate CLIP image embeddings.
-
-Inference provides a HTTP API through which you can run vision models.
-
-Inference powers the Roboflow hosted API, and is available as an open source utility. In this guide, we are going to run Inference locally, which enables you to calculate CLIP embeddings on your own hardware. We will also show you how to use the hosted Roboflow CLIP API, which is ideal if you need to scale and do not want to manage a system for calculating embeddings.
-
-To get started, first install the Inference CLI:
-
-```
-pip install inference-cli
-```
-
-Next, install Docker. Refer to the official Docker installation instructions for your operating system to get Docker set up. Once Docker is ready, you can start Inference using the following command:
-
-```
-inference server start
-```
-
-An Inference server will start running at ‘http://localhost:9001’.
-
-## Step #2: Set Up a LanceDB Vector Database
-
-Now that we have Inference running, we can set up a LanceDB vector database. You can run LanceDB in JavaScript and Python. For this guide, we will use the Python API. But, you can take the HTTP requests we make below and change them to JavaScript if required.
-
-For this guide, we are going to search the [COCO 128 dataset](https://universe.roboflow.com/team-roboflow/coco-128), which contains a wide range of objects. The variability in objects present in this dataset makes it a good dataset to demonstrate the capabilities of vector search. If you want to use this dataset, you can download [COCO 128 from Roboflow Universe](https://universe.roboflow.com/team-roboflow/coco-128). With that said, you can search whatever folder of images you want.
-
-Once you have a dataset ready, install LanceDB with the following command:
-
-```
-pip install lancedb
-```
-
-We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
-
-```
-pip install tantivy
-```
-
-Create a new Python file and add the following code:
-
-```python
-import cv2
-import supervision as sv
-import requests
-
-import lancedb
-
-db = lancedb.connect("./embeddings")
-
-IMAGE_DIR = "images/"
-API_KEY = os.environ.get("ROBOFLOW_API_KEY")
-SERVER_URL = "http://localhost:9001"
-
-results = []
-
-for i, image in enumerate(os.listdir(IMAGE_DIR)):
-    infer_clip_payload = {
-        #Images can be provided as urls or as base64 encoded strings
-        "image": {
-            "type": "base64",
-            "value": base64.b64encode(open(IMAGE_DIR + image, "rb").read()).decode("utf-8"),
-        },
-    }
-
-    res = requests.post(
-        f"{SERVER_URL}/clip/embed_image?api_key={API_KEY}",
-        json=infer_clip_payload,
-    )
-
-    embeddings = res.json()['embeddings']
-
-    print("Calculated embedding for image: ", image)
-
-    image = {"vector": embeddings[0], "name": os.path.join(IMAGE_DIR, image)}
-
-    results.append(image)
-
-tbl = db.create_table("images", data=results)
-
-tbl.create_fts_index("name")
-```
-
-To use the code above, you will need a Roboflow API key. [Learn how to retrieve a Roboflow API key](https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key). Run the following command to set up your API key in your environment:
-
-```
-export ROBOFLOW_API_KEY=""
-```
-
-Replace the `IMAGE_DIR` value with the folder in which you are storing the images for which you want to calculate embeddings. If you want to use the Roboflow CLIP API to calculate embeddings, replace the `SERVER_URL` value with `https://infer.roboflow.com`.
-
-Run the script above to create a new LanceDB database. This database will be stored on your local machine. The database will be called `embeddings` and the table will be called `images`.
-
-The script above calculates all embeddings for a folder then creates a new table. To add additional images, use the following code:
-
-```python
-def make_batches():
-    for i in range(5):
-        yield [
-                {"vector": [3.1, 4.1], "name": "image1.png"},
-                {"vector": [5.9, 26.5], "name": "image2.png"}
-            ]
-
-tbl = db.open_table("images")
-tbl.add(make_batches())
-```
-
-Replacing the `make_batches()` function with code to load embeddings for images.
-
-## Step #3: Run a Search Query
-
-We are now ready to run a search query. To run a search query, we need a text embedding that represents a text query. We can use this embedding to search our LanceDB database for an entry.
-
-Let’s calculate a text embedding for the query “cat”, then run a search query:
-
-```python
-infer_clip_payload = {
-    "text": "cat",
-}
-
-res = requests.post(
-    f"{SERVER_URL}/clip/embed_text?api_key={API_KEY}",
-    json=infer_clip_payload,
-)
-
-embeddings = res.json()['embeddings']
-
-df = tbl.search(embeddings[0]).limit(3).to_list()
-
-print("Results:")
-
-for i in df:
-    print(i["name"])
-```
-
-This code will search for the three images most closely related to the prompt “cat”. The names of the most similar three images will be printed to the console. Here are the three top results:
-
-```
-dataset/images/train/000000000650_jpg.rf.1b74ba165c5a3513a3211d4a80b69e1c.jpg
-dataset/images/train/000000000138_jpg.rf.af439ef1c55dd8a4e4b142d186b9c957.jpg
-dataset/images/train/000000000165_jpg.rf.eae14d5509bf0c9ceccddbb53a5f0c66.jpg
-```
-
-Let’s open the top image:
-
-![Cat](https://media.roboflow.com/cat_lancedb.jpg)
-
-The top image was a cat. Our search was successful.
-
-## Conclusion
-
-LanceDB is a vector database that you can use to store and efficiently search your image embeddings. You can use Roboflow Inference, a scalable computer vision inference server, to calculate CLIP embeddings that you can store in LanceDB.
-
-You can use Inference and LanceDB together to build a range of applications with image embeddings, from a media search engine to a retrieval-augmented generation pipeline for use with LMMs.
-
-To learn more about Inference and its capabilities, refer to the Inference documentation.

docs/src/examples/index.md

@@ -1,12 +0,0 @@
-# Example projects and recipes
-
-## Recipes and example code
-
-LanceDB provides language APIs, allowing you to embed a database in your language of choice.
-
-* 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](examples_js.md) examples
-* 🦀 Rust examples (coming soon)
-
-!!! tip "Hosted LanceDB"
-    If you want S3 cost-efficiency and local performance via a simple serverless API, checkout **LanceDB Cloud**. For private deployments, high performance at extreme scale, or if you have strict security requirements, talk to us about **LanceDB Enterprise**. [Learn more](https://docs.lancedb.com/)

docs/src/examples/modal_langchain.py

@@ -1,119 +0,0 @@
-import pickle
-import re
-import zipfile
-from pathlib import Path
-
-import requests
-from langchain.chains import RetrievalQA
-from langchain.document_loaders import UnstructuredHTMLLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import LanceDB
-from modal import Image, Secret, Stub, web_endpoint
-
-import lancedb
-
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb", "langchain", "openai", "pandas", "tiktoken", "unstructured", "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-
-
-def get_document_title(document):
-    m = str(document.metadata["source"])
-    title = re.findall("pandas.documentation(.*).html", m)
-    if title[0] is not None:
-        return title[0]
-    return ""
-
-
-def download_docs():
-    pandas_docs = requests.get(
-        "https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"
-    )
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-
-
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-
-
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path)
-    table = db.create_table(
-        "pandas_docs",
-        data=[
-            {
-                "vector": embeddings.embed_query("Hello World"),
-                "text": "Hello World",
-                "id": "1",
-            }
-        ],
-        mode="overwrite",
-    )
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(
-        llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever()
-    )
-    return qa.run(query)
-
-
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-
-
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)

docs/src/examples/multimodal_search.md

@@ -1,7 +0,0 @@
-# Image multimodal search
-
-## Search through an image dataset using natural language, full text and SQL
-
-<img id="splash" width="400" alt="multimodal search" src="https://github.com/lancedb/lancedb/assets/917119/993a7c9f-be01-449d-942e-1ce1d4ed63af">
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/multimodal_search.ipynb)

docs/src/examples/python_examples/aiagent.md

@@ -1,27 +0,0 @@
-# AI Agents: Intelligent Collaboration🤖
-
-Think of a platform where AI Agents can seamlessly exchange information, coordinate over tasks, and achieve shared targets with great efficiency💻📈.
-
-## Vector-Based Coordination: The Technical Advantage
-Leveraging LanceDB's vector-based capabilities, we can enable **AI agents 🤖** to communicate and collaborate through dense vector representations. AI agents can exchange information, coordinate on a task or work towards a common goal, just by giving queries📝.
-
-| **AI Agents** | **Description** | **Links** |
-|:--------------|:----------------|:----------|
-| **AI Agents: Reducing Hallucinationt📊** | 🤖💡 **Reduce AI hallucinations** using Critique-Based Contexting! Learn by Simplifying and Automating tedious workflows by going through fitness trainer agent example.💪 | [![Github](../../assets/github.svg)][hullucination_github] <br>[![Open In Collab](../../assets/colab.svg)][hullucination_colab] <br>[![Python](../../assets/python.svg)][hullucination_python] <br>[![Ghost](../../assets/ghost.svg)][hullucination_ghost] |                                                                                                                                                                                                                                                
-| **AI Trends Searcher: CrewAI🔍️** | 🔍️ Learn about **CrewAI Agents** ! Utilize the features of CrewAI - Role-based Agents, Task Management, and Inter-agent Delegation ! Make AI agents work together to do tricky stuff 😺| [![Github](../../assets/github.svg)][trend_github] <br>[![Open In Collab](../../assets/colab.svg)][trend_colab] <br>[![Ghost](../../assets/ghost.svg)][trend_ghost] |                                                                                                                                                                                                                                                
-| **SuperAgent Autogen🤖** | 💻 AI interactions with the Super Agent! Integrating **Autogen**, **LanceDB**, **LangChain**, **LiteLLM**, and **Ollama** to create AI agent that excels in understanding and processing complex queries.🤖 | [![Github](../../assets/github.svg)][superagent_github] <br>[![Open In Collab](../../assets/colab.svg)][superagent_colab] |
-
-
-[hullucination_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents
-[hullucination_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb
-[hullucination_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.py
-[hullucination_ghost]: https://blog.lancedb.com/how-to-reduce-hallucinations-from-llm-powered-agents-using-long-term-memory-72f262c3cc1f/
-
-[trend_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/AI-Trends-with-CrewAI
-[trend_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/AI-Trends-with-CrewAI/CrewAI_AI_Trends.ipynb 
-[trend_ghost]: https://blog.lancedb.com/track-ai-trends-crewai-agents-rag/
-
-[superagent_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/SuperAgent_Autogen
-[superagent_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/SuperAgent_Autogen/main.ipynb
-
-

docs/src/examples/python_examples/build_from_scratch.md

@@ -1,13 +0,0 @@
-# **Build from Scratch with LanceDB 🛠️🚀**
-
-Start building your GenAI applications from the ground up using **LanceDB's** efficient vector-based document retrieval capabilities! 📑
-
-**Get Started in Minutes ⏱️**
-
-These examples provide a solid foundation for building your own GenAI applications using LanceDB. Jump from idea to **proof of concept** quickly with applied examples. Get started and see what you can create! 💻
-
-| **Build From Scratch**                     | **Description**                                                                                              | **Links**                                                                                                                                                                                                                                                                                                                                                                                                                |
-|:-------------------------------------------|:-------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Build RAG from Scratch🚀💻**             | 📝 Create a **Retrieval-Augmented Generation** (RAG) model from scratch using LanceDB.                       | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/RAG-from-Scratch)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)]()                                                                                                                                  |
-| **Local RAG from Scratch with Llama3🔥💡** | 🐫 Build a local RAG model using **Llama3** and **LanceDB** for fast and efficient text generation.          | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Local-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Local-RAG-from-Scratch/rag.py)    |
-| **Multi-Head RAG from Scratch📚💻**        | 🤯 Develop a **Multi-Head RAG model** from scratch, enabling generation of text based on multiple documents. | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch) |

docs/src/examples/python_examples/chatbot.md

@@ -1,41 +0,0 @@
-**Chatbot applications with LanceDB 🤖**
-====================================================================
-
- Create innovative chatbot applications that utilizes LanceDB for efficient vector-based response generation! 🌐✨ 
- 
-**Introduction 👋✨**
-
- Users can input their queries, allowing the chatbot to retrieve relevant context seamlessly. 🔍📚 This enables the generation of coherent and context-aware replies that enhance user experience. 🌟🤝 Dive into the world of advanced conversational AI and streamline interactions with powerful data management! 🚀💡
-
-
-| **Chatbot**  | **Description**  | **Links**  |
-|:----------------|:-----------------|:-----------|
-| **Databricks DBRX Website Bot ⚡️** | Engage with the **Hogwarts chatbot**, that uses Open-source RAG with **DBRX**, **LanceDB** and **LLama-index with Hugging Face Embeddings**, to provide interactive and engaging user experiences. ✨ | [![GitHub](../../assets/github.svg)][databricks_github] <br>[![Python](../../assets/python.svg)][databricks_python] |
-| **CLI SDK Manual Chatbot Locally 💻** | CLI chatbot for SDK/hardware documents using **Local RAG** with **LLama3**, **Ollama**, **LanceDB**, and **Openhermes Embeddings**, built with **Phidata** Assistant and Knowledge Base 🤖 | [![GitHub](../../assets/github.svg)][clisdk_github] <br>[![Python](../../assets/python.svg)][clisdk_python] |
-| **Youtube Transcript Search QA Bot 📹** |  Search through **youtube transcripts** using natural language with a Q&A bot, leveraging **LanceDB** for effortless data storage and management 💬 | [![GitHub](../../assets/github.svg)][youtube_github] <br>[![Open In Collab](../../assets/colab.svg)][youtube_colab] <br>[![Python](../../assets/python.svg)][youtube_python] |
-| **Code Documentation Q&A Bot with LangChain 🤖** | Query your own documentation easily using questions in natural language with a Q&A bot, powered by **LangChain** and **LanceDB**, demonstrated with **Numpy 1.26 docs** 📚 | [![GitHub](../../assets/github.svg)][docs_github] <br>[![Open In Collab](../../assets/colab.svg)][docs_colab] <br>[![Python](../../assets/python.svg)][docs_python] |
-| **Context-aware Chatbot using Llama 2 & LanceDB 🤖** | Build **conversational AI** with a **context-aware chatbot**, powered by **Llama 2**, **LanceDB**, and **LangChain**, that enables intuitive and meaningful conversations with your data 📚💬 | [![GitHub](../../assets/github.svg)][aware_github] <br>[![Open In Collab](../../assets/colab.svg)][aware_colab] <br>[![Ghost](../../assets/ghost.svg)][aware_ghost] |
-| **Chat with csv using Hybrid Search 📊** | **Chat** application that interacts with **CSV** and **Excel files** using **LanceDB’s** hybrid search capabilities, performing direct operations on large-scale columnar data efficiently 🚀 | [![GitHub](../../assets/github.svg)][csv_github] <br>[![Open In Collab](../../assets/colab.svg)][csv_colab] <br>[![Ghost](../../assets/ghost.svg)][csv_ghost] |
-
-
-[databricks_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/databricks_DBRX_website_bot
-[databricks_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/databricks_DBRX_website_bot/main.py
-
-[clisdk_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/CLI-SDK-Manual-Chatbot-Locally
-[clisdk_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/CLI-SDK-Manual-Chatbot-Locally/assistant.py
-
-[youtube_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Youtube-Search-QA-Bot
-[youtube_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Youtube-Search-QA-Bot/main.ipynb
-[youtube_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Youtube-Search-QA-Bot/main.py
-
-[docs_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot
-[docs_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb
-[docs_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.py
-
-[aware_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/chatbot_using_Llama2_&_lanceDB
-[aware_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/chatbot_using_Llama2_&_lanceDB/main.ipynb
-[aware_ghost]: https://blog.lancedb.com/context-aware-chatbot-using-llama-2-lancedb-as-vector-database-4d771d95c755
-
-[csv_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Chat_with_csv_file
-[csv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Chat_with_csv_file/main.ipynb
-[csv_ghost]: https://blog.lancedb.com/p/d8c71df4-e55f-479a-819e-cde13354a6a3/

docs/src/examples/python_examples/evaluations.md

@@ -1,21 +0,0 @@
-**Evaluation: Assessing Text Performance with Precision 📊💡**
-====================================================================
-
-Evaluation is a comprehensive tool designed to measure the performance of text-based inputs, enabling data-driven optimization and improvement 📈. 
-
-**Text Evaluation 101 📚**
-
-Using robust framework for assessing reference and candidate texts across various metrics📊, ensure that the text outputs are high-quality and meet specific requirements and standards📝.
-
-| **Evaluation** | **Description** | **Links** |
-| -------------- | --------------- | --------- |
-| **Evaluating Prompts with Prompttools 🤖** | Compare, visualize & evaluate **embedding functions** (incl. OpenAI) across metrics like latency & custom evaluation 📈📊 | [![Github](../../assets/github.svg)][prompttools_github] <br>[![Open In Collab](../../assets/colab.svg)][prompttools_colab] |
-| **Evaluating RAG with RAGAs and GPT-4o 📊** | Evaluate **RAG pipelines** with cutting-edge metrics and tools, integrate with CI/CD for continuous performance checks, and generate responses with GPT-4o 🤖📈 | [![Github](../../assets/github.svg)][RAGAs_github] <br>[![Open In Collab](../../assets/colab.svg)][RAGAs_colab] |
-
-
-
-[prompttools_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts
-[prompttools_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb
-
-[RAGAs_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Evaluating_RAG_with_RAGAs
-[RAGAs_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Evaluating_RAG_with_RAGAs/Evaluating_RAG_with_RAGAs.ipynb

docs/src/examples/python_examples/multimodal.md

@@ -1,28 +0,0 @@
-# **Multimodal Search with LanceDB 🤹‍♂️🔍**
-
-Using LanceDB's multimodal capabilities, combine text and image queries to find the most relevant results in your corpus ! 🔓💡
-
-**Explore the Future of Search 🚀**
-
-LanceDB supports multimodal search by indexing and querying vector representations of text and image data 🤖. This enables efficient retrieval of relevant documents and images using vector-based similarity search 📊. The platform facilitates cross-modal search, allowing for text-image and image-text retrieval, and supports scalable indexing of high-dimensional vector spaces 💻.
-
-
-
-| **Multimodal**  | **Description**  | **Links**  |
-|:----------------|:-----------------|:-----------|
-| **Multimodal CLIP: DiffusionDB 🌐💥**    | Multi-Modal Search with **CLIP** and **LanceDB** Using **DiffusionDB** Data for Combined Text and Image Understanding ! 🔓 | [![GitHub](../../assets/github.svg)][Clip_diffusionDB_github] <br>[![Open In Collab](../../assets/colab.svg)][Clip_diffusionDB_colab] <br>[![Python](../../assets/python.svg)][Clip_diffusionDB_python] <br>[![Ghost](../../assets/ghost.svg)][Clip_diffusionDB_ghost] |
-| **Multimodal CLIP: Youtube Videos 📹👀** | Search **Youtube videos** using Multimodal CLIP, finding relevant content with ease and accuracy! 🎯                                                                                                                                                                | [![Github](../../assets/github.svg)][Clip_youtube_github]                                                                                                    <br>[![Open In Collab](../../assets/colab.svg)][Clip_youtube_colab]                                                                                           <br> [![Python](../../assets/python.svg)][Clip_youtube_python]         <br>[![Ghost](../../assets/ghost.svg)][Clip_youtube_python] |
-| **Multimodal Image + Text Search 📸🔍**               | Find **relevant documents** and **images** with a single query using **LanceDB's** multimodal search capabilities, to seamlessly integrate text and visuals ! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
-| **Cambrian-1: Vision-Centric Image Exploration 🔍👀** | Learn how **Cambrian-1** works, using an example of **Vision-Centric** exploration on images found through vector search ! Work on **Flickr-8k** dataset 🔎                        | [![Kaggle](https://img.shields.io/badge/Kaggle-035a7d?style=for-the-badge&logo=kaggle&logoColor=white)](https://www.kaggle.com/code/prasantdixit/cambrian-1-vision-centric-exploration-of-images/)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/cambrian-1-vision-centric-exploration/)                                                                                                                                                                                                                   |
-
-
-[Clip_diffusionDB_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb
-[Clip_diffusionDB_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.ipynb
-[Clip_diffusionDB_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.py
-[Clip_diffusionDB_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/
-
-
-[Clip_youtube_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search
-[Clip_youtube_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb
-[Clip_youtube_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.py
-[Clip_youtube_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/

docs/src/examples/python_examples/rag.md

@@ -1,83 +0,0 @@
-**RAG (Retrieval-Augmented Generation) with LanceDB 🔓🧐**
-====================================================================
-
-Build RAG (Retrieval-Augmented Generation) with  LanceDB, a powerful solution for efficient vector-based information retrieval 📊. 
-
-**Experience the Future of Search 🔄**
-
-🤖 RAG enables AI to **retrieve** relevant information from external sources and use it to **generate** more accurate and context-specific responses. 💻 LanceDB provides a robust framework for integrating LLMs with external knowledge sources 📝.
-
-| **RAG**                                      | **Description**                                                                                                                                                  | **Links**                  |
-|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|                                                                                                                                                     
-| **RAG with Matryoshka Embeddings and LlamaIndex** 🪆🔗  | Utilize **Matryoshka embeddings** and **LlamaIndex** to improve the efficiency and accuracy of your RAG models. 📈✨ | [![Github](../../assets/github.svg)][matryoshka_github] <br>[![Open In Collab](../../assets/colab.svg)][matryoshka_colab]  |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with Re-ranking** 📈🔄 | Enhance your RAG applications by implementing **re-ranking strategies** for more relevant document retrieval. 📚🔍 | [![Github](../../assets/github.svg)][rag_reranking_github] <br>[![Open In Collab](../../assets/colab.svg)][rag_reranking_colab] <br>[![Ghost](../../assets/ghost.svg)][rag_reranking_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Instruct-Multitask** 🧠🎯 |  Integrate the **Instruct Embedding Model** with LanceDB to streamline your embedding API, reducing redundant code and overhead. 🌐📊 | [![Github](../../assets/github.svg)][instruct_multitask_github] <br>[![Open In Collab](../../assets/colab.svg)][instruct_multitask_colab] <br>[![Python](../../assets/python.svg)][instruct_multitask_python] <br>[![Ghost](../../assets/ghost.svg)][instruct_multitask_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with HyDE** 🌌🔍 | Use **Hypothetical Document Embeddings** for efficient, accurate, and unsupervised dense retrieval. 📄🔍 | [![Github](../../assets/github.svg)][hyde_github] <br>[![Open In Collab](../../assets/colab.svg)][hyde_colab]<br>[![Ghost](../../assets/ghost.svg)][hyde_ghost]                 |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with LOTR** 🧙‍♂️📜 | Enhance RAG with **Lord of the Retriever (LOTR)** to address 'Lost in the Middle' challenges, especially in medical data. 🌟📜 | [![Github](../../assets/github.svg)][lotr_github] <br>[![Open In Collab](../../assets/colab.svg)][lotr_colab] <br>[![Ghost](../../assets/ghost.svg)][lotr_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Advanced RAG: Parent Document Retriever** 📑🔗 | Use **Parent Document & Bigger Chunk Retriever** to maintain context and relevance when generating related content. 🎵📄 | [![Github](../../assets/github.svg)][parent_doc_retriever_github] <br>[![Open In Collab](../../assets/colab.svg)][parent_doc_retriever_colab] <br>[![Ghost](../../assets/ghost.svg)][parent_doc_retriever_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Corrective RAG with Langgraph** 🔧📊 | Enhance RAG reliability with **Corrective RAG (CRAG)** by self-reflecting and fact-checking for accurate and trustworthy results. ✅🔍 |[![Github](../../assets/github.svg)][corrective_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][corrective_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][corrective_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Contextual Compression with RAG** 🗜️🧠 | Apply **contextual compression techniques** to condense large documents while retaining essential information. 📄🗜️                                               | [![Github](../../assets/github.svg)][compression_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][compression_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][compression_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with FLARE** 🔥|  Enable users to ask questions directly to **academic papers**, focusing on **ArXiv papers**, with **F**orward-**L**ooking **A**ctive **RE**trieval augmented generation.🚀🌟                                                                       | [![Github](../../assets/github.svg)][flare_github] <br>[![Open In Collab](../../assets/colab.svg)][flare_colab] <br>[![Ghost](../../assets/ghost.svg)][flare_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Query Expansion and Reranker** 🔍🔄 | Enhance RAG with query expansion using Large Language Models and advanced **reranking methods** like **Cross Encoders**, **ColBERT v2**, and **FlashRank** for improved document retrieval precision and recall 🔍📈                                                         | [![Github](../../assets/github.svg)][query_github] <br>[![Open In Collab](../../assets/colab.svg)][query_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **RAG Fusion** ⚡🌐 | Build RAG Fusion, utilize the **RRF algorithm** to rerank documents based on user queries ! Use **LanceDB** as vector database to store and retrieve documents related to queries via **OPENAI Embeddings**⚡🌐                                                                          | [![Github](../../assets/github.svg)][fusion_github] <br>[![Open In Collab](../../assets/colab.svg)][fusion_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Agentic RAG** 🤖📚 | Build autonomous information retrieval with **Agentic RAG**, a framework of **intelligent agents** that collaborate to synthesize, summarize, and compare data across sources, that enables proactive and informed decision-making 🤖📚                                                              | [![Github](../../assets/github.svg)][agentic_github] <br>[![Open In Collab](../../assets/colab.svg)][agentic_colab] |
-
-
-
-
-
-
-
-
-
-
-
-
-[matryoshka_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex
-[matryoshka_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex/RAG_with_MatryoshkaEmbedding_and_Llamaindex.ipynb
-
-[rag_reranking_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking
-[rag_reranking_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking/main.ipynb
-[rag_reranking_ghost]: https://blog.lancedb.com/simplest-method-to-improve-rag-pipeline-re-ranking-cf6eaec6d544
-
-
-[instruct_multitask_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask
-[instruct_multitask_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.ipynb
-[instruct_multitask_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.py
-[instruct_multitask_ghost]: https://blog.lancedb.com/multitask-embedding-with-lancedb-be18ec397543
-
-[hyde_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE
-[hyde_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb
-[hyde_ghost]: https://blog.lancedb.com/advanced-rag-precise-zero-shot-dense-retrieval-with-hyde-0946c54dfdcb
-
-[lotr_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR
-[lotr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR/main.ipynb
-[lotr_ghost]: https://blog.lancedb.com/better-rag-with-lotr-lord-of-retriever-23c8336b9a35
-
-[parent_doc_retriever_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever
-[parent_doc_retriever_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever/main.ipynb
-[parent_doc_retriever_ghost]: https://blog.lancedb.com/modified-rag-parent-document-bigger-chunk-retriever-62b3d1e79bc6
-
-[corrective_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph
-[corrective_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb
-[corrective_rag_ghost]: https://blog.lancedb.com/implementing-corrective-rag-in-the-easiest-way-2/
-
-[compression_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG
-[compression_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG/main.ipynb
-[compression_rag_ghost]: https://blog.lancedb.com/enhance-rag-integrate-contextual-compression-and-filtering-for-precision-a29d4a810301/
-
-[flare_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR
-[flare_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb
-[flare_ghost]: https://blog.lancedb.com/better-rag-with-active-retrieval-augmented-generation-flare-3b66646e2a9f/
-
-[query_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/QueryExpansion%26Reranker
-[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/QueryExpansion&Reranker/main.ipynb
-
-
-[fusion_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/RAG_Fusion
-[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/RAG_Fusion/main.ipynb
-
-[agentic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG
-[agentic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb
-
-

docs/src/examples/python_examples/recommendersystem.md

@@ -1,37 +0,0 @@
-**Recommender Systems: Personalized Discovery🍿📺**
-==============================================================
-Deliver personalized experiences with Recommender Systems. 🎁
-
-**Technical Overview📜**
-
-🔍️ LanceDB's powerful vector database capabilities can efficiently store and query item embeddings. Recommender Systems can utilize it and provide personalized recommendations based on user preferences 🤝 and item features 📊 and therefore enhance the user experience.🗂️ 
-
-| **Recommender System** | **Description** | **Links** |
-| ---------------------- | --------------- | --------- |
-| **Movie Recommender System🎬** | 🤝 Use **collaborative filtering** to predict user preferences, assuming similar users will like similar movies, and leverage **Singular Value Decomposition** (SVD) from Numpy for precise matrix factorization and accurate recommendations📊 | [![Github](../../assets/github.svg)][movie_github] <br>[![Open In Collab](../../assets/colab.svg)][movie_colab] <br>[![Python](../../assets/python.svg)][movie_python] |
-| **🎥 Movie Recommendation with Genres** | 🔍 Creates movie embeddings using **Doc2Vec**, capturing genre and characteristic nuances, and leverages VectorDB for efficient storage and querying, enabling accurate genre classification and personalized movie recommendations through **similarity searches**🎥 | [![Github](../../assets/github.svg)][genre_github] <br>[![Open In Collab](../../assets/colab.svg)][genre_colab] <br>[![Ghost](../../assets/ghost.svg)][genre_ghost] |
-| **🛍️ Product Recommender using Collaborative Filtering and LanceDB** | 📈 Using **Collaborative Filtering** and **LanceDB** to analyze your past purchases, recommends products based on user's past purchases. Demonstrated with the Instacart dataset in our example🛒 | [![Github](../../assets/github.svg)][product_github] <br>[![Open In Collab](../../assets/colab.svg)][product_colab] <br>[![Python](../../assets/python.svg)][product_python] |
-| **🔍 Arxiv Search with OpenCLIP and LanceDB** | 💡 Build a semantic search engine for **Arxiv papers** using **LanceDB**, and benchmarks its performance against traditional keyword-based search on **Nomic's Atlas**, to demonstrate the power of semantic search in finding relevant research papers📚 | [![Github](../../assets/github.svg)][arxiv_github] <br>[![Open In Collab](../../assets/colab.svg)][arxiv_colab] <br>[![Python](../../assets/python.svg)][arxiv_python] |
-| **Food Recommendation System🍴** | 🍔 Build a food recommendation system with **LanceDB**, featuring vector-based recommendations, full-text search, hybrid search, and reranking model integration for personalized and accurate food suggestions👌 | [![Github](../../assets/github.svg)][food_github] <br>[![Open In Collab](../../assets/colab.svg)][food_colab] |
-
-[movie_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommender
-[movie_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb
-[movie_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.py
-
-
-[genre_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/movie-recommendation-with-genres
-[genre_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/movie-recommendation-with-genres/movie_recommendation_with_doc2vec_and_lancedb.ipynb
-[genre_ghost]: https://blog.lancedb.com/movie-recommendation-system-using-lancedb-and-doc2vec/
-
-[product_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/product-recommender
-[product_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/product-recommender/main.ipynb
-[product_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/product-recommender/main.py
-
-
-[arxiv_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender
-[arxiv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender/main.ipynb
-[arxiv_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender/main.py
- 
-
-[food_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Food_recommendation
-[food_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Food_recommendation/main.ipynb

docs/src/examples/python_examples/vector_search.md

@@ -1,80 +0,0 @@
-**Vector Search: Efficient Retrieval 🔓👀**
-====================================================================
-
-Vector search with LanceDB, is a solution for  efficient and accurate similarity searches in large datasets 📊. 
-
-**Vector Search Capabilities in LanceDB🔝**
-
-LanceDB implements vector search algorithms for efficient document retrieval and analysis 📊. This enables fast and accurate discovery of relevant documents, leveraging dense vector representations 🤖. The platform supports scalable indexing and querying of high-dimensional vector spaces, facilitating precise document matching and retrieval 📈.
-
-| **Vector Search** | **Description** | **Links** |
-|:-----------------|:---------------|:---------|
-| **Inbuilt Hybrid Search 🔄** | Perform hybrid search in **LanceDB** by combining the results of semantic and full-text search via a reranking algorithm of your choice 📊 | [![Github](../../assets/github.svg)][inbuilt_hybrid_search_github] <br>[![Open In Collab](../../assets/colab.svg)][inbuilt_hybrid_search_colab] |                                                                                                                                                                                                                                                                                                    
-| **Hybrid Search with BM25 and LanceDB 💡** | Use **Synergizes BM25's** keyword-focused precision (term frequency, document length normalization, bias-free retrieval) with **LanceDB's** semantic understanding (contextual analysis, query intent alignment) for nuanced search results in complex datasets 📈 | [![Github](../../assets/github.svg)][BM25_github] <br>[![Open In Collab](../../assets/colab.svg)][BM25_colab] <br>[![Ghost](../../assets/ghost.svg)][BM25_ghost] |                                                                                                                                                                                                                                                                                                    
-| **NER-powered Semantic Search 🔎** | Extract and identify essential information from text with Named Entity Recognition **(NER)** methods: Dictionary-Based, Rule-Based, and Deep Learning-Based, to accurately extract and categorize entities, enabling precise semantic search results 🗂️ | [![Github](../../assets/github.svg)][NER_github] <br>[![Open In Collab](../../assets/colab.svg)][NER_colab] <br>[![Ghost](../../assets/ghost.svg)][NER_ghost]|                                                                                                                                                                                                                                                                                                    
-| **Audio Similarity Search using Vector Embeddings 🎵** | Create vector **embeddings of audio files** to find similar audio content, enabling efficient audio similarity search and retrieval in **LanceDB's** vector store 📻 |[![Github](../../assets/github.svg)][audio_search_github] <br>[![Open In Collab](../../assets/colab.svg)][audio_search_colab] <br>[![Python](../../assets/python.svg)][audio_search_python]|                                                                                                                                                                                                                                                                                                    
-| **LanceDB Embeddings API: Multi-lingual Semantic Search 🌎** | Build a universal semantic search table with **LanceDB's Embeddings API**, supporting multiple languages (e.g., English, French) using **cohere's** multi-lingual model, for accurate cross-lingual search results 📄 | [![Github](../../assets/github.svg)][mls_github] <br>[![Open In Collab](../../assets/colab.svg)][mls_colab] <br>[![Python](../../assets/python.svg)][mls_python] |                                                                                                                                                                                                                                                                                                    
-| **Facial Recognition: Face Embeddings 🤖** | Detect, crop, and embed faces using Facenet, then store and query face embeddings in **LanceDB** for efficient facial recognition and top-K matching results 👥 | [![Github](../../assets/github.svg)][fr_github] <br>[![Open In Collab](../../assets/colab.svg)][fr_colab] |                                                                                                                                                                                                                                                                                                    
-| **Sentiment Analysis: Hotel Reviews 🏨** | Analyze customer sentiments towards the hotel industry using **BERT models**, storing sentiment labels, scores, and embeddings in **LanceDB**, enabling queries on customer opinions and potential areas for improvement 💬 | [![Github](../../assets/github.svg)][sentiment_analysis_github] <br>[![Open In Collab](../../assets/colab.svg)][sentiment_analysis_colab] <br>[![Ghost](../../assets/ghost.svg)][sentiment_analysis_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Vector Arithmetic with LanceDB ⚖️** | Perform **vector arithmetic** on embeddings, enabling complex relationships and nuances in data to be captured, and simplifying the process of retrieving semantically similar results 📊 | [![Github](../../assets/github.svg)][arithmetic_github] <br>[![Open In Collab](../../assets/colab.svg)][arithmetic_colab] <br>[![Ghost](../../assets/ghost.svg)][arithmetic_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Imagebind Demo 🖼️** | Explore the multi-modal capabilities of **Imagebind** through a Gradio app, use **LanceDB API** for seamless image search and retrieval experiences 📸 | [![Github](../../assets/github.svg)][imagebind_github] <br> [![Open in Spaces](../../assets/open_hf_space.svg)][imagebind_huggingface] |                                                                                                                                                                                                                                                                                                    
-| **Search Engine using SAM & CLIP 🔍** | Build a search engine within an image using **SAM** and **CLIP** models, enabling object-level search and retrieval, with LanceDB indexing and search capabilities to find the closest match between image embeddings and user queries 📸 | [![Github](../../assets/github.svg)][swi_github] <br>[![Open In Collab](../../assets/colab.svg)][swi_colab] <br>[![Ghost](../../assets/ghost.svg)][swi_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Zero Shot Object Localization and Detection with CLIP 🔎** | Perform object detection on images using **OpenAI's CLIP**, enabling zero-shot localization and detection of objects, with capabilities to split images into patches, parse with CLIP, and plot bounding boxes 📊 | [![Github](../../assets/github.svg)][zsod_github] <br>[![Open In Collab](../../assets/colab.svg)][zsod_colab] |                                                                                                                                                                                                                                                                                                    
-| **Accelerate Vector Search with OpenVINO 🚀** | Boost vector search applications using **OpenVINO**, achieving significant speedups with **CLIP** for text-to-image and image-to-image searching, through PyTorch model optimization, FP16 and INT8 format conversion, and quantization with **OpenVINO NNCF** 📈 | [![Github](../../assets/github.svg)][openvino_github] <br>[![Open In Collab](../../assets/colab.svg)][openvino_colab] <br>[![Ghost](../../assets/ghost.svg)][openvino_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Zero-Shot Image Classification with CLIP and LanceDB 📸** | Achieve zero-shot image classification using **CLIP** and **LanceDB**, enabling models to classify images without prior training on specific use cases, unlocking flexible and adaptable image classification capabilities 🔓 | [![Github](../../assets/github.svg)][zsic_github] <br>[![Open In Collab](../../assets/colab.svg)][zsic_colab] <br>[![Ghost](../../assets/ghost.svg)][zsic_ghost] |                                                                                                                                                                                                                                                                                                    
-
-
-
-
-[inbuilt_hybrid_search_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Inbuilt-Hybrid-Search
-[inbuilt_hybrid_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Inbuilt-Hybrid-Search/Inbuilt_Hybrid_Search_with_LanceDB.ipynb
-
-[BM25_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Hybrid_search_bm25_lancedb
-[BM25_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Hybrid_search_bm25_lancedb/main.ipynb
-[BM25_ghost]: https://blog.lancedb.com/hybrid-search-combining-bm25-and-semantic-search-for-better-results-with-lan-1358038fe7e6
-
-[NER_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/NER-powered-Semantic-Search
-[NER_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/NER-powered-Semantic-Search/NER_powered_Semantic_Search_with_LanceDB.ipynb
-[NER_ghost]: https://blog.lancedb.com/ner-powered-semantic-search-using-lancedb-51051dc3e493
-
-[audio_search_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/audio_search
-[audio_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.ipynb
-[audio_search_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.py
-
-[mls_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multi-lingual-wiki-qa
-[mls_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.ipynb
-[mls_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.py
-
-[fr_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/facial_recognition
-[fr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/facial_recognition/main.ipynb
-
-[sentiment_analysis_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews
-[sentiment_analysis_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews/Sentiment_Analysis_using_LanceDB.ipynb
-[sentiment_analysis_ghost]: https://blog.lancedb.com/sentiment-analysis-using-lancedb-2da3cb1e3fa6
-
-[arithmetic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Vector-Arithmetic-with-LanceDB
-[arithmetic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Vector-Arithmetic-with-LanceDB/main.ipynb
-[arithmetic_ghost]: https://blog.lancedb.com/vector-arithmetic-with-lancedb-an-intro-to-vector-embeddings/
-
-[imagebind_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/imagebind_demo
-[imagebind_huggingface]: https://huggingface.co/spaces/raghavd99/imagebind2
-
-[swi_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/search-within-images-with-sam-and-clip
-[swi_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/search-within-images-with-sam-and-clip/main.ipynb
-[swi_ghost]: https://blog.lancedb.com/search-within-an-image-331b54e4285e
-
-[zsod_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/zero-shot-object-detection-CLIP
-[zsod_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/zero-shot-object-detection-CLIP/zero_shot_object_detection_clip.ipynb
-
-[openvino_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Accelerate-Vector-Search-Applications-Using-OpenVINO
-[openvino_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Accelerate-Vector-Search-Applications-Using-OpenVINO/clip_text_image_search.ipynb
-[openvino_ghost]: https://blog.lancedb.com/accelerate-vector-search-applications-using-openvino-lancedb/
-
-[zsic_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/zero-shot-image-classification
-[zsic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/zero-shot-image-classification/main.ipynb
-[zsic_ghost]: https://blog.lancedb.com/zero-shot-image-classification-with-vector-search/
-
-
-
-
-

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -1,106 +0,0 @@
-# Serverless LanceDB
-
-## Store your data on S3 and use Lambda to compute embeddings and retrieve queries in production easily.
-
-<img id="splash" width="400" alt="s3-lambda" src="https://user-images.githubusercontent.com/917119/234653050-305a1e90-9305-40ab-b014-c823172a948c.png">
-
-This is a great option if you're wanting to scale with your use case and save effort and costs of maintenance.
-
-Let's walk through how to get a simple Lambda function that queries the SIFT dataset on S3.
-
-Before we start, you'll need to ensure you create a secure account access to AWS. We recommend using user policies, as this way AWS can share credentials securely without you having to pass around environment variables into Lambda.
-
-We'll also use a container to ship our Lambda code. This is a good option for Lambda as you don't have the space limits that you would otherwise by building a package yourself.
-
-# Initial setup: creating a LanceDB Table and storing it remotely on S3
-
-We'll use the SIFT vector dataset as an example. To make it easier, we've already made a Lance-format SIFT dataset publicly available, which we can access and use to populate our LanceDB Table. 
-
-To do this, download the Lance files locally first from:
-
-```
-s3://eto-public/datasets/sift/vec_data.lance
-```
-
-Then, we can write a quick Python script to populate our LanceDB Table:
-
-```python
-import lance
-sift_dataset = lance.dataset("/path/to/local/vec_data.lance")
-df = sift_dataset.to_table().to_pandas()
-
-import lancedb
-db = lancedb.connect(".")
-table = db.create_table("vector_example", df)
-```
-
-Once we've created our Table, we are free to move this data over to S3 so we can remotely host it.
-
-# Building our Lambda app: a simple event handler for vector search
-
-Now that we've got a remotely hosted LanceDB Table, we'll want to be able to query it from Lambda. To do so, let's create a new `Dockerfile` using the AWS python container base:
-
-```docker
-FROM public.ecr.aws/lambda/python:3.10
-
-RUN pip3 install --upgrade pip
-RUN pip3 install --no-cache-dir -U numpy --target "${LAMBDA_TASK_ROOT}"
-RUN pip3 install --no-cache-dir -U lancedb --target "${LAMBDA_TASK_ROOT}"
-
-COPY app.py ${LAMBDA_TASK_ROOT}
-
-CMD [ "app.handler" ]
-```
-
-Now let's make a simple Lambda function that queries the SIFT dataset in `app.py`.
-
-```python    
-import json
-import numpy as np
-import lancedb
-
-db = lancedb.connect("s3://eto-public/tables")
-table = db.open_table("vector_example")
-
-def handler(event, context):
-    status_code = 200
-
-    if event['query_vector'] is None:
-        status_code = 404
-        return {
-            "statusCode": status_code,
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body": json.dumps({
-                "Error ": "No vector to query was issued"
-            })
-        }
-    
-    # Shape of SIFT is (128,1M), d=float32
-    query_vector = np.array(event['query_vector'], dtype=np.float32)
-
-    rs = table.search(query_vector).limit(2).to_list()
-
-    return {
-        "statusCode": status_code,
-        "headers": {
-            "Content-Type": "application/json"
-        },
-        "body": json.dumps(rs)
-    }
-``` 
-
-# Deploying the container to ECR
-
-The next step is to build and push the container to ECR, where it can then be used to create a new Lambda function. 
-
-It's best to follow the official AWS documentation for how to do this, which you can view here:
-
-```
-https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-upload
-```
-
-# Final step: setting up your Lambda function
-
-Once the container is pushed, you can create a Lambda function by selecting the container. 

docs/src/examples/serverless_qa_bot_with_modal_and_langchain.md

@@ -1,166 +0,0 @@
-# Serverless QA Bot with Modal and LangChain
-
-## use LanceDB's LangChain integration with Modal to run a serverless app
-
-<img id="splash" width="400" alt="modal" src="https://github.com/lancedb/lancedb/assets/917119/7d80a40f-60d7-48a6-972f-dab05000eccf">
-
-We're going to build a QA bot for your documentation using LanceDB's LangChain integration and use Modal for deployment.
-
-Modal is an end-to-end compute platform for model inference, batch jobs, task queues, web apps and more. It's a great way to deploy your LanceDB models and apps.
-
-To get started, ensure that you have created an account and logged into [Modal](https://modal.com/). To follow along, the full source code is available on Github [here](https://github.com/lancedb/lancedb/blob/main/docs/src/examples/modal_langchain.py).
-
-### Setting up Modal
-
-We'll start by specifying our dependencies and creating a new Modal `Stub`:
-
-```python
-lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
-)
-
-stub = Stub(
-    name="example-langchain-lancedb",
-    image=lancedb_image,
-    secrets=[Secret.from_name("my-openai-secret")],
-)
-```
-
-We're using Modal's Secrets injection to secure our OpenAI key. To set your own, you can access the Modal UI and enter your key.
-
-### Setting up caches for LanceDB and LangChain
-
-Next, we can setup some globals to cache our LanceDB database, as well as our LangChain docsource:
-
-```python
-docsearch = None
-docs_path = Path("docs.pkl")
-db_path = Path("lancedb")
-```
-
-### Downloading our dataset
-
-We're going use a pregenerated dataset, which stores HTML files of the Pandas 2.0 documentation. 
-You could switch this out for your own dataset.
-
-```python
-def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
-    with open(Path("pandas.documentation.zip"), "wb") as f:
-        f.write(pandas_docs.content)
-
-    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
-    file.extractall(path=Path("pandas_docs"))
-```
-
-### Pre-processing the dataset and generating metadata
-
-Once we've downloaded it, we want to parse and pre-process them using LangChain, and then vectorize them and store it in LanceDB.
-Let's first create a function that uses LangChains `UnstructuredHTMLLoader` to parse them.
-We can then add our own metadata to it and store it alongside the data, we'll later be able to use this for filtering metadata.
-
-```python
-def store_docs():
-    docs = []
-
-    if not docs_path.exists():
-        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
-            if p.is_dir():
-                continue
-            loader = UnstructuredHTMLLoader(p)
-            raw_document = loader.load()
-
-            m = {}
-            m["title"] = get_document_title(raw_document[0])
-            m["version"] = "2.0rc0"
-            raw_document[0].metadata = raw_document[0].metadata | m
-            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
-            docs = docs + raw_document
-
-        with docs_path.open("wb") as fh:
-            pickle.dump(docs, fh)
-    else:
-        with docs_path.open("rb") as fh:
-            docs = pickle.load(fh)
-
-    return docs
-```
-
-### Simple LangChain chain for a QA bot
-
-Now we can create a simple LangChain chain for our QA bot. We'll use the `RecursiveCharacterTextSplitter` to split our documents into chunks, and then use the `OpenAIEmbeddings` to vectorize them.
-
-Lastly, we'll create a LanceDB table and store the vectorized documents in it, then create a `RetrievalQA` model from the chain and return it.
-
-```python
-def qanda_langchain(query):
-    download_docs()
-    docs = store_docs()
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
-    documents = text_splitter.split_documents(docs)
-    embeddings = OpenAIEmbeddings()
-
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
-    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
-    return qa.run(query)
-```
-
-### Creating our Modal entry points
-
-Now we can create our Modal entry points for our CLI and web endpoint:
-
-```python
-@stub.function()
-@web_endpoint(method="GET")
-def web(query: str):
-    answer = qanda_langchain(query)
-    return {
-        "answer": answer,
-    }
-    
-@stub.function()
-def cli(query: str):
-    answer = qanda_langchain(query)
-    print(answer)
-```
-
-# Testing it out!
-
-Testing the CLI:
-
-```bash
-modal run modal_langchain.py --query "What are the major differences in pandas 2.0?"
-```
-
-Testing the web endpoint:
-
-```bash
-modal serve modal_langchain.py
-```
-
-In the CLI, Modal will provide you a web endpoint. Copy this endpoint URI for the next step.
-Once this is served, then we can hit it with `curl`. 
-
-Note, the first time this runs, it will take a few minutes to download the dataset and vectorize it.
-An actual production example would pre-cache/load the dataset and vectorized documents prior
-
-```bash
-curl --get --data-urlencode "query=What are the major differences in pandas 2.0?" https://your-modal-endpoint-app.modal.run
-
-{"answer":" The major differences in pandas 2.0 include the ability to use any numpy numeric dtype in a Index, installing optional dependencies with pip extras, and enhancements, bug fixes, and performance improvements."}
-```
-

docs/src/examples/serverless_website_chatbot.md

@@ -1,61 +0,0 @@
-# LanceDB Chatbot - Vercel Next.js Template
-Use an AI chatbot with website context retrieved from a vector store like LanceDB. LanceDB is lightweight and can be embedded directly into Next.js, with data stored on-prem.
-
-## One click deploy on Vercel
-[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&env=OPENAI_API_KEY&envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&project-name=lancedb-vercel-chatbot&repository-name=lancedb-vercel-chatbot&demo-title=LanceDB%20Chatbot%20Demo&demo-description=Demo%20website%20chatbot%20with%20LanceDB.&demo-url=https%3A%2F%2Flancedb.vercel.app&demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png)
-
-![Demo website landing page](../assets/vercel-template.gif)
-
-## Development
-
-First, rename `.env.example` to `.env.local`, and fill out `OPENAI_API_KEY` with your OpenAI API key. You can get one [here](https://openai.com/blog/openai-api).
-
-Run the development server:
-
-```bash
-npm run dev
-# or
-yarn dev
-# or
-pnpm dev
-```
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
-
-## Learn More
-
-To learn more about LanceDB or Next.js, take a look at the following resources:
-
-- [LanceDB Documentation](https://lancedb.github.io/lancedb/) - learn about LanceDB, the developer-friendly serverless vector database.
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
-
-## LanceDB on Next.js and Vercel
-
-FYI: these configurations have been pre-implemented in this template.
-
-Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
-```js
-/** @type {import('next').NextConfig} */
-module.exports = ({
-  webpack(config) {
-    config.externals.push({ vectordb: 'vectordb' })
-    return config;
-  }
-})
-```
-
-To deploy on Vercel, we need to make sure that the NodeJS runtime static file analysis for Vercel can find the binary, since LanceDB uses dynamic imports by default. We can do this by modifying `package.json` in the `scripts` section.
-```json
-{
-  ...
-  "scripts": {
-    ...
-    "vercel-build": "sed -i 's/nativeLib = require(`@lancedb\\/vectordb-\\${currentTarget()}`);/nativeLib = require(`@lancedb\\/vectordb-linux-x64-gnu`);/' node_modules/vectordb/native.js && next build",
-    ...
-  },
-  ...
-}
-```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -1,121 +0,0 @@
-# Vector embedding search using TransformersJS
-
-## Embed and query data from LanceDB using TransformersJS
-
-<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
-
-This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
-
-
-### Setting up
-First, install the dependencies:
-```bash
-npm install vectordb
-npm i @xenova/transformers
-```
-
-We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
-
-Within our `index.js` file we will import the necessary libraries and define our model and database:
-
-```javascript
-const lancedb = require('vectordb')
-const { pipeline } = await import('@xenova/transformers')
-const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-```
-
-### Creating the embedding function
-
-Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
-
-```javascript
-// Define the function. `sourceColumn` is required for LanceDB to know
-// which column to use as input.
-const embed_fun = {}
-embed_fun.sourceColumn = 'text'
-embed_fun.embed = async function (batch) {
-    let result = []
-    // Given a batch of strings, we will use the `pipe` function to get
-    // the vector embedding of each string.
-    for (let text of batch) {
-        // 'mean' pooling and normalizing allows the embeddings to share the
-        // same length.
-        const res = await pipe(text, { pooling: 'mean', normalize: true })
-        result.push(Array.from(res['data']))
-    }
-    return (result)
-}
-```
-
-### Creating the database
-
-Now, we will create the LanceDB database and add the embedding function we defined earlier.
-
-```javascript
-// Link a folder and create a table with data
-const db = await lancedb.connect('data/sample-lancedb')
-
-// You can also import any other data, but make sure that you have a column
-// for the embedding function to use.
-const data = [
-    { id: 1, text: 'Cherry', type: 'fruit' },
-    { id: 2, text: 'Carrot', type: 'vegetable' },
-    { id: 3, text: 'Potato', type: 'vegetable' },
-    { id: 4, text: 'Apple', type: 'fruit' },
-    { id: 5, text: 'Banana', type: 'fruit' }
-]
-
-// Create the table with the embedding function
-const table = await db.createTable('food_table', data, "create", embed_fun)
-```
-
-### Performing the search
-
-Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
-
-```javascript
-// Query the table
-const results = await table
-    .search("a sweet fruit to eat")
-    .metricType("cosine")
-    .limit(2)
-    .execute()
-console.log(results.map(r => r.text))
-```
-```bash
-[ 'Banana', 'Cherry' ]
-```
-
-Output of `results`:
-```bash
-[
-  {
-    vector: Float32Array(384) [
-      -0.057455405592918396,
-      0.03617725893855095,
-      -0.0367760956287384,
-      ... 381 more items
-    ],
-    id: 5,
-    text: 'Banana',
-    type: 'fruit',
-    _distance: 0.4919965863227844
-  },
-  {
-    vector: Float32Array(384) [
-      0.0009714411571621895,
-      0.008223623037338257,
-      0.009571489877998829,
-      ... 381 more items
-    ],
-    id: 1,
-    text: 'Cherry',
-    type: 'fruit',
-    _distance: 0.5540297031402588
-  }
-]
-```
-
-### Wrapping it up
-
-In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!

docs/src/examples/youtube_transcript_bot.md

@@ -1,13 +0,0 @@
-# YouTube transcript search
-
-## Search through youtube transcripts using natural language with LanceDB
-
-<img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
-
-
-<a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
-
-Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipesexamples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/examples/youtube_bot/index.js)
-
-
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)

docs/src/examples/youtube_transcript_bot_with_nodejs.md

@@ -1,139 +0,0 @@
-# YouTube transcript QA bot with NodeJS
-
-## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
-
-<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
-
-This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how to use LanceDB's Javascript API to store and manage your data easily.
-
-```bash
-npm install vectordb
-```
-
-## Download the data
-
-For this example, we're using a sample of a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`. Download and extract this file under the `data` folder:
-
-```bash
-wget -c https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-transcriptions_sample.jsonl
-```
-
-## Prepare Context
-
-Each item in the dataset contains just a short chunk of text. We'll need to merge a bunch of these chunks together on a rolling basis. For this demo, we'll look back 20 records to create a more complete context for each sentence.
-
-First, we need to read and parse the input file.
-
-```javascript
-const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
-  .toString()
-  .split('\n')
-  .filter(line => line.length > 0)
-  .map(line => JSON.parse(line))
-
-const data = contextualize(lines, 20, 'video_id')
-```
-
-The contextualize function groups the transcripts by video_id and then creates the expanded context for each item.
-
-```javascript
-function contextualize (rows, contextSize, groupColumn) {
-  const grouped = []
-  rows.forEach(row => {
-    if (!grouped[row[groupColumn]]) {
-      grouped[row[groupColumn]] = []
-    }
-    grouped[row[groupColumn]].push(row)
-  })
-
-  const data = []
-  Object.keys(grouped).forEach(key => {
-    for (let i = 0; i < grouped[key].length; i++) {
-      const start = i - contextSize > 0 ? i - contextSize : 0
-      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
-    }
-    data.push(...grouped[key])
-  })
-  return data
-}
-```
-
-## Create the LanceDB Table
-
-To load our data into LanceDB, we need to create embedding (vectors) for each item. For this example, we will use the OpenAI embedding functions, which have a native integration with LanceDB.
-
-```javascript
-// You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-const apiKey = process.env.OPENAI_API_KEY
-// The embedding function will create embeddings for the 'context' column
-const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
-// Connects to LanceDB
-const db = await lancedb.connect('data/youtube-lancedb')
-const tbl = await db.createTable('vectors', data, embedFunction)
-```
-
-## Create and answer the prompt
-
-We will accept questions in natural language and use our corpus stored in LanceDB to answer them. First, we need to set up the OpenAI client:
-
-```javascript
-const configuration = new Configuration({ apiKey })
-const openai = new OpenAIApi(configuration)
-```
-
-Then we can prompt questions and use LanceDB to retrieve the three most relevant transcripts for this prompt.
-
-```javascript
-const query = await rl.question('Prompt: ')
-const results = await tbl
-  .search(query)
-  .select(['title', 'text', 'context'])
-  .limit(3)
-  .execute()
-```
-
-The query and the transcripts' context are appended together in a single prompt:
-
-```javascript
-function createPrompt (query, context) {
-    let prompt =
-        'Answer the question based on the context below.\n\n' +
-        'Context:\n'
-
-    // need to make sure our prompt is not larger than max size
-    prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
-    prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
-    return prompt
-}
-```
-
-We can now use the OpenAI Completion API to process our custom prompt and give us an answer.
-
-```javascript
-const response = await openai.createCompletion({
-  model: 'text-davinci-003',
-  prompt: createPrompt(query, results),
-  max_tokens: 400,
-  temperature: 0,
-  top_p: 1,
-  frequency_penalty: 0,
-  presence_penalty: 0
-})
-console.log(response.data.choices[0].text)
-```
-
-## Let's put it all together now
-
-Now we can provide queries and have them answered based on your local LanceDB data.
-
-```bash
-Prompt: who was the 12th person on the moon and when did they land?
- The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
-Prompt: Which training method should I use for sentence transformers when I only have pairs of related sentences?
- NLI with multiple negative ranking loss.
-```
-
-## That's a wrap
-
-In this example, you learned how to use LanceDB to store and query embedding representations of your local data. The complete example code is on [GitHub](https://github.com/lancedb/lancedb/tree/main/node/examples), and you can also download the LanceDB dataset using [this link](https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-lancedb.zip).
-

docs/src/extra_js/init_ask_ai_widget.js

@@ -1,79 +0,0 @@
-// Creates an SVG robot icon (from Lucide)
-function robotSVG() {
-  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
-  svg.setAttribute("width", "24");
-  svg.setAttribute("height", "24");
-  svg.setAttribute("viewBox", "0 0 24 24");
-  svg.setAttribute("fill", "none");
-  svg.setAttribute("stroke", "currentColor");
-  svg.setAttribute("stroke-width", "2");
-  svg.setAttribute("stroke-linecap", "round");
-  svg.setAttribute("stroke-linejoin", "round");
-  svg.setAttribute("class", "lucide lucide-bot-message-square");
-
-  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path1.setAttribute("d", "M12 6V2H8");
-  svg.appendChild(path1);
-
-  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
-  svg.appendChild(path2);
-
-  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path3.setAttribute("d", "M2 12h2");
-  svg.appendChild(path3);
-
-  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path4.setAttribute("d", "M9 11v2");
-  svg.appendChild(path4);
-
-  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path5.setAttribute("d", "M15 11v2");
-  svg.appendChild(path5);
-
-  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path6.setAttribute("d", "M20 12h2");
-  svg.appendChild(path6);
-
-  return svg
-}
-
-// Creates the Fluidic Chatbot buttom
-function fluidicButton() {
-  var btn = document.createElement("a");
-  btn.href = "https://asklancedb.com";
-  btn.target = "_blank";
-  btn.style.position = "fixed";
-  btn.style.fontWeight = "bold";
-  btn.style.fontSize = ".8rem";
-  btn.style.right = "10px";
-  btn.style.bottom = "10px";
-  btn.style.width = "80px";
-  btn.style.height = "80px";
-  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
-  btn.style.color = "white";
-  btn.style.borderRadius = "5px";
-  btn.style.display = "flex";
-  btn.style.flexDirection = "column";
-  btn.style.justifyContent = "center";
-  btn.style.alignItems = "center";
-  btn.style.zIndex = "1000";
-  btn.style.opacity = "0";
-  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
-  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
-
-  setTimeout(function() {
-      btn.style.opacity = "1";
-      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
-  }, 0);
-
-  return btn
-}
-
-document.addEventListener("DOMContentLoaded", function() {
-  var btn = fluidicButton()
-  btn.appendChild(robotSVG());
-  var text = document.createTextNode("Ask AI");
-  btn.appendChild(text);
-  document.body.appendChild(btn);
-});

docs/src/faq.md

@@ -1,87 +0,0 @@
-This section covers some common questions and issues that you may encounter when using LanceDB.
-
-### Is LanceDB open source?
-
-Yes, LanceDB is an open source vector database available under an Apache 2.0 license. We also have a serverless SaaS solution, LanceDB Cloud, available under a commercial license.
-
-### What is the difference between Lance and LanceDB?
-
-[Lance](https://github.com/lancedb/lance) is a modern columnar data format for AI, written in Rust 🦀. It’s perfect for building search engines, feature stores and being the foundation of large-scale ML training jobs requiring high performance IO and shuffles. It also has native support for storing, querying, and inspecting deeply nested data for robotics or large blobs like images, point clouds, and more.
-
-LanceDB is the vector database that’s built on top of Lance, and utilizes the underlying optimized storage format to build efficient disk-based indexes that power semantic search & retrieval applications, from RAGs to QA Bots to recommender systems.
-
-### Why invent another data format instead of using Parquet?
-
-As we mention in our talk titled “[Lance, a modern columnar data format](https://www.youtube.com/watch?v=ixpbVyrsuL8)”, Parquet and other tabular formats that derive from it are rather dated (Parquet is over 10 years old), especially when it comes to random access on vectors. We needed a format that’s able to handle the complex trade-offs involved in shuffling, scanning, OLAP and filtering large datasets involving vectors, and our extensive experiments with Parquet didn't yield sufficient levels of performance for modern ML. [Our benchmarks](https://blog.lancedb.com/benchmarking-random-access-in-lance-ed690757a826) show that Lance is up to 1000x faster than Parquet for random access, which we believe justifies our decision to create a new data format for AI.
-
-### Why build in Rust? 🦀
-
-We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
-
-### What is the difference between LanceDB OSS and LanceDB Cloud?
-
-LanceDB OSS is an **embedded** (in-process) solution that can be used as the vector store of choice for your LLM and RAG applications. It can be embedded inside an existing application backend, or used in-process alongside existing ML and data engineering pipelines.
-
-LanceDB Cloud is a **serverless** solution — the database and data sit on the cloud and we manage the scalability of the application side via a remote client, without the need to manage any infrastructure.
-
-Both flavors of LanceDB benefit from the blazing fast Lance data format and are built on the same open source foundations.
-
-### What makes LanceDB different?
-
-LanceDB is among the few embedded vector DBs out there that we believe can unlock a whole new class of LLM-powered applications in the browser or via edge functions. Lance’s multi-modal nature allows you to store the raw data, metadata and the embeddings all at once, unlike other solutions that typically store just the embeddings and metadata.
-
-The Lance data format that powers our storage system also provides true zero-copy access and seamless interoperability with numerous other data formats (like Pandas, Polars, Pydantic) via Apache Arrow, as well as automatic data versioning and data management without needing extra infrastructure.
-
-### How large of a dataset can LanceDB handle?
-
-LanceDB and its underlying data format, Lance, are built to scale to really large amounts of data (hundreds of terabytes). We are currently working with customers who regularly perform operations on 200M+ vectors, and we’re fast approaching billion scale and beyond, which are well-handled by our disk-based indexes, without you having to break the bank.
-
-### Do I need to build an ANN index to run vector search?
-
-No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
-
-For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. See the [ANN indexes](ann_indexes.md) section for more details.
-
-### Does LanceDB support full-text search?
-
-Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
-
-### How can I speed up data inserts?
-
-It's highly recommend to perform bulk inserts via batches (for e.g., Pandas DataFrames or lists of dicts in Python) to speed up inserts for large datasets. Inserting records one at a time is slow and can result in suboptimal performance because each insert creates a new data fragment on disk. Batching inserts allows LanceDB to create larger fragments (and their associated manifests), which are more efficient to read and write.
-
-### Do I need to set a refine factor when using an index?
-
-Yes. LanceDB uses PQ, or Product Quantization, to compress vectors and speed up search when using an ANN index. However, because PQ is a lossy compression algorithm, it tends to reduce recall while also reducing the index size. To address this trade-off, we introduce a process called **refinement**. The normal process computes distances by operating on the compressed PQ vectors. The refinement factor (*rf*) is a multiplier that takes the top-k similar PQ vectors to a given query, fetches `rf * k` *full* vectors and computes the raw vector distances between them and the query vector, reordering the top-k results based on these scores instead.
-
-For example, if you're retrieving the top 10 results and set `refine_factor` to 25, LanceDB will fetch the 250 most similar vectors (according to PQ), compute the distances again based on the full vectors for those 250 and then re-rank based on their scores. This can significantly improve recall, with a small added latency cost (typically a few milliseconds), so it's recommended you set a `refine_factor` of anywhere between 5-50 and measure its impact on latency prior to deploying your solution.
-
-### How can I improve IVF-PQ recall while keeping latency low?
-
-When using an IVF-PQ index, there's a trade-off between recall and latency at query time. You can improve recall by increasing the number of probes and the `refine_factor`. In our benchmark on the GIST-1M dataset, we show that it's possible to achieve >0.95 recall with a latency of under 10 ms on most systems, using ~50 probes and a `refine_factor` of 50. This is, of course, subject to the dataset at hand and a quick sensitivity study can be performed on your own data. You can find more details on the benchmark in our [blog post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a).
-
-![](assets/recall-vs-latency.webp)
-
-### How do I connect to MinIO?
-
-MinIO supports an S3 compatible API. In order to connect to a MinIO instance, you need to:
-
-- Set the envvar `AWS_ENDPOINT` to the URL of your MinIO API
-- Set the envvars `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your MinIO credential
-- Call `lancedb.connect("s3://minio_bucket_name")`
-
-### Where can I find benchmarks for LanceDB?
-
-Refer to this [post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a) for recent benchmarks.
-
-### How much data can LanceDB practically manage without effecting performance?
-
-We target good performance on ~10-50 billion rows and ~10-30 TB of data.
-
-### Does LanceDB support concurrent operations?
-
-LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the [storage layer](https://lancedb.github.io/lancedb/concepts/storage/) you've chosen scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit
-
-!!! info "Multiprocessing with LanceDB"
-
-    For multiprocessing you should probably not use ```fork``` as lance is multi-threaded internally and ```fork``` and multi-thread do not work well.[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)

docs/src/fts.md

@@ -1,258 +0,0 @@
-# Full-text search (Native FTS)
-
-LanceDB provides support for full-text search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
-
-!!! note
-    The Python SDK uses tantivy-based FTS by default, need to pass `use_tantivy=False` to use native FTS.
-
-## Example
-
-Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
-
-=== "Python"
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    import * as lancedb from "@lancedb/lancedb";
-    const uri = "data/sample-lancedb"
-    const db = await lancedb.connect(uri);
-
-    const data = [
-    { vector: [3.1, 4.1], text: "Frodo was a happy puppy" },
-    { vector: [5.9, 26.5], text: "There are several kittens playing" },
-    ];
-    const tbl = await db.createTable("my_table", data, { mode: "overwrite" });
-    await tbl.createIndex("text", {
-        config: lancedb.Index.fts(),
-    });
-
-    await tbl
-        .search("puppy", "fts")
-        .select(["text"])
-        .limit(10)
-        .toArray();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let uri = "data/sample-lancedb";
-    let db = connect(uri).execute().await?;
-    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    let tbl = db
-        .create_table("my_table", initial_data)
-        .execute()
-        .await?;
-    tbl
-        .create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
-        .execute()
-        .await?;
-
-    tbl
-        .query()
-        .full_text_search(FullTextSearchQuery::new("puppy".to_owned()))
-        .select(lancedb::query::Select::Columns(vec!["text".to_owned()]))
-        .limit(10)
-        .execute()
-        .await?;
-    ```
-
-It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
-
-Passing `fts_columns="text"` if you want to specify the columns to search.
-
-!!! note
-    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
-
-## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are with length greater than 40, and lowercase all words.
-
-Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.
-
-For example, to enable stemming for English:
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem_async"
-    ```
-
-the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
-
-The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.
-
-For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding_async"
-    ```
-
-## Filtering
-
-LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported.
-
-This can be invoked via the familiar `where` syntax.
- 
-With pre-filtering:
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl
-    .search("puppy")
-    .select(["id", "doc"])
-    .limit(10)
-    .where("meta='foo'")
-    .prefilter(true)
-    .toArray();
-    ```
-
-=== "Rust"
-
-    ```rust
-    table
-        .query()
-        .full_text_search(FullTextSearchQuery::new("puppy".to_owned()))
-        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
-        .limit(10)
-        .only_if("meta='foo'")
-        .execute()
-        .await?;
-    ```
-
-With post-filtering:
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl
-    .search("apple")
-    .select(["id", "doc"])
-    .limit(10)
-    .where("meta='foo'")
-    .prefilter(false)
-    .toArray();
-    ```
-
-=== "Rust"
-
-    ```rust
-    table
-        .query()
-        .full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
-        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
-        .postfilter()
-        .limit(10)
-        .only_if("meta='foo'")
-        .execute()
-        .await?;
-    ```
-
-## Phrase queries vs. terms queries
-
-!!! warning "Warn"
-    Lance-based FTS doesn't support queries using boolean operators `OR`, `AND`.
-
-For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `old man sea`. For more details on the terms
-query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
-
-To search for a phrase, the index must be created with `with_position=True`:
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position_async"
-    ```
-This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
-
-
-## Incremental indexing
-
-LanceDB supports incremental indexing, which means you can add new records to the table without reindexing the entire table.
-
-This can make the query more efficient, especially when the table is large and the new records are relatively small.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [3.1, 4.1], text: "Frodo was a happy puppy" }]);
-    await tbl.optimize();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
-    ```
-!!! note
-
-    New data added after creating the FTS index will appear in search results while incremental index is still progress, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 

docs/src/fts_tantivy.md

@@ -1,160 +0,0 @@
-# Full-text search (Tantivy-based FTS)
-
-LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
-
-The tantivy-based FTS is only available in Python synchronous APIs and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
-
-## Installation
-
-To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
-
-```sh
-# Say you want to use tantivy==0.20.1
-pip install tantivy==0.20.1
-```
-
-## Example
-
-Consider that we have a LanceDB table named `my_table`, whose string column `content` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
-
-```python
-import lancedb
-
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-table = db.create_table(
-    "my_table",
-    data=[
-        {"id": 1, "vector": [3.1, 4.1], "title": "happy puppy", "content": "Frodo was a happy puppy", "meta": "foo"},
-        {"id": 2, "vector": [5.9, 26.5], "title": "playing kittens", "content": "There are several kittens playing around the puppy", "meta": "bar"},
-    ],
-)
-
-# passing `use_tantivy=False` to use lance FTS index
-# `use_tantivy=True` by default
-table.create_fts_index("content", use_tantivy=True)
-table.search("puppy").limit(10).select(["content"]).to_list()
-# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
-# ...
-```
-
-It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
-
-!!! note
-    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
-
-## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
-
-```python
-table.create_fts_index("content", use_tantivy=True, tokenizer_name="en_stem", replace=True)
-```
-
-the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
-
-## Index multiple columns
-
-If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
-
-```python
-table.create_fts_index(["title", "content"], use_tantivy=True, replace=True)
-```
-
-Note that the search API call does not change - you can search over all indexed columns at once.
-
-## Filtering
-
-Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
-applied on top of the full text search results (see [native FTS](fts.md) if you need pre-filtering). This can be invoked via the familiar
-`where` syntax:
-
-```python
-table.search("puppy").limit(10).where("meta='foo'").to_list()
-```
-
-## Sorting
-
-You can pre-sort the documents by specifying `ordering_field_names` when
-creating the full-text search index. Once pre-sorted, you can then specify
-`ordering_field_name` while searching to return results sorted by the given
-field. For example,
-
-```python
-table.create_fts_index(["content"], use_tantivy=True, ordering_field_names=["id"], replace=True)
-
-(table.search("puppy", ordering_field_name="id")
- .limit(20)
- .to_list())
-```
-
-!!! note
-    If you wish to specify an ordering field at query time, you must also
-    have specified it during indexing time. Otherwise at query time, an
-    error will be raised that looks like `ValueError: The field does not exist: xxx`
-
-!!! note
-    The fields to sort on must be of typed unsigned integer, or else you will see
-    an error during indexing that looks like
-    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
-
-!!! note
-    You can specify multiple fields for ordering at indexing time.
-    But at query time only one ordering field is supported.
-
-
-## Phrase queries vs. terms queries
-
-For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
-query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
-
-!!! tip "Note"
-    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
-
-    ```py
-    # This raises a syntax error
-    table.search("they could have been dogs OR cats")
-    ```
-
-    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
-    the query is treated as a phrase query.
-
-    ```py
-    # This works!
-    table.search("they could have been dogs or cats")
-    ```
-
-It can be cumbersome to have to remember what will cause a syntax error depending on the type of
-query you want to perform. To make this simpler, when you want to perform a phrase query, you can
-enforce it in one of two ways:
-
-1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
-a phrase query.
-1. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
-itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
-is treated as a phrase query.
-
-In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
-double quotes replaced by single quotes.
-
-
-## Configurations
-
-By default, LanceDB configures a 1GB heap size limit for creating the index. You can
-reduce this if running on a smaller node, or increase this for faster performance while
-indexing a larger corpus.
-
-```python
-# configure a 512MB heap size
-heap = 1024 * 1024 * 512
-table.create_fts_index(["title", "content"], use_tantivy=True, writer_heap_size=heap, replace=True)
-```
-
-## Current limitations
-
-1. New data added after creating the FTS index will appear in search results, but with increased latency due to a flat search on the unindexed portion. Re-indexing with `create_fts_index` will reduce latency. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 
-
-2. We currently only support local filesystem paths for the FTS index.
-   This is a tantivy limitation. We've implemented an object store plugin
-   but there's no way in tantivy-py to specify to use it.

docs/src/guides/multi-vector.md

@@ -1,85 +0,0 @@
-# Late interaction & MultiVector embedding type
-Late interaction is a technique used in retrieval that calculates the relevance of a query to a document by comparing their multi-vector representations. The key difference between late interaction and other popular methods:
-
-![late interaction vs other methods](https://raw.githubusercontent.com/lancedb/assets/b035a0ceb2c237734e0d393054c146d289792339/docs/assets/integration/colbert-blog-interaction.svg)
-
-
-[ Illustration from https://jina.ai/news/what-is-colbert-and-late-interaction-and-why-they-matter-in-search/]
-
-<b>No interaction:</b> Refers to independently embedding the query and document, that are compared to calcualte similarity without any interaction between them. This is typically used in vector search operations.
-
-<b>Partial interaction</b> Refers to a specific approach where the similarity computation happens primarily between query vectors and document vectors, without extensive interaction between individual components of each. An example of this is dual-encoder models like BERT.
-
-<b>Early full interaction</b> Refers to techniques like cross-encoders that process query and docs in pairs with full interaction across various stages of encoding. This is a powerful, but relatively slower technique. Because it requires processing query and docs in pairs, doc embeddings can't be pre-computed for fast retrieval. This is why cross encoders are typically used as reranking models combined with vector search. Learn more about [LanceDB Reranking support](https://lancedb.github.io/lancedb/reranking/).
-
-<b>Late interaction</b> Late interaction is a technique that calculates the doc and query similarity independently and then the interaction or evaluation happens during the retrieval process. This is typically used in retrieval models like ColBERT. Unlike early interaction, It allows speeding up the retrieval process without compromising the depth of semantic analysis.
-
-## Internals of ColBERT 
-Let's take a look at the steps involved in performing late interaction based retrieval using ColBERT:
-
-• ColBERT employs BERT-based encoders for both queries `(fQ)` and documents `(fD)`
-• A single BERT model is shared between query and document encoders and special tokens distinguish input types: `[Q]` for queries and `[D]` for documents
-
-**Query Encoder (fQ):**
-• Query q is tokenized into WordPiece tokens: `q1, q2, ..., ql`. `[Q]` token is prepended right after BERT's `[CLS]` token
-• If query length < Nq, it's padded with [MASK] tokens up to Nq.
-• The padded sequence goes through BERT's transformer architecture
-• Final embeddings are L2-normalized.
-
-**Document Encoder (fD):**
-• Document d is tokenized into tokens `d1, d2, ..., dm`. `[D]` token is prepended after `[CLS]` token
-• Unlike queries, documents are NOT padded with `[MASK]` tokens
-• Document tokens are processed through BERT and the same linear layer
-
-**Late Interaction:**
-• Late interaction estimates relevance score `S(q,d)` using embedding `Eq` and `Ed`. Late interaction happens after independent encoding
-• For each query embedding, maximum similarity is computed against all document embeddings
-• The similarity measure can be cosine similarity or squared L2 distance
-
-**MaxSim Calculation:**
-```
-S(q,d) := Σ max(Eqi⋅EdjT)
-          i∈|Eq| j∈|Ed|
-```
-• This finds the best matching document embedding for each query embedding
-• Captures relevance based on strongest local matches between contextual embeddings
-
-## LanceDB MultiVector type
-LanceDB supports multivector type, this is useful when you have multiple vectors for a single item (e.g. with ColBert and ColPali).
-
-You can index on a column with multivector type and search on it, the query can be single vector or multiple vectors. For now, only cosine metric is supported for multivector search. The vector value type can be float16, float32 or float64. LanceDB integrateds [ConteXtualized Token Retriever(XTR)](https://arxiv.org/abs/2304.01982), which introduces a simple, yet novel, objective function that encourages the model to retrieve the most important document tokens first. 
-
-```python
-import lancedb
-import numpy as np
-import pyarrow as pa
-
-db = lancedb.connect("data/multivector_demo")
-schema = pa.schema(
-    [
-        pa.field("id", pa.int64()),
-        # float16, float32, and float64 are supported
-        pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
-    ]
-)
-data = [
-    {
-        "id": i,
-        "vector": np.random.random(size=(2, 256)).tolist(),
-    }
-    for i in range(1024)
-]
-tbl = db.create_table("my_table", data=data, schema=schema)
-
-# only cosine similarity is supported for multi-vectors
-tbl.create_index(metric="cosine")
-
-# query with single vector
-query = np.random.random(256).astype(np.float16)
-tbl.search(query).to_arrow()
-
-# query with multiple vectors
-query = np.random.random(size=(2, 256))
-tbl.search(query).to_arrow()
-```
-Find more about vector search in LanceDB [here](https://lancedb.github.io/lancedb/search/#multivector-type).

docs/src/guides/scalar_index.md

@@ -1,156 +0,0 @@
-# Building a Scalar Index
-
-Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
-
-Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
-over scalar columns.
-
-- `BTREE`: The most common type is BTREE. The index stores a copy of the
-  column in sorted order. This sorted copy allows a binary search to be used to
-  satisfy queries.
-- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
-  uses a series of bits to indicate whether a value is present in a row of a table
-- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
-  support queries with `array_contains_all` and `array_contains_any`
-  using an underlying bitmap index.
-  For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.
-
-!!! tips "How to choose the right scalar index type"
-
-    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
-    
-    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
-    
-    `LABEL_LIST`: This index should be used for columns containing list-type data.
-
-| Data Type                                                       | Filter                                    | Index Type   |
-| --------------------------------------------------------------- | ----------------------------------------- | ------------ |
-| Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
-| Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
-| List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |
-
-### Create a scalar index
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```js
-        const db = await lancedb.connect("data");
-        const tbl = await db.openTable("my_vectors");
-
-        await tbl.create_index("book_id");
-        await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
-        ```
-
-The following scan will be faster if the column `book_id` has a scalar index:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```js
-        const db = await lancedb.connect("data");
-        const tbl = await db.openTable("books");
-
-        await tbl
-          .query()
-          .where("book_id = 2")
-          .limit(10)
-          .toArray();
-        ```
-
-Scalar indices can also speed up scans containing a vector search or full text search, and a prefilter:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```js
-        const db = await lancedb.connect("data/lance");
-        const tbl = await db.openTable("book_with_embeddings");
-
-        await tbl.search(Array(1536).fill(1.2))
-          .where("book_id != 3")  // prefilter is default behavior.
-          .limit(10)
-          .toArray();
-        ```
-### Update a scalar index
-Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
-    await tbl.optimize();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
-    ```
-
-!!! note
-
-    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.

docs/src/guides/sql_querying.md

@@ -1,60 +0,0 @@
-# SQL Querying
-
-You can use DuckDB and Apache Datafusion to query your LanceDB tables using SQL.
-This guide will show how to query Lance tables them using both.
-
-We will re-use the dataset [created previously](./tables.md):
-
-```python
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-data = [
-    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
-]
-table = db.create_table("pd_table", data=data)
-```
-
-## Querying a LanceDB Table with DuckDb
-
-The `to_lance` method converts the LanceDB table to a `LanceDataset`, which is accessible to DuckDB through the Arrow compatibility layer.
-To query the resulting Lance dataset in DuckDB, all you need to do is reference the dataset by the same name in your SQL query.
-
-```python
-import duckdb
-
-arrow_table = table.to_lance()
-
-duckdb.query("SELECT * FROM arrow_table")
-```
-
-| vector      | item | price |
-| ----------- | ---- | ----- |
-| [3.1, 4.1]  | foo  | 10.0  |
-| [5.9, 26.5] | bar  | 20.0  |
-
-## Querying a LanceDB Table with Apache Datafusion
-
-Have the required imports before doing any querying.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-session-context"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-ffi-dataset"
-    ```
-
-Register the table created with the Datafusion session context.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:lance_sql_basic"
-    ```
-
-| vector      | item | price |
-| ----------- | ---- | ----- |
-| [3.1, 4.1]  | foo  | 10.0  |
-| [5.9, 26.5] | bar  | 20.0  |

docs/src/guides/storage.md

@@ -1,812 +0,0 @@
-# Configuring cloud storage
-
-<!-- TODO: When we add documentation for how to configure other storage types
-           we can change the name to a more general "Configuring storage" -->
-
-When using LanceDB OSS, you can choose where to store your data. The tradeoffs between different storage options are discussed in the [storage concepts guide](../concepts/storage.md). This guide shows how to configure LanceDB to use different storage options.
-
-## Object Stores
-
-LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure Blob Store, and Google Cloud Storage. Which object store to use is determined by the URI scheme of the dataset path. `s3://` is used for AWS S3, `az://` is used for Azure Blob Storage, and `gs://` is used for Google Cloud Storage. These URIs are passed to the `connect` function:
-
-=== "Python"
-
-    AWS S3:
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        ```
-
-    Google Cloud Storage:
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("gs://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("gs://bucket/path")
-        ```
-
-    Azure Blob Storage:
-
-    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("az://bucket/path")
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("az://bucket/path")
-        ```
-    Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
-
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        AWS S3:
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("s3://bucket/path");
-        ```
-
-        Google Cloud Storage:
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("gs://bucket/path");
-        ```
-
-        Azure Blob Storage:
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("az://bucket/path");
-        ```
-
-
-    === "vectordb (deprecated)"
-
-        AWS S3:
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("s3://bucket/path");
-        ```
-
-        Google Cloud Storage:
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("gs://bucket/path");
-        ```
-
-        Azure Blob Storage:
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("az://bucket/path");
-        ```
-
-In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
-
-```bash
-export TIMEOUT=60s
-```
-
-If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-
-        const db = await lancedb.connect("s3://bucket/path", {
-            storageOptions: {timeout: "60s"}
-        });
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("s3://bucket/path", {
-            storageOptions: {timeout: "60s"}
-        });
-        ```
-
-Getting even more specific, you can set the `timeout` for only a particular table:
-
-=== "Python"
-
-    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        table = db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        async_table = await async_db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        <!-- skip-test -->
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("s3://bucket/path");
-        const table = db.createTable(
-            "table",
-            [{ a: 1, b: 2}],
-            {storageOptions: {timeout: "60s"}}
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        <!-- skip-test -->
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("s3://bucket/path");
-        const table = db.createTable(
-            "table",
-            [{ a: 1, b: 2}],
-            {storageOptions: {timeout: "60s"}}
-        );
-        ```
-
-!!! info "Storage option casing"
-
-    The storage option keys are case-insensitive. So `connect_timeout` and `CONNECT_TIMEOUT` are the same setting. Usually lowercase is used in the `storage_options` argument and uppercase is used for environment variables. In the `lancedb` Node package, the keys can also be provided in `camelCase` capitalization. For example, `connectTimeout` is equivalent to `connect_timeout`.
-
-### General configuration
-
-There are several options that can be set for all object stores, mostly related to network client configuration.
-
-<!-- from here: https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html -->
-
-| Key                        | Description                                                                                      |
-|----------------------------|--------------------------------------------------------------------------------------------------|
-| `allow_http`               | Allow non-TLS, i.e. non-HTTPS connections. Default: `False`.                                      |
-| `allow_invalid_certificates`| Skip certificate validation on HTTPS connections. Default: `False`.                               |
-| `connect_timeout`          | Timeout for only the connect phase of a Client. Default: `5s`.                                    |
-| `timeout`                  | Timeout for the entire request, from connection until the response body has finished. Default: `30s`. |
-| `user_agent`               | User agent string to use in requests.                                                             |
-| `proxy_url`                | URL of a proxy server to use for requests. Default: `None`.                                       |
-| `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
-| `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
-
-### AWS S3
-
-To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
-These can be set as environment variables or passed in the `storage_options` parameter:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    awsAccessKeyId: "my-access-key",
-                    awsSecretAccessKey: "my-secret-key",
-                    awsSessionToken: "my-session-token",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    awsAccessKeyId: "my-access-key",
-                    awsSecretAccessKey: "my-secret-key",
-                    awsSessionToken: "my-session-token",
-                }
-            }
-        );
-        ```
-
-Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
-
-The following keys can be used as both environment variables or keys in the `storage_options` parameter:
-
-| Key                                | Description                                                                                          |
-|------------------------------------|------------------------------------------------------------------------------------------------------|
-| `aws_region` / `region`             | The AWS region the bucket is in. This can be automatically detected when using AWS S3, but must be specified for S3-compatible stores. |
-| `aws_access_key_id` / `access_key_id` | The AWS access key ID to use.                                                                       |
-| `aws_secret_access_key` / `secret_access_key` | The AWS secret access key to use.                                                               |
-| `aws_session_token` / `session_token` | The AWS session token to use.                                                                     |
-| `aws_endpoint` / `endpoint`         | The endpoint to use for S3-compatible stores.                                                       |
-| `aws_virtual_hosted_style_request` / `virtual_hosted_style_request` | Whether to use virtual hosted-style requests, where the bucket name is part of the endpoint. Meant to be used with `aws_endpoint`. Default: `False`. |
-| `aws_s3_express` / `s3_express`     | Whether to use S3 Express One Zone endpoints. Default: `False`. See more details below.             |
-| `aws_server_side_encryption`        | The server-side encryption algorithm to use. Must be one of `"AES256"`, `"aws:kms"`, or `"aws:kms:dsse"`. Default: `None`. |
-| `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
-| `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
-
-!!! tip "Automatic cleanup for failed writes"
-
-    LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
-
-    **[Configuring a bucket lifecycle configuration to delete incomplete multipart uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html)**
-
-#### AWS IAM Permissions
-
-If a bucket is private, then an IAM policy must be specified to allow access to it. For many development scenarios, using broad permissions such as a PowerUser account is more than sufficient for working with LanceDB. However, in many production scenarios, you may wish to have as narrow as possible permissions.
-
-For **read and write access**, LanceDB will need a policy such as:
-
-```json
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-              "s3:PutObject",
-              "s3:GetObject",
-              "s3:DeleteObject"
-            ],
-            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
-        },
-        {
-            "Effect": "Allow",
-            "Action": [
-                "s3:ListBucket",
-                "s3:GetBucketLocation"
-            ],
-            "Resource": "arn:aws:s3:::<bucket>",
-            "Condition": {
-                "StringLike": {
-                    "s3:prefix": [
-                        "<prefix>/*"
-                    ]
-                }
-            }
-        }
-    ]
-}
-```
-
-For **read-only access**, LanceDB will need a policy such as:
-
-```json
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-              "s3:GetObject"
-            ],
-            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
-        },
-        {
-            "Effect": "Allow",
-            "Action": [
-                "s3:ListBucket",
-                "s3:GetBucketLocation"
-            ],
-            "Resource": "arn:aws:s3:::<bucket>",
-            "Condition": {
-                "StringLike": {
-                    "s3:prefix": [
-                        "<prefix>/*"
-                    ]
-                }
-            }
-        }
-    ]
-}
-```
-
-#### DynamoDB Commit Store for concurrent writes
-
-By default, S3 does not support concurrent writes. Having two or more processes
-writing to the same table at the same time can lead to data corruption. This is
-because S3, unlike other object stores, does not have any atomic put or copy
-operation.
-
-To enable concurrent writes, you can configure LanceDB to use a DynamoDB table
-as a commit store. This table will be used to coordinate writes between
-different processes. To enable this feature, you must modify your connection
-URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the
-name of the table to use.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )    
-        ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-
-    const db = await lancedb.connect(
-        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-    );
-    ```
-
-The DynamoDB table must be created with the following schema:
-
-- Hash key: `base_uri` (string)
-- Range key: `version` (number)
-
-You can create this programmatically with:
-
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import boto3
-
-    dynamodb = boto3.client("dynamodb")
-    table = dynamodb.create_table(
-        TableName=table_name,
-        KeySchema=[
-            {"AttributeName": "base_uri", "KeyType": "HASH"},
-            {"AttributeName": "version", "KeyType": "RANGE"},
-        ],
-        AttributeDefinitions=[
-            {"AttributeName": "base_uri", "AttributeType": "S"},
-            {"AttributeName": "version", "AttributeType": "N"},
-        ],
-        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
-    )
-    ```
-
-=== "JavaScript"
-
-    <!-- skip-test -->
-    ```javascript
-    import {
-      CreateTableCommand,
-      DynamoDBClient,
-    } from "@aws-sdk/client-dynamodb";
-
-    const dynamodb = new DynamoDBClient({
-      region: CONFIG.awsRegion,
-      credentials: {
-        accessKeyId: CONFIG.awsAccessKeyId,
-        secretAccessKey: CONFIG.awsSecretAccessKey,
-      },
-      endpoint: CONFIG.awsEndpoint,
-    });
-    const command = new CreateTableCommand({
-      TableName: table_name,
-      AttributeDefinitions: [
-        {
-          AttributeName: "base_uri",
-          AttributeType: "S",
-        },
-        {
-          AttributeName: "version",
-          AttributeType: "N",
-        },
-      ],
-      KeySchema: [
-        { AttributeName: "base_uri", KeyType: "HASH" },
-        { AttributeName: "version", KeyType: "RANGE" },
-      ],
-      ProvisionedThroughput: {
-        ReadCapacityUnits: 1,
-        WriteCapacityUnits: 1,
-      },
-    });
-    await client.send(command);
-    ```
-
-
-#### S3-compatible stores
-
-LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )    
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    endpoint: "http://minio:9000",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    endpoint: "http://minio:9000",
-                }
-            }
-        );
-        ```
-
-This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
-
-!!! tip "Local servers"
-
-    For local development, the server often has a `http` endpoint rather than a
-    secure `https` endpoint. In this case, you must also set the `ALLOW_HTTP`
-    environment variable to `true` to allow non-TLS connections, or pass the
-    storage option `allow_http` as `true`. If you do not do this, you will get
-    an error like `URL scheme is not allowed`.
-
-#### S3 Express
-
-LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional infrastructure configuration for the compute service, such as EC2 or Lambda. Please refer to [Networking requirements for S3 Express One Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html).
-
-To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )    
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    s3Express: "true",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    s3Express: "true",
-                }
-            }
-        );
-        ```
-
-### Google Cloud Storage
-
-GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. Alternatively, you can pass the path to the JSON file in the `storage_options`:
-
-=== "Python"
-
-    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )    
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "gs://my-bucket/my-database",
-            {
-                storageOptions: {
-                    serviceAccount: "path/to/service-account.json",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "gs://my-bucket/my-database",
-            {
-                storageOptions: {
-                    serviceAccount: "path/to/service-account.json",
-                }
-            }
-        );
-        ```
-
-!!! info "HTTP/2 support"
-
-    By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
-
-The following keys can be used as both environment variables or keys in the `storage_options` parameter:
-<!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
-
-| Key                                   | Description                                  |
-|---------------------------------------|----------------------------------------------|
-| ``google_service_account`` / `service_account` | Path to the service account JSON file.       |
-| ``google_service_account_key``        | The serialized service account key.          |
-| ``google_application_credentials``    | Path to the application credentials.         |
-
-### Azure Blob Storage
-
-Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
-
-=== "Python"
-
-    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )   
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "az://my-container/my-database",
-            {
-                storageOptions: {
-                    accountName: "some-account",
-                    accountKey: "some-key",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "az://my-container/my-database",
-            {
-                storageOptions: {
-                    accountName: "some-account",
-                    accountKey: "some-key",
-                }
-            }
-        );
-        ```
-
-These keys can be used as both environment variables or keys in the `storage_options` parameter:
-
-<!-- source: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html -->
-
-| Key                                   | Description                                                                                      |
-|---------------------------------------|--------------------------------------------------------------------------------------------------|
-| ``azure_storage_account_name``        | The name of the azure storage account.                                                           |
-| ``azure_storage_account_key``         | The serialized service account key.                                                              |
-| ``azure_client_id``                   | Service principal client id for authorizing requests.                                            |
-| ``azure_client_secret``               | Service principal client secret for authorizing requests.                                        |
-| ``azure_tenant_id``                   | Tenant id used in oauth flows.                                                                   |
-| ``azure_storage_sas_key``             | Shared access signature. The signature is expected to be percent-encoded, much like they are provided in the azure storage explorer or azure portal. |
-| ``azure_storage_token``               | Bearer token.                                                                                    |
-| ``azure_storage_use_emulator``        | Use object store with azurite storage emulator.                                                  |
-| ``azure_endpoint``                    | Override the endpoint used to communicate with blob storage.                                      |
-| ``azure_use_fabric_endpoint``         | Use object store with url scheme account.dfs.fabric.microsoft.com.                               |
-| ``azure_msi_endpoint``                | Endpoint to request a imds managed identity token.                                               |
-| ``azure_object_id``                   | Object id for use with managed identity authentication.                                          |
-| ``azure_msi_resource_id``             | Msi resource id for use with managed identity authentication.                                    |
-| ``azure_federated_token_file``        | File containing token for Azure AD workload identity federation.                                 |
-| ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
-| ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
-
-<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tables/merge_insert.md

@@ -1,135 +0,0 @@
-The merge insert command is a flexible API that can be used to perform:
-
-1. Upsert
-2. Insert-if-not-exists
-3. Replace range
-
-It works by joining the input data with the target table on a key you provide.
-Often this key is a unique row id key. You can then specify what to do when
-there is a match and when there is not a match. For example, for upsert you want
-to update if the row has a match and insert if the row doesn't have a match.
-Whereas for insert-if-not-exists you only want to insert if the row doesn't have
-a match.
-
-You can also read more in the API reference:
-
-* Python
-    * Sync: [lancedb.table.Table.merge_insert][]
-    * Async: [lancedb.table.AsyncTable.merge_insert][]
-* Typescript: [lancedb.Table.mergeInsert](../../js/classes/Table.md/#mergeinsert)
-
-!!! tip "Use scalar indices to speed up merge insert"
-
-    The merge insert command needs to perform a join between the input data and the
-    target table on the `on` key you provide. This requires scanning that entire
-    column, which can be expensive for large tables. To speed up this operation,
-    you can create a scalar index on the `on` column, which will allow LanceDB to
-    find matches without having to scan the whole tables.
-
-    Read more about scalar indices in [Building a Scalar Index](../scalar_index.md)
-    guide.
-
-!!! info "Embedding Functions"
-
-    Like the create table and add APIs, the merge insert API will automatically
-    compute embeddings if the table has a embedding definition in its schema.
-    If the input data doesn't contain the source column, or the vector column
-    is already filled, then the embeddings won't be computed. See the
-    [Embedding Functions](../../embeddings/embedding_functions.md) guide for more
-    information.
-
-## Upsert
-
-Upsert updates rows if they exist and inserts them if they don't. To do this
-with merge insert, enable both `when_matched_update_all()` and
-`when_not_matched_insert_all()`.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:upsert_basic"
-        ```
-
-!!! note "Providing subsets of columns"
-
-    If a column is nullable, it can be omitted from input data and it will be
-    considered `null`. Columns can also be provided in any order.
-
-## Insert-if-not-exists
-
-To avoid inserting duplicate rows, you can use the insert-if-not-exists command.
-This will only insert rows that do not have a match in the target table. To do
-this with merge insert, enable just `when_not_matched_insert_all()`.
-
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:insert_if_not_exists"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:insert_if_not_exists_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:insert_if_not_exists"
-        ```
-
-
-## Replace range
-
-You can also replace a range of rows in the target table with the input data.
-For example, if you have a table of document chunks, where each chunk has
-both a `doc_id` and a `chunk_id`, you can replace all chunks for a given
-`doc_id` with updated chunks. This can be tricky otherwise because if you
-try to use upsert when the new data has fewer chunks you will end up with
-extra chunks. To avoid this, add another clause to delete any chunks for
-the document that are not in the new data, with
-`when_not_matched_by_source_delete`.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:replace_range"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:replace_range_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:replace_range"
-        ```

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,131 +0,0 @@
-## Improving retriever performance
-
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-
-VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retrievers are a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
-
-There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
-
-* Using different query types
-* Using hybrid search
-* Fine-tuning the embedding models
-* Using different embedding models
-
-Using different embedding models is something that's very specific to the use case and the data. So we will not discuss it here. In this section, we will discuss the first three techniques.
-
-
-!!! note "Note"
-    We'll be using a simple metric called "hit-rate" for evaluating the performance of the retriever across this guide. Hit-rate is the percentage of queries for which the retriever returned the correct answer in the top-k results. For example, if the retriever returned the correct answer in the top-3 results for 70% of the queries, then the hit-rate@3 is 0.7.
-
-
-## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv).
-
-### Using different query types
-Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
-
-```python
-import lancedb
-import pandas as pd
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import Vector, LanceModel
-
-db = lancedb.connect("~/lancedb/query_types")
-df = pd.read_csv("data_qa.csv")
-
-embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.")
-
-class Schema(LanceModel):
-    context: str = embed_fcn.SourceField()
-    vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField()
-
-table = db.create_table("qa", schema=Schema)
-table.add(df[["context"]].to_dict(orient="records"))
-
-queries = df["query"].tolist()
-```
-
-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset:
-
-* <b> Vector Search: </b>
-
-    ```python
-    table.search(quries[0], query_type="vector").limit(5).to_pandas()
-    ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement:
-
-    ```python
-    table.search(quries[0]).limit(5).to_pandas()
-    ```
-
-    Vector or semantic search is useful when you want to find documents that are similar to the query in terms of meaning.
-
----
-
-* <b> Full-text Search: </b>
-    
-    FTS requires creating an index on the column you want to search on. `replace=True` will replace the existing index if it exists.
-    Once the index is created, you can search using the `fts` query type.
-    ```python
-    table.create_fts_index("context", replace=True)
-    table.search(quries[0], query_type="fts").limit(5).to_pandas()
-    ```
-
-    Full-text search is useful when you want to find documents that contain the query terms.
-
----
-
-* <b> Hybrid Search: </b>
-
-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset:
-    ```python
-    table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
-    ```
-    Hybrid search requires a reranker to combine and rank the results from vector and full-text search. We'll cover reranking as a concept in the next section.
-
-    Hybrid search is useful when you want to combine the benefits of both vector and full-text search.
-
-    !!! note "Note"
-        By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/).
-
-    
-
-### Hit rate evaluation results
-
-Now that we have seen how to run different query types on the dataset, let's evaluate the hit-rate of each query type on the dataset.
-For brevity, the entire evaluation script is not shown here. You can find the complete evaluation and benchmarking utility scripts [here](https://github.com/lancedb/ragged).
-
-Here are the hit-rate results for the dataset:
-
-| Query Type | Hit-rate@5 |
-| --- | --- |
-| Vector Search | 0.640 |
-| Full-text Search | 0.595 |
-| Hybrid Search (w/ LinearCombinationReranker) | 0.645 |
-
-**Choosing query type** is very specific to the use case and the data. This synthetic dataset has been generated to be semantically challenging, i.e, the queries don't have a lot of keywords in common with the context. So, vector search performs better than full-text search. However, in real-world scenarios, full-text search might perform better than vector search. Hybrid search is a good choice when you want to combine the benefits of both vector and full-text search.
-
-### Evaluation results on other datasets
-
-The hit-rate results can vary based on the dataset and the query type. Here are the hit-rate results for the other datasets using the same embedding function.
-
-* <b> SQuAD Dataset: </b>
-
-    | Query Type | Hit-rate@5 |
-    | --- | --- |
-    | Vector Search | 0.822 |
-    | Full-text Search | 0.835 |
-    | Hybrid Search (w/ LinearCombinationReranker) | 0.8874 |
-
-* <b> Uber10K sec filing Dataset: </b>
-
-    | Query Type | Hit-rate@5 |
-    | --- | --- |
-    | Vector Search | 0.608 |
-    | Full-text Search | 0.82 |
-    | Hybrid Search (w/ LinearCombinationReranker) | 0.80 |
-
-In these standard datasets, FTS seems to perform much better than vector search because the queries have a lot of keywords in common with the context. So, in general choosing the query type is very specific to the use case and the data.
-
-