Bump version: 0.9.0-beta.6 → 0.9.0-beta.7

# Conflicts:
#	python/pyproject.toml

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.0-beta.0"
+current_version = "0.5.2-final.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -24,102 +24,34 @@ commit = true
 message = "Bump version: {current_version} → {new_version}"
 commit_args = ""
 
-# Java maven files
-pre_commit_hooks = [
-  """
-    NEW_VERSION="${BVHOOK_NEW_MAJOR}.${BVHOOK_NEW_MINOR}.${BVHOOK_NEW_PATCH}"
-    if [ ! -z "$BVHOOK_NEW_PRE_L" ] && [ ! -z "$BVHOOK_NEW_PRE_N" ]; then
-        NEW_VERSION="${NEW_VERSION}-${BVHOOK_NEW_PRE_L}.${BVHOOK_NEW_PRE_N}"
-    fi
-    echo "Constructed new version: $NEW_VERSION"
-    cd java && mvn versions:set -DnewVersion=$NEW_VERSION && mvn versions:commit
-
-    # Check for any modified but unstaged pom.xml files
-    MODIFIED_POMS=$(git ls-files -m | grep pom.xml)
-    if [ ! -z "$MODIFIED_POMS" ]; then
-        echo "The following pom.xml files were modified but not staged. Adding them now:"
-        echo "$MODIFIED_POMS" | while read -r file; do
-            git add "$file"
-            echo "Added: $file"
-        done
-    fi
-    """,
-]
-
 [tool.bumpversion.parts.pre_l]
-optional_value = "final"
 values = ["beta", "final"]
+optional_value = "final"
 
 [[tool.bumpversion.files]]
 filename = "node/package.json"
-replace = "\"version\": \"{new_version}\","
 search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
 
 [[tool.bumpversion.files]]
 filename = "nodejs/package.json"
-replace = "\"version\": \"{new_version}\","
 search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
 
 # nodejs binary packages
 [[tool.bumpversion.files]]
 glob = "nodejs/npm/*/package.json"
-replace = "\"version\": \"{new_version}\","
 search = "\"version\": \"{current_version}\","
-
-# vectodb node binary packages
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-darwin-arm64\": \"{new_version}\""
-search = "\"@lancedb/vectordb-darwin-arm64\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-darwin-x64\": \"{new_version}\""
-search = "\"@lancedb/vectordb-darwin-x64\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-linux-arm64-gnu\": \"{new_version}\""
-search = "\"@lancedb/vectordb-linux-arm64-gnu\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\""
-search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-linux-arm64-musl\": \"{new_version}\""
-search = "\"@lancedb/vectordb-linux-arm64-musl\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-linux-x64-musl\": \"{new_version}\""
-search = "\"@lancedb/vectordb-linux-x64-musl\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""
-search = "\"@lancedb/vectordb-win32-x64-msvc\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-win32-arm64-msvc\": \"{new_version}\""
-search = "\"@lancedb/vectordb-win32-arm64-msvc\": \"{current_version}\""
+replace = "\"version\": \"{new_version}\","
 
 # Cargo files
 # ------------
 [[tool.bumpversion.files]]
 filename = "rust/ffi/node/Cargo.toml"
-replace = "\nversion = \"{new_version}\""
 search = "\nversion = \"{current_version}\""
+replace = "\nversion = \"{new_version}\""
 
 [[tool.bumpversion.files]]
 filename = "rust/lancedb/Cargo.toml"
-replace = "\nversion = \"{new_version}\""
 search = "\nversion = \"{current_version}\""
-
-[[tool.bumpversion.files]]
-filename = "nodejs/Cargo.toml"
 replace = "\nversion = \"{new_version}\""
-search = "\nversion = \"{current_version}\""

.cargo/config.toml

@@ -31,9 +31,6 @@ rustflags = [
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 
-[target.x86_64-unknown-linux-musl]
-rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
-
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
 
@@ -41,7 +38,3 @@ rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm
 # not found errors on systems that are missing it.
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-Ctarget-feature=+crt-static"]
-
-# Experimental target for Arm64 Windows
-[target.aarch64-pc-windows-msvc]
-rustflags = ["-Ctarget-feature=+crt-static"]

.github/workflows/build_mac_wheel/action.yml

@@ -20,7 +20,7 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
-        # TODO: pass through interpreter
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/docs.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Install dependecies needed for ubuntu
         run: |
           sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default
+          rustup update && rustup default        
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -41,8 +41,8 @@ jobs:
       - name: Build Python
         working-directory: python
         run: |
-          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .
-          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
+          python -m pip install -e .
+          python -m pip install -r ../docs/requirements.txt
       - name: Set up node
         uses: actions/setup-node@v3
         with:
@@ -72,9 +72,9 @@ jobs:
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v1
         with:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -24,19 +24,15 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: ubuntu-24.04
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
     steps:
     - name: Checkout
       uses: actions/checkout@v4
     - name: Print CPU capabilities
       run: cat /proc/cpuinfo
-    - name: Install protobuf
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler
     - name: Install dependecies needed for ubuntu
       run: |
-        sudo apt install -y libssl-dev
+        sudo apt install -y protobuf-compiler libssl-dev
         rustup update && rustup default
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -49,7 +45,7 @@ jobs:
     - name: Build Python
       working-directory: docs/test
       run:
-        python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r requirements.txt
+        python -m pip install -r requirements.txt
     - name: Create test files
       run: |
         cd docs/test
@@ -60,7 +56,7 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: ubuntu-24.04
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
     timeout-minutes: 60
     strategy:
       fail-fast: false
@@ -76,13 +72,9 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: 20
-    - name: Install protobuf
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler
     - name: Install dependecies needed for ubuntu
       run: |
-        sudo apt install -y libssl-dev
+        sudo apt install -y protobuf-compiler libssl-dev
         rustup update && rustup default
     - name: Rust cache
       uses: swatinem/rust-cache@v2

.github/workflows/java-publish.yml

@@ -1,114 +0,0 @@
-name: Build and publish Java packages
-on:
-  release:
-    types: [released]
-  pull_request:
-    paths:
-      - .github/workflows/java-publish.yml
-
-jobs:
-  macos-arm64:
-    name: Build on MacOS Arm64
-    runs-on: macos-14
-    timeout-minutes: 45
-    defaults:
-      run:
-        working-directory: ./java/core/lancedb-jni
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-      - name: Install dependencies
-        run: |
-          brew install protobuf
-      - name: Build release
-        run: |
-          cargo build --release
-      - uses: actions/upload-artifact@v4
-        with:
-          name: liblancedb_jni_darwin_aarch64.zip
-          path: target/release/liblancedb_jni.dylib
-          retention-days: 1
-          if-no-files-found: error
-  linux-arm64:
-    name: Build on Linux Arm64
-    runs-on: warp-ubuntu-2204-arm64-8x
-    timeout-minutes: 45
-    defaults:
-      run:
-        working-directory: ./java/core/lancedb-jni
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: "1.79.0"
-          cache-workspaces: "./java/core/lancedb-jni"
-          # Disable full debug symbol generation to speed up CI build and keep memory down
-          # "1" means line tables only, which is useful for panic tracebacks.
-          rustflags: "-C debuginfo=1"
-      - name: Install dependencies
-        run: |
-          sudo apt -y -qq update
-          sudo apt install -y protobuf-compiler libssl-dev pkg-config
-      - name: Build release
-        run: |
-          cargo build --release
-      - uses: actions/upload-artifact@v4
-        with:
-          name: liblancedb_jni_linux_aarch64.zip
-          path: target/release/liblancedb_jni.so
-          retention-days: 1
-          if-no-files-found: error
-  linux-x86:
-    runs-on: warp-ubuntu-2204-x64-8x
-    timeout-minutes: 30
-    needs: [macos-arm64, linux-arm64]
-    defaults:
-      run:
-        working-directory: ./java
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-      - name: Set up Java 8
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 8
-          cache: "maven"
-          server-id: ossrh
-          server-username: SONATYPE_USER
-          server-password: SONATYPE_TOKEN
-          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
-          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
-      - name: Install dependencies
-        run: |
-          sudo apt -y -qq update
-          sudo apt install -y protobuf-compiler libssl-dev pkg-config
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-      - name: Copy native libs
-        run: |
-          mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64
-          cp ../liblancedb_jni_darwin_aarch64.zip/liblancedb_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblancedb_jni.dylib
-          cp ../liblancedb_jni_linux_aarch64.zip/liblancedb_jni.so ./core/target/classes/nativelib/linux-aarch64/liblancedb_jni.so
-      - name: Dry run
-        if: github.event_name == 'pull_request'
-        run: |
-          mvn --batch-mode -DskipTests package
-      - name: Set github
-        run: |
-          git config --global user.email "LanceDB Github Runner"
-          git config --global user.name "dev+gha@lancedb.com"
-      - name: Publish with Java 8
-        if: github.event_name == 'release'
-        run: |
-          echo "use-agent" >> ~/.gnupg/gpg.conf
-          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
-        env:
-          SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
-          SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}

.github/workflows/java.yml

@@ -3,8 +3,6 @@ on:
   push:
     branches:
       - main
-    paths:
-      - java/**
   pull_request:
     paths:
       - java/**
@@ -23,42 +21,9 @@ env:
   CARGO_INCREMENTAL: "0"
   CARGO_BUILD_JOBS: "1"
 jobs:
-  linux-build-java-11:
+  linux-build:
     runs-on: ubuntu-22.04
-    name: ubuntu-22.04 + Java 11
-    defaults:
-      run:
-        working-directory: ./java
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: java/core/lancedb-jni
-      - name: Run cargo fmt
-        run: cargo fmt --check
-        working-directory: ./java/core/lancedb-jni
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install Java 11
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 11
-          cache: "maven"
-      - name: Java Style Check
-        run: mvn checkstyle:check
-      # Disable because of issues in lancedb rust core code
-      # - name: Rust Clippy
-      #   working-directory: java/core/lancedb-jni
-      #   run: cargo clippy --all-targets -- -D warnings
-      - name: Running tests with Java 11
-        run: mvn clean test
-  linux-build-java-17:
-    runs-on: ubuntu-22.04
-    name: ubuntu-22.04 + Java 17
+    name: ubuntu-22.04 + Java 11 & 17
     defaults:
       run:
         working-directory: ./java
@@ -82,12 +47,20 @@ jobs:
           java-version: 17
           cache: "maven"
       - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
+      - name: Install Java 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+          cache: "maven"
       - name: Java Style Check
         run: mvn checkstyle:check
       # Disable because of issues in lancedb rust core code
       # - name: Rust Clippy
       #   working-directory: java/core/lancedb-jni
       #   run: cargo clippy --all-targets -- -D warnings
+      - name: Running tests with Java 11
+        run: mvn clean test
       - name: Running tests with Java 17
         run: |
           export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
@@ -110,4 +83,3 @@ jobs:
           -Djdk.reflect.useDirectMethodHandle=false \
           -Dio.netty.tryReflectionSetAccessible=true"
           JAVA_HOME=$JAVA_17 mvn clean test
-  

.github/workflows/make-release-commit.yml

@@ -30,7 +30,7 @@ on:
         default: true
         type: boolean
       other:
-        description: 'Make a Node/Rust/Java release'
+        description: 'Make a Node/Rust release'
         required: true
         default: true
         type: boolean
@@ -43,7 +43,7 @@ on:
 jobs:
   make-release:
     # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
@@ -57,14 +57,15 @@ jobs:
           # trigger any workflows watching for new tags. See:
           # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
           token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-      - name: Validate Lance dependency is at stable version
-        if: ${{ inputs.type == 'stable' }}
-        run: python ci/validate_stable_lance.py
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
       - name: Bump Python version
         if: ${{ inputs.python }}
         working-directory: python
@@ -96,7 +97,3 @@ jobs:
         if: ${{ !inputs.dry_run && inputs.other }}
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        if: ${{ !inputs.dry_run && inputs.other }}
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nodejs.yml

@@ -53,9 +53,6 @@ jobs:
         cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint-ci
-    - name: Lint examples
-      working-directory: nodejs/examples
-      run: npm ci && npm run lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -94,18 +91,6 @@ jobs:
       env:
         S3_TEST: "1"
       run: npm run test
-    - name: Setup examples
-      working-directory: nodejs/examples
-      run: npm ci
-    - name: Test examples
-      working-directory: ./
-      env:
-        OPENAI_API_KEY: test
-        OPENAI_BASE_URL: http://0.0.0.0:8000
-      run: |
-        python ci/mock_openai.py &
-        cd nodejs/examples
-        npm test
   macos:
     timeout-minutes: 30
     runs-on: "macos-14"

.github/workflows/npm-publish.yml

@@ -7,7 +7,6 @@ on:
 
 jobs:
   node:
-    name: vectordb Typescript
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -40,7 +39,6 @@ jobs:
             node/vectordb-*.tgz
 
   node-macos:
-    name: vectordb ${{ matrix.config.arch }}
     strategy:
       matrix:
         config:
@@ -71,7 +69,6 @@ jobs:
             node/dist/lancedb-vectordb-darwin*.tgz
 
   nodejs-macos:
-    name: lancedb ${{ matrix.config.arch }}
     strategy:
       matrix:
         config:
@@ -101,8 +98,8 @@ jobs:
           path: |
             nodejs/dist/*.node
 
-  node-linux-gnu:
-    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
+  node-linux:
+    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -133,68 +130,16 @@ jobs:
           free -h
       - name: Build Linux Artifacts
         run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: node-native-linux-${{ matrix.config.arch }}-gnu
+          name: node-native-linux-${{ matrix.config.arch }}
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
-  node-linux-musl:
-    name: vectordb (${{ matrix.config.arch}}-unknown-linux-musl)
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install common dependencies
-        run: |
-          apk add protobuf-dev curl clang mold grep npm bash
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          source "$HOME/.cargo/env"
-          rustup target add aarch64-unknown-linux-musl
-          crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
-          sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
-          apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
-          curl -sSf $apk_url > apk_list
-          for pkg in gcc libgcc musl; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done
-          mkdir -p $sysroot_lib
-          echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so
-          cp usr/lib/libgcc_s.so.1 $sysroot_lib
-          cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib
-          cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so
-          echo '!<arch>' > $sysroot_lib/libdl.a
-          (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib)
-          echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env
-          echo "export RUSTFLAGS='-Ctarget-cpu=apple-m1 -Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env
-      - name: Build Linux Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-musl
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-linux-${{ matrix.config.arch }}-musl
-          path: |
-            node/dist/lancedb-vectordb-linux*.tgz
-
-  nodejs-linux-gnu:
-    name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
+  nodejs-linux:
+    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -230,7 +175,7 @@ jobs:
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}-gnu
+          name: nodejs-native-linux-${{ matrix.config.arch }}
           path: |
             nodejs/dist/*.node
       # The generic files are the same in all distros so we just pick
@@ -244,64 +189,7 @@ jobs:
             nodejs/dist/*
             !nodejs/dist/*.node
 
-  nodejs-linux-musl:
-    name: lancedb (${{ matrix.config.arch}}-unknown-linux-musl
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install common dependencies
-        run: |
-          apk add protobuf-dev curl clang mold grep npm bash openssl-dev openssl-libs-static
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
-          echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=/usr/include" >> saved_env
-          echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=/usr/lib" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          source "$HOME/.cargo/env"
-          rustup target add aarch64-unknown-linux-musl
-          crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
-          sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
-          apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
-          curl -sSf $apk_url > apk_list
-          for pkg in gcc libgcc musl openssl-dev openssl-libs-static; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done
-          mkdir -p $sysroot_lib
-          echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so
-          cp usr/lib/libgcc_s.so.1 $sysroot_lib
-          cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib
-          cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so
-          echo '!<arch>' > $sysroot_lib/libdl.a
-          (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib)
-          echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env
-          echo "export RUSTFLAGS='-Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env
-          echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=$(realpath usr/include)" >> saved_env
-          echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=$(realpath usr/lib)" >> saved_env
-      - name: Build Linux Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}-musl
-          path: |
-            nodejs/dist/*.node
-
   node-windows:
-    name: vectordb ${{ matrix.target }}
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -334,53 +222,7 @@ jobs:
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
-  node-windows-arm64:
-    name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
-    # if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          # - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export AR=llvm-ar" >> saved_env
-          source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
-          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
-          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
-          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
-      - name: Configure x86_64 build
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
-      - name: Build Windows Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-pc-windows-msvc
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-windows-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-win32*.tgz
-
   nodejs-windows:
-    name: lancedb ${{ matrix.target }}
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -413,57 +255,8 @@ jobs:
           path: |
             nodejs/dist/*.node
 
-  nodejs-windows-arm64:
-    name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
-    # Only runs on tags that matches the make-release action
-    # if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          # - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export AR=llvm-ar" >> saved_env
-          source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
-          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
-          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
-          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
-          printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin
-          chmod u+x $HOME/.cargo/bin/cargo-xwin
-      - name: Configure x86_64 build
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
-      - name: Build Windows Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-
   release:
-    name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows, node-windows-arm64]
+    needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -481,7 +274,7 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
-          # Tag beta as "preview" instead of default "latest". See lancedb
+          # Tag beta as "preview" instead of default "latest". See lancedb 
           # npm publish step for more info.
           if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
             PUBLISH_ARGS="--tag preview"
@@ -491,19 +284,9 @@ jobs:
           for filename in *.tgz; do
             npm publish $PUBLISH_ARGS $filename
           done
-      - name: Notify Slack Action
-        uses: ravsamhq/notify-slack-action@2.3.0
-        if: ${{ always() }}
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-          notification_title: "{workflow} is failing"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   release-nodejs:
-    name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows, nodejs-windows-arm64]
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -550,18 +333,8 @@ jobs:
           else
             npm publish --access public
           fi
-      - name: Notify Slack Action
-        uses: ravsamhq/notify-slack-action@2.3.0
-        if: ${{ always() }}
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-          notification_title: "{workflow} is failing"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   update-package-lock:
-    if: startsWith(github.ref, 'refs/tags/v')
     needs: [release]
     runs-on: ubuntu-latest
     permissions:
@@ -571,7 +344,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: main
-          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          persist-credentials: false
           fetch-depth: 0
           lfs: true
       - uses: ./.github/workflows/update_package_lock
@@ -579,7 +352,6 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   update-package-lock-nodejs:
-    if: startsWith(github.ref, 'refs/tags/v')
     needs: [release-nodejs]
     runs-on: ubuntu-latest
     permissions:
@@ -589,7 +361,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: main
-          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          persist-credentials: false
           fetch-depth: 0
           lfs: true
       - uses: ./.github/workflows/update_package_lock_nodejs
@@ -597,7 +369,6 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   gh-release:
-    if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/pypi-publish.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.12
+          python-version: 3.8
       - uses: ./.github/workflows/build_windows_wheel
         with:
           python-minor-version: 8

.github/workflows/python.yml

@@ -30,14 +30,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: "3.11"
       - name: Install ruff
         run: |
-          pip install ruff==0.8.4
+          pip install ruff==0.2.2
       - name: Format check
         run: ruff format --check .
       - name: Lint
-        run: ruff check .
+        run: ruff .
   doctest:
     name: "Doctest"
     timeout-minutes: 30
@@ -138,7 +138,7 @@ jobs:
         run: rm -rf target/wheels
   windows:
     name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 60
+    timeout-minutes: 30
     strategy:
       matrix:
         config:

.github/workflows/rust.yml

@@ -26,78 +26,68 @@ env:
 jobs:
   lint:
     timeout-minutes: 30
-    runs-on: ubuntu-24.04
-    defaults:
-      run:
-        shell: bash
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Run format
-        run: cargo fmt --all -- --check
-      - name: Run clippy
-        run: cargo clippy --workspace --tests --all-features -- -D warnings
-
-  linux:
-    timeout-minutes: 30
-    # To build all features, we need more disk space than is available
-    # on the free OSS github runner. This is mostly due to the the
-    # sentence-transformers feature.
-    runs-on: ubuntu-2404-4x-x64
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
         working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
+      CC: gcc-12
+      CXX: g++-12
     steps:
-      - uses: actions/checkout@v4
-        with:
+    - uses: actions/checkout@v4
+      with:
           fetch-depth: 0
           lfs: true
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install dependencies
-        run: |
+    - uses: Swatinem/rust-cache@v2
+      with:
+        workspaces: rust
+    - name: Install dependencies
+      run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - name: Make Swap
-        run: |
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-      - name: Start S3 integration test environment
-        working-directory: .
-        run: docker compose up --detach --wait
-      - name: Build
-        run: cargo build --all-features
-      - name: Run tests
-        run: cargo test --all-features
-      - name: Run examples
-        run: cargo run --example simple
-
+    - name: Run format
+      run: cargo fmt --all -- --check
+    - name: Run clippy
+      run: cargo clippy --all --all-features -- -D warnings
+  linux:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Start S3 integration test environment
+      working-directory: .
+      run: docker compose up --detach --wait
+    - name: Build
+      run: cargo build --all-features
+    - name: Run tests
+      run: cargo test --all-features
+    - name: Run examples
+      run: cargo run --example simple
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: ["macos-13", "macos-14"]
+        mac-runner: [ "macos-13", "macos-14" ]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
@@ -106,8 +96,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0
-          lfs: true
+            fetch-depth: 0
+            lfs: true
       - name: CPU features
         run: sysctl -a | grep cpu
       - uses: Swatinem/rust-cache@v2
@@ -120,7 +110,6 @@ jobs:
       - name: Run tests
         # Run with everything except the integration tests.
         run: cargo test --features remote,fp16kernels
-
   windows:
     runs-on: windows-2022
     steps:
@@ -142,137 +131,4 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
-
-  windows-arm64:
-    runs-on: windows-4x-arm
-    steps:
-      - name: Install Git
-        run: |
-          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-        shell: powershell
-      - name: Add Git to PATH
-        run: |
-          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-        shell: powershell
-      - name: Configure Git symlinks
-        run: git config --global core.symlinks true
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-      - name: Install Visual Studio Build Tools
-        run: |
-          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-            "--installPath", "C:\BuildTools", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-        shell: powershell
-      - name: Add Visual Studio Build Tools to PATH
-        run: |
-          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
-
-          # Add MSVC runtime libraries to LIB
-          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
-                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
-                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
-
-          # Add INCLUDE paths
-          $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
-          Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
-        shell: powershell
-      - name: Install Rust
-        run: |
-          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-        shell: powershell
-      - name: Add Rust to PATH
-        run: |
-          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-        shell: powershell
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install 7-Zip ARM
-        run: |
-          New-Item -Path 'C:\7zip' -ItemType Directory
-          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-        shell: powershell
-      - name: Add 7-Zip to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-        shell: powershell
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          if (Test-Path 'C:\protoc') {
-            Write-Host "Protoc directory exists, skipping installation"
-            return
-          }
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-        shell: powershell
-      - name: Add Protoc to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Run tests
-        run: |
-          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build --target aarch64-pc-windows-msvc
-          cargo test --target aarch64-pc-windows-msvc
-
-  msrv:
-    # Check the minimum supported Rust version
-    name: MSRV Check - Rust v${{ matrix.msrv }}
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install ${{ matrix.msrv }}
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ matrix.msrv }}
-      - name: Downgrade  dependencies
-        # These packages have newer requirements for MSRV
-        run: |
-          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
-          cargo update -p aws-sdk-dynamodb --precise 1.55.0
-          cargo update -p aws-config --precise 1.5.10
-          cargo update -p aws-sdk-kms --precise 1.51.0
-          cargo update -p aws-sdk-s3 --precise 1.65.0
-          cargo update -p aws-sdk-sso --precise 1.50.0
-          cargo update -p aws-sdk-ssooidc --precise 1.51.0
-          cargo update -p aws-sdk-sts --precise 1.51.0
-          cargo update -p home --precise 0.5.9
-      - name: cargo +${{ matrix.msrv }} check
-        run: cargo check --workspace --tests --benches --all-features
+  

.github/workflows/upload_wheel/action.yml

@@ -17,12 +17,11 @@ runs:
     run: |
       python -m pip install --upgrade pip
       pip install twine
-      python3 -m pip install --upgrade pkginfo
   - name: Choose repo
     shell: bash
     id: choose_repo
     run: |
-      if [[ ${{ github.ref }} == *beta* ]]; then
+      if [ ${{ github.ref }} == "*beta*" ]; then
         echo "repo=fury" >> $GITHUB_OUTPUT
       else
         echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -33,7 +32,7 @@ runs:
       FURY_TOKEN: ${{ inputs.fury_token }}
       PYPI_TOKEN: ${{ inputs.pypi_token }}
     run: |
-      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
+      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
         WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
         echo "Uploading $WHEEL to Fury"
         curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/

.gitignore

@@ -4,7 +4,6 @@
 **/__pycache__
 .DS_Store
 venv
-.venv
 
 .vscode
 .zed

.pre-commit-config.yaml

@@ -14,8 +14,8 @@ repos:
   hooks:
     - id: local-biome-check
       name: biome check
-      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+      entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/
       language: system
       types: [text]
       files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*

CONTRIBUTING.md

@@ -1,78 +0,0 @@
-# Contributing to LanceDB
-
-LanceDB is an open-source project and we welcome contributions from the community.
-This document outlines the process for contributing to LanceDB.
-
-## Reporting Issues
-
-If you encounter a bug or have a feature request, please open an issue on the
-[GitHub issue tracker](https://github.com/lancedb/lancedb).
-
-## Picking an issue
-
-We track issues on the GitHub issue tracker. If you are looking for something to
-work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
-
-If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
-
-## Configuring Git
-
-First, fork the repository on GitHub, then clone your fork:
-
-```bash
-git clone https://github.com/<username>/lancedb.git
-cd lancedb
-```
-
-Then add the main repository as a remote:
-
-```bash
-git remote add upstream https://github.com/lancedb/lancedb.git
-git fetch upstream
-```
-
-## Setting up your development environment
-
-We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
-
-* [Python](python/CONTRIBUTING.md)
-* [Typescript](nodejs/CONTRIBUTING.md)
-<!-- TODO: add Java contributing guide -->
-* [Documentation](docs/README.md)
-
-
-## Best practices for pull requests
-
-For the best chance of having your pull request accepted, please follow these guidelines:
-
-1. Unit test all bug fixes and new features. Your code will not be merged if it
-   doesn't have tests.
-1. If you change the public API, update the documentation in the `docs` directory.
-1. Aim to minimize the number of changes in each pull request. Keep to solving
-   one problem at a time, when possible.
-1. Before marking a pull request ready-for-review, do a self review of your code.
-   Is it clear why you are making the changes? Are the changes easy to understand?
-1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
-    * New feature: `feat: adding foo API`
-    * Bug fix: `fix: issue with foo API`
-    * Documentation change: `docs: adding foo API documentation`
-1. If your pull request is a work in progress, leave the pull request as a draft.
-   We will assume the pull request is ready for review when it is opened.
-1. When writing tests, test the error cases. Make sure they have understandable
-   error messages.
-
-## Project structure
-
-The core library is written in Rust. The Python, Typescript, and Java libraries
-are wrappers around the Rust library.
-
-* `src/lancedb`: Rust library source code
-* `python`: Python package source code
-* `nodejs`: Typescript package source code
-* `node`: **Deprecated** Typescript package source code
-* `java`: Java package source code
-* `docs`: Documentation source code
-
-## Release process
-
-For information on the release process, see: [release_process.md](release_process.md)

Cargo.toml

@@ -18,42 +18,34 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.22.0", "features" = ["dynamodb"] }
-lance-io = "=0.22.0"
-lance-index = "=0.22.0"
-lance-linalg = "=0.22.0"
-lance-table = "=0.22.0"
-lance-testing = "=0.22.0"
-lance-datafusion = "=0.22.0"
-lance-encoding = "=0.22.0"
+lance = { "version" = "=0.13.0", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.13.0" }
+lance-linalg = { "version" = "=0.13.0" }
+lance-testing = { "version" = "=0.13.0" }
+lance-datafusion = { "version" = "=0.13.0" }
 # Note that this one does not include pyarrow
-arrow = { version = "53.2", optional = false }
-arrow-array = "53.2"
-arrow-data = "53.2"
-arrow-ipc = "53.2"
-arrow-ord = "53.2"
-arrow-schema = "53.2"
-arrow-arith = "53.2"
-arrow-cast = "53.2"
+arrow = { version = "51.0", optional = false }
+arrow-array = "51.0"
+arrow-data = "51.0"
+arrow-ipc = "51.0"
+arrow-ord = "51.0"
+arrow-schema = "51.0"
+arrow-arith = "51.0"
+arrow-cast = "51.0"
 async-trait = "0"
 chrono = "0.4.35"
-datafusion-common = "44.0"
-datafusion-physical-plan = "44.0"
-env_logger = "0.10"
+datafusion-physical-plan = "37.1"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
-moka = { version = "0.11", features = ["future"] }
-object_store = "0.10.2"
+object_store = "0.9.0"
 pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
-rand = "0.8"
 regex = "1.10"
 lazy_static = "1"

README.md

@@ -7,10 +7,9 @@
 
 <a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
 <a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/)
-[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd)
+[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/) 
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
 [![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
-[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20LanceDB%20Guru-006BFF?style=for-the-badge)](https://gurubase.io/g/lancedb)
 
 </p>
 
@@ -45,24 +44,26 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 
 **Javascript**
 ```shell
-npm install @lancedb/lancedb
+npm install vectordb
 ```
 
 ```javascript
-import * as lancedb from "@lancedb/lancedb";
+const lancedb = require('vectordb');
+const db = await lancedb.connect('data/sample-lancedb');
 
-const db = await lancedb.connect("data/sample-lancedb");
-const table = await db.createTable("vectors", [
-	{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-	{ id: 2, vector: [1.1, 1.2], item: "bar", price: 50 },
-], {mode: 'overwrite'});
+const table = await db.createTable({
+  name: 'vectors',
+  data:  [
+    { id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+    { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }
+  ]
+})
 
-
-const query = table.vectorSearch([0.1, 0.3]).limit(2);
-const results = await query.toArray();
+const query = table.search([0.1, 0.3]).limit(2);
+const results = await query.execute();
 
 // You can also search for rows by specific criteria without involving a vector search.
-const rowsByCriteria = await table.query().where("price >= 10").toArray();
+const rowsByCriteria = await table.search(undefined).where("price >= 10").execute();
 ```
 
 **Python**
@@ -83,4 +84,4 @@ result = table.search([100, 100]).limit(2).to_pandas()
 
 ## Blogs, Tutorials & Videos
 * 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/vectordb-recipes/tree/main/examples/Youtube-Search-QA-Bot">Build a question and answer bot with LanceDB</a>
+* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 set -e
 ARCH=${1:-x86_64}
-TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
-# We pass down the current user so that when we later mount the local files
+# We pass down the current user so that when we later mount the local files 
 # into the container, the files are accessible by the current user.
 pushd ci/manylinux_node
 docker build \
@@ -19,4 +18,4 @@ docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
     lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH $TARGET_TRIPLE
+    bash ci/manylinux_node/build.sh $ARCH

ci/build_linux_artifacts_nodejs.sh

@@ -4,9 +4,9 @@ ARCH=${1:-x86_64}
 
 # We pass down the current user so that when we later mount the local files
 # into the container, the files are accessible by the current user.
-pushd ci/manylinux_node
+pushd ci/manylinux_nodejs
 docker build \
-    -t lancedb-node-manylinux-$ARCH \
+    -t lancedb-nodejs-manylinux \
     --build-arg="ARCH=$ARCH" \
     --build-arg="DOCKER_USER=$(id -u)" \
     --progress=plain \
@@ -17,5 +17,5 @@ popd
 docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
-    lancedb-node-manylinux-$ARCH \
-    bash ci/manylinux_node/build_lancedb.sh $ARCH
+    lancedb-nodejs-manylinux \
+    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_windows_artifacts.ps1

@@ -3,7 +3,6 @@
 # Targets supported:
 # - x86_64-pc-windows-msvc
 # - i686-pc-windows-msvc
-# - aarch64-pc-windows-msvc
 
 function Prebuild-Rust {
     param (
@@ -32,7 +31,7 @@ function Build-NodeBinaries {
 
 $targets = $args[0]
 if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"
+    $targets = "x86_64-pc-windows-msvc"
 }
 
 Write-Host "Building artifacts for targets: $targets"

ci/build_windows_artifacts_nodejs.ps1

@@ -3,7 +3,6 @@
 # Targets supported:
 # - x86_64-pc-windows-msvc
 # - i686-pc-windows-msvc
-# - aarch64-pc-windows-msvc
 
 function Prebuild-Rust {
     param (
@@ -32,7 +31,7 @@ function Build-NodeBinaries {
 
 $targets = $args[0]
 if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"
+    $targets = "x86_64-pc-windows-msvc"
 }
 
 Write-Host "Building artifacts for targets: $targets"

ci/manylinux_node/Dockerfile

@@ -4,7 +4,7 @@
 # range of linux distributions.
 ARG ARCH=x86_64
 
-FROM quay.io/pypa/manylinux_2_28_${ARCH}
+FROM quay.io/pypa/manylinux2014_${ARCH}
 
 ARG ARCH=x86_64
 ARG DOCKER_USER=default_user
@@ -18,8 +18,8 @@ COPY install_protobuf.sh install_protobuf.sh
 RUN ./install_protobuf.sh ${ARCH}
 
 ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user, but only if it doesn't exist
-RUN echo ${ARCH} && id -u ${DOCKER_USER} >/dev/null 2>&1 || adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
 
 # We switch to the user to install Rust and Node, since those like to be 
 # installed at the user level.

ci/manylinux_node/build.sh

@@ -2,20 +2,18 @@
 # Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
 set -e
 ARCH=${1:-x86_64}
-TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
 if [ "$ARCH" = "x86_64" ]; then
     export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
+else 
     export OPENSSL_LIB_DIR=/usr/local/lib/
 fi
 export OPENSSL_STATIC=1
 export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
 
-#Alpine doesn't have .bashrc
-FILE=$HOME/.bashrc && test -f $FILE && source $FILE
+source $HOME/.bashrc
 
 cd node
 npm ci
 npm run build-release
-npm run pack-build -- -t $TARGET_TRIPLE
+npm run pack-build

ci/manylinux_node/install_openssl.sh

@@ -6,7 +6,7 @@
 # /usr/bin/ld: failed to set dynamic section sizes: Bad value
 set -e
 
-git clone -b OpenSSL_1_1_1v \
+git clone -b OpenSSL_1_1_1u \
     --single-branch \
     https://github.com/openssl/openssl.git
 

ci/manylinux_node/prepare_manylinux_node.sh

@@ -8,7 +8,7 @@ install_node() {
 
     source "$HOME"/.bashrc
 
-    nvm install --no-progress 18
+    nvm install --no-progress 16
 }
 
 install_rust() {

ci/manylinux_nodejs/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -11,8 +11,7 @@ fi
 export OPENSSL_STATIC=1
 export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
 
-#Alpine doesn't have .bashrc
-FILE=$HOME/.bashrc && test -f $FILE && source $FILE
+source $HOME/.bashrc
 
 cd nodejs
 npm ci

ci/manylinux_nodejs/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

ci/mock_openai.py

@@ -1,57 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
-import argparse
-import json
-import http.server
-
-
-class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
-    def do_POST(self):
-        content_length = int(self.headers["Content-Length"])
-        post_data = self.rfile.read(content_length)
-        post_data = json.loads(post_data.decode("utf-8"))
-        # See: https://platform.openai.com/docs/api-reference/embeddings/create
-
-        if isinstance(post_data["input"], str):
-            num_inputs = 1
-        else:
-            num_inputs = len(post_data["input"])
-
-        model = post_data.get("model", "text-embedding-ada-002")
-
-        data = []
-        for i in range(num_inputs):
-            data.append({
-                "object": "embedding",
-                "embedding": [0.1] * 1536,
-                "index": i,
-            })
-
-        response = {
-            "object": "list",
-            "data": data,
-            "model": model,
-            "usage": {
-                "prompt_tokens": 0,
-                "total_tokens": 0,
-            }
-        }
-
-        self.send_response(200)
-        self.send_header("Content-type", "application/json")
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode("utf-8"))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
-    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
-    args = parser.parse_args()
-    port = args.port
-
-    print(f"server started on port {port}. Press Ctrl-C to stop.")
-    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
-
-    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
-        server.serve_forever()

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -1,105 +0,0 @@
-#!/bin/sh
-
-# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
-# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
-
-# function dl() {
-# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
-# }
-
-# [[.h]]
-
-# "id": "Win11SDK_10.0.26100"
-# "version": "10.0.26100.7"
-
-# libucrt.lib
-
-# example: <assert.h>
-# dir: ucrt/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
-
-# example: <windows.h>
-# dir: um/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
-
-# example: <winapifamily.h>
-# dir: /shared
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
-
-
-# "id": "Microsoft.VisualC.14.16.CRT.Headers"
-# "version": "14.16.27045"
-
-# example: <vcruntime.h>
-# dir: MSVC/
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
-
-# [[.lib]]
-
-# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
-
-# dbghelp.lib fwpuclnt.lib arm64rt.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
-
-# libcmt.lib libvcruntime.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
-
-
-msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
-unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
-unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
-
-mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
-mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
-
-# lowercase folder/file names
-echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
-
-# .h
-(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
-
-cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
-
-# lowercase #include "" and #include <>
-find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
-
-# ARM intrinsics
-# original dir: MSVC/
-
-# '__n128x4' redefined in arm_neon.h
-# "arm64_neon.h" included from intrin.h
-
-(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
-
-# .lib
-
-# _Interlocked intrinsics
-# must always link with arm64rt.lib
-# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
-# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
-
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
-
-(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
-
-cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib

ci/sysroot-x86_64-pc-windows-msvc.sh

@@ -1,105 +0,0 @@
-#!/bin/sh
-
-# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
-# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
-
-# function dl() {
-# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
-# }
-
-# [[.h]]
-
-# "id": "Win11SDK_10.0.26100"
-# "version": "10.0.26100.7"
-
-# libucrt.lib
-
-# example: <assert.h>
-# dir: ucrt/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
-
-# example: <windows.h>
-# dir: um/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
-
-# example: <winapifamily.h>
-# dir: /shared
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
-
-
-# "id": "Microsoft.VisualC.14.16.CRT.Headers"
-# "version": "14.16.27045"
-
-# example: <vcruntime.h>
-# dir: MSVC/
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
-
-# [[.lib]]
-
-# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
-
-# dbghelp.lib fwpuclnt.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
-
-# libcmt.lib libvcruntime.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
-
-
-msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
-unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
-unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
-
-mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
-mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
-
-# lowercase folder/file names
-echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
-
-# .h
-(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
-
-cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
-
-# lowercase #include "" and #include <>
-find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
-
-# x86 intrinsics
-# original dir: MSVC/
-
-# '_mm_movemask_epi8' defined in emmintrin.h
-# '__v4sf' defined in xmmintrin.h
-# '__v2si' defined in mmintrin.h
-# '__m128d' redefined in immintrin.h
-# '__m128i' redefined in intrin.h
-# '_mm_comlt_epu8' defined in ammintrin.h
-
-(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
-
-# .lib
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
-
-(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
-
-cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib

ci/validate_stable_lance.py

@@ -1,34 +0,0 @@
-import tomllib
-
-found_preview_lance = False
-
-with open("Cargo.toml", "rb") as f:
-    cargo_data = tomllib.load(f)
-
-    for name, dep in cargo_data["workspace"]["dependencies"].items():
-        if name == "lance" or name.startswith("lance-"):
-            if isinstance(dep, str):
-                version = dep
-            elif isinstance(dep, dict):
-                # Version doesn't have the beta tag in it, so we instead look
-                # at the git tag.
-                version = dep.get('tag', dep.get('version'))
-            else:
-                raise ValueError("Unexpected type for dependency: " + str(dep))
-
-            if "beta" in version:
-                found_preview_lance = True
-                print(f"Dependency '{name}' is a preview version: {version}")
-
-with open("python/pyproject.toml", "rb") as f:
-    py_proj_data = tomllib.load(f)
-
-    for dep in py_proj_data["project"]["dependencies"]:
-        if dep.startswith("pylance"):
-            if "b" in dep:
-                found_preview_lance = True
-                print(f"Dependency '{dep}' is a preview version")
-            break  # Only one pylance dependency
-
-if found_preview_lance:
-    raise ValueError("Found preview version of Lance in dependencies")

docs/README.md

@@ -9,81 +9,36 @@ unreleased features.
 ## Building the docs
 
 ### Setup
-1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
-   Run `make develop` to install the Python package.
-2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed
 
-### Preview the docs
+### Building node module and create markdown files
 
-```shell
+See [Javascript docs README](./src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+### Run local server
+
+You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
+
+```bash
 cd docs
 mkdocs serve
 ```
 
-If you want to just generate the HTML files:
+### Run doctest for typescript example
 
-```shell
-PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
-```
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-## Adding examples
-
-To make sure examples are correct, we put examples in test files so they can be
-run as part of our test suites.
-
-You can see the tests are at:
-
-* Python: `python/python/tests/docs`
-* Typescript: `nodejs/examples/`
-
-### Checking python examples
-
-```shell
-cd python
-pytest -vv python/tests/docs
-```
-
-### Checking typescript examples
-
-The `@lancedb/lancedb` package must be built before running the tests:
-
-```shell
-pushd nodejs
-npm ci
+```bash
+cd lancedb/docs
+npm i
 npm run build
-popd
-```
-
-Then you can run the examples by going to the `nodejs/examples` directory and
-running the tests like a normal npm package:
-
-```shell
-pushd nodejs/examples
-npm ci
-npm test
-popd
-```
-
-## API documentation
-
-### Python
-
-The Python API documentation is organized based on the file `docs/src/python/python.md`.
-We manually add entries there so we can control the organization of the reference page.
-**However, this means any new types must be manually added to the file.** No additional
-steps are needed to generate the API documentation.
-
-### Typescript
-
-The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
-
-When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
-The new files should be checked into the repository.
-
-```shell
-pushd nodejs
-npm run docs
-popd
+npm run all
 ```

docs/mkdocs.yml

@@ -26,7 +26,6 @@ theme:
     - content.code.copy
     - content.tabs.link
     - content.action.edit
-    - content.tooltips
     - toc.follow
     - navigation.top
     - navigation.tabs
@@ -34,10 +33,8 @@ theme:
     - navigation.footer
     - navigation.tracking
     - navigation.instant
-    - content.footnote.tooltips
   icon:
     repo: fontawesome/brands/github
-    annotation: material/arrow-right-circle
   custom_dir: overrides
 
 plugins:
@@ -55,26 +52,15 @@ plugins:
             show_signature_annotations: true
             show_root_heading: true
             members_order: source
-            docstring_section_style: list
-            signature_crossrefs: true
-            separate_signature: true
           import:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
-            - https://lancedb.github.io/lance/objects.inv
   - mkdocs-jupyter
-  - render_swagger:
-      allow_arbitrary_locations: true
 
 markdown_extensions:
   - admonition
   - footnotes
-  - pymdownx.critic
-  - pymdownx.caret
-  - pymdownx.keys
-  - pymdownx.mark
-  - pymdownx.tilde
   - pymdownx.details
   - pymdownx.highlight:
       anchor_linenums: true
@@ -88,15 +74,7 @@ markdown_extensions:
   - pymdownx.tabbed:
       alternate_style: true
   - md_in_html
-  - abbr
   - attr_list
-  - pymdownx.snippets
-  - pymdownx.emoji:
-      emoji_index: !!python/name:material.extensions.emoji.twemoji
-      emoji_generator: !!python/name:material.extensions.emoji.to_svg
-  - markdown.extensions.toc:
-      baselevel: 1
-      permalink: ""
 
 nav:
   - Home:
@@ -104,79 +82,37 @@ nav:
       - 🏃🏼‍♂️ Quick start: basic.md
       - 📚 Concepts:
           - Vector search: concepts/vector_search.md
-          - Indexing:
-            - IVFPQ: concepts/index_ivfpq.md
-            - HNSW: concepts/index_hnsw.md
+          - Indexing: concepts/index_ivfpq.md
           - Storage: concepts/storage.md
           - Data management: concepts/data_management.md
       - 🔨 Guides:
           - Working with tables: guides/tables.md
-          - Building a vector index: ann_indexes.md
+          - Building an ANN index: ann_indexes.md
           - Vector Search: search.md
-          - Full-text search (native): fts.md
-          - Full-text search (tantivy-based): fts_tantivy.md
-          - Building a scalar index: guides/scalar_index.md
+          - Full-text search: fts.md
           - Hybrid search:
               - Overview: hybrid_search/hybrid_search.md
               - Comparing Rerankers: hybrid_search/eval.md
               - Airbnb financial data example: notebooks/hybrid_search.ipynb
-          - RAG:
-              - Vanilla RAG: rag/vanilla_rag.md
-              - Multi-head RAG: rag/multi_head_rag.md
-              - Corrective RAG: rag/corrective_rag.md
-              - Agentic RAG: rag/agentic_rag.md
-              - Graph RAG: rag/graph_rag.md
-              - Self RAG: rag/self_rag.md
-              - Adaptive RAG: rag/adaptive_rag.md
-              - SFR RAG: rag/sfr_rag.md
-              - Advanced Techniques:
-                - HyDE: rag/advanced_techniques/hyde.md
-                - FLARE: rag/advanced_techniques/flare.md
           - Reranking:
               - Quickstart: reranking/index.md
               - Cohere Reranker: reranking/cohere.md
               - Linear Combination Reranker: reranking/linear_combination.md
-              - Reciprocal Rank Fusion Reranker: reranking/rrf.md
               - Cross Encoder Reranker: reranking/cross_encoder.md
               - ColBERT Reranker: reranking/colbert.md
-              - Jina Reranker: reranking/jina.md
               - OpenAI Reranker: reranking/openai.md
-              - AnswerDotAi Rerankers: reranking/answerdotai.md
-              - Voyage AI Rerankers: reranking/voyageai.md
               - Building Custom Rerankers: reranking/custom_reranker.md
-              - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
-          - Versioning & Reproducibility: 
-              - sync API: notebooks/reproducibility.ipynb
-              - async API: notebooks/reproducibility_async.ipynb
+          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
           - Configuring Storage: guides/storage.md
-          - Migration Guide: migration.md
+          - Sync -> Async Migration Guide: migration.md
           - Tuning retrieval performance:
               - Choosing right query type: guides/tuning_retrievers/1_query_types.md
               - Reranking: guides/tuning_retrievers/2_reranking.md
-              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
       - 🧬 Managing embeddings:
-          - Understand Embeddings: embeddings/understanding_embeddings.md
-          - Get Started: embeddings/index.md
+          - Overview: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
-          - Available models:
-              - Overview: embeddings/default_embedding_functions.md
-              - Text Embedding Functions:
-                  - Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
-                  - Huggingface Embedding Models: embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md
-                  - Ollama Embeddings: embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md
-                  - OpenAI Embeddings: embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
-                  - Instructor Embeddings: embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md
-                  - Gemini Embeddings: embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md
-                  - Cohere Embeddings: embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md
-                  - Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
-                  - AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
-                  - IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
-                  - Voyage AI Embeddings: embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
-              - Multimodal Embedding Functions:
-                  - OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
-                  - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
-                  - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
+          - Available models: embeddings/default_embedding_functions.md
           - User-defined embedding functions: embeddings/custom_embedding_function.md
           - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
           - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
@@ -186,32 +122,24 @@ nav:
           - Polars: python/polars_arrow.md
           - DuckDB: python/duckdb.md
           - LangChain:
-              - LangChain 🔗: integrations/langchain.md
-              - LangChain demo: notebooks/langchain_demo.ipynb
-              - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+            - LangChain 🔗: integrations/langchain.md
+            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
           - LlamaIndex 🦙:
-              - LlamaIndex docs: integrations/llamaIndex.md
-              - LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
+            - LlamaIndex docs: integrations/llamaIndex.md
+            - LlamaIndex demo:  https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
           - Pydantic: python/pydantic.md
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
-          - dlt: integrations/dlt.md
-          - phidata: integrations/phidata.md
       - 🎯 Examples:
           - Overview: examples/index.md
           - 🐍 Python:
               - Overview: examples/examples_python.md
-              - Build From Scratch: examples/python_examples/build_from_scratch.md
-              - Multimodal: examples/python_examples/multimodal.md
-              - Rag: examples/python_examples/rag.md
-              - Vector Search: examples/python_examples/vector_search.md
-              - Chatbot: examples/python_examples/chatbot.md
-              - Evaluation: examples/python_examples/evaluations.md
-              - AI Agent: examples/python_examples/aiagent.md
-              - Recommender System: examples/python_examples/recommendersystem.md
-              - Miscellaneous:
-                  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-                  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+              - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+              - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+              - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+              - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
+              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
           - 👾 JavaScript:
               - Overview: examples/examples_js.md
               - Serverless Website Chatbot: examples/serverless_website_chatbot.md
@@ -219,97 +147,52 @@ nav:
               - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
           - 🦀 Rust:
               - Overview: examples/examples_rust.md
-      - 📓 Studies:
-          - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
       - 💭 FAQs: faq.md
-      - 🔍 Troubleshooting: troubleshooting.md
       - ⚙️ API reference:
           - 🐍 Python: python/python.md
           - 👾 JavaScript (vectordb): javascript/modules.md
-          - 👾 JavaScript (lancedb): js/globals.md
+          - 👾 JavaScript (lancedb): javascript/modules.md
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
       - ☁️ LanceDB Cloud:
           - Overview: cloud/index.md
           - API reference:
               - 🐍 Python: python/saas-python.md
               - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
-          - FAQs: cloud/cloud_faq.md
 
   - Quick start: basic.md
   - Concepts:
       - Vector search: concepts/vector_search.md
-      - Indexing:
-        - IVFPQ: concepts/index_ivfpq.md
-        - HNSW: concepts/index_hnsw.md
+      - Indexing: concepts/index_ivfpq.md
       - Storage: concepts/storage.md
       - Data management: concepts/data_management.md
   - Guides:
       - Working with tables: guides/tables.md
       - Building an ANN index: ann_indexes.md
       - Vector Search: search.md
-      - Full-text search (native): fts.md
-      - Full-text search (tantivy-based): fts_tantivy.md
-      - Building a scalar index: guides/scalar_index.md
+      - Full-text search: fts.md
       - Hybrid search:
           - Overview: hybrid_search/hybrid_search.md
           - Comparing Rerankers: hybrid_search/eval.md
           - Airbnb financial data example: notebooks/hybrid_search.ipynb
-      - RAG:
-          - Vanilla RAG: rag/vanilla_rag.md
-          - Multi-head RAG: rag/multi_head_rag.md
-          - Corrective RAG: rag/corrective_rag.md
-          - Agentic RAG: rag/agentic_rag.md
-          - Graph RAG: rag/graph_rag.md
-          - Self RAG: rag/self_rag.md
-          - Adaptive RAG: rag/adaptive_rag.md
-          - SFR RAG: rag/sfr_rag.md
-          - Advanced Techniques:
-            - HyDE: rag/advanced_techniques/hyde.md
-            - FLARE: rag/advanced_techniques/flare.md
       - Reranking:
           - Quickstart: reranking/index.md
           - Cohere Reranker: reranking/cohere.md
           - Linear Combination Reranker: reranking/linear_combination.md
-          - Reciprocal Rank Fusion Reranker: reranking/rrf.md
           - Cross Encoder Reranker: reranking/cross_encoder.md
           - ColBERT Reranker: reranking/colbert.md
-          - Jina Reranker: reranking/jina.md
           - OpenAI Reranker: reranking/openai.md
-          - AnswerDotAi Rerankers: reranking/answerdotai.md
           - Building Custom Rerankers: reranking/custom_reranker.md
-          - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
-      - Versioning & Reproducibility: 
-          - sync API: notebooks/reproducibility.ipynb
-          - async API: notebooks/reproducibility_async.ipynb
+      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
       - Configuring Storage: guides/storage.md
-      - Migration Guide: migration.md
+      - Sync -> Async Migration Guide: migration.md
       - Tuning retrieval performance:
           - Choosing right query type: guides/tuning_retrievers/1_query_types.md
           - Reranking: guides/tuning_retrievers/2_reranking.md
-          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
   - Managing Embeddings:
-      - Understand Embeddings: embeddings/understanding_embeddings.md
-      - Get Started: embeddings/index.md
+      - Overview: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
-      - Available models:
-          - Overview: embeddings/default_embedding_functions.md
-          - Text Embedding Functions:
-              - Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
-              - Huggingface Embedding Models: embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md
-              - Ollama Embeddings: embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md
-              - OpenAI Embeddings: embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
-              - Instructor Embeddings: embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md
-              - Gemini Embeddings: embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md
-              - Cohere Embeddings: embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md
-              - Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
-              - AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
-              - IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
-          - Multimodal Embedding Functions:
-              - OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
-              - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
-              - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
+      - Available models: embeddings/default_embedding_functions.md
       - User-defined embedding functions: embeddings/custom_embedding_function.md
       - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
       - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
@@ -318,52 +201,33 @@ nav:
       - Pandas and PyArrow: python/pandas_and_pyarrow.md
       - Polars: python/polars_arrow.md
       - DuckDB: python/duckdb.md
-      - LangChain 🦜️🔗↗: integrations/langchain.md
+      - LangChain 🦜️🔗↗: https://python.langchain.com/docs/integrations/vectorstores/lancedb
       - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-      - LlamaIndex 🦙↗: integrations/llamaIndex.md
+      - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
       - Pydantic: python/pydantic.md
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
-      - dlt: integrations/dlt.md
-      - phidata: integrations/phidata.md
   - Examples:
       - examples/index.md
-      - 🐍 Python:
-          - Overview: examples/examples_python.md
-          - Build From Scratch: examples/python_examples/build_from_scratch.md
-          - Multimodal: examples/python_examples/multimodal.md
-          - Rag: examples/python_examples/rag.md
-          - Vector Search: examples/python_examples/vector_search.md
-          - Chatbot: examples/python_examples/chatbot.md
-          - Evaluation: examples/python_examples/evaluations.md
-          - AI Agent: examples/python_examples/aiagent.md
-          - Recommender System: examples/python_examples/recommendersystem.md
-          - Miscellaneous:
-              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-      - 👾 JavaScript:
-          - Overview: examples/examples_js.md
-          - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-          - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-          - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-      - 🦀 Rust:
-          - Overview: examples/examples_rust.md
-  - Studies:
-    - studies/overview.md
-    - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
+      - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+      - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+      - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+      - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+      - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+      - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
+      - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
+      - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
   - API reference:
       - Overview: api_reference.md
       - Python: python/python.md
       - Javascript (vectordb): javascript/modules.md
-      - Javascript (lancedb): js/globals.md
+      - Javascript (lancedb): js/modules.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
   - LanceDB Cloud:
       - Overview: cloud/index.md
       - API reference:
           - 🐍 Python: python/saas-python.md
           - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
-      - FAQs: cloud/cloud_faq.md
 
 extra_css:
   - styles/global.css

docs/openapi.yml

@@ -1,487 +0,0 @@
-openapi: 3.1.0
-info:
-  version: 1.0.0
-  title: LanceDB Cloud API
-  description: |
-    LanceDB Cloud API is a RESTful API that allows users to access and modify data stored in LanceDB Cloud.
-    Table actions are considered temporary resource creations and all use POST method.
-  contact:
-    name: LanceDB support
-    url: https://lancedb.com
-    email: contact@lancedb.com
-
-servers:
-  - url: https://{db}.{region}.api.lancedb.com
-    description: LanceDB Cloud REST endpoint.
-    variables:
-      db:
-        default: ""
-        description: the name of DB
-      region:
-        default: "us-east-1"
-        description: the service region of the DB
-
-security:
-  - key_auth: []
-
-components:
-  securitySchemes:
-    key_auth:
-      name: x-api-key
-      type: apiKey
-      in: header
-  parameters:
-    table_name:
-      name: name
-      in: path
-      description: name of the table
-      required: true
-      schema:
-        type: string
-  responses:
-    invalid_request:
-      description: Invalid request
-      content:
-        text/plain:
-          schema:
-            type: string
-    not_found:
-      description: Not found
-      content:
-        text/plain:
-          schema:
-            type: string
-    unauthorized:
-      description: Unauthorized
-      content:
-        text/plain:
-          schema:
-            type: string
-  requestBodies:
-    arrow_stream_buffer:
-      description: Arrow IPC stream buffer
-      required: true
-      content:
-        application/vnd.apache.arrow.stream:
-          schema:
-            type: string
-            format: binary
-
-paths:
-  /v1/table/:
-    get:
-      description: List tables, optionally, with pagination.
-      tags:
-        - Tables
-      summary: List Tables
-      operationId: listTables
-      parameters:
-        - name: limit
-          in: query
-          description: Limits the number of items to return.
-          schema:
-            type: integer
-        - name: page_token
-          in: query
-          description: Specifies the starting position of the next query
-          schema:
-            type: string
-      responses:
-        "200":
-          description: Successfully returned a list of tables in the DB
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  tables:
-                    type: array
-                    items:
-                      type: string
-                  page_token:
-                    type: string
-
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/create/:
-    post:
-      description: Create a new table
-      summary: Create a new table
-      operationId: createTable
-      tags:
-        - Tables
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Table successfully created
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/query/:
-    post:
-      description: Vector Query
-      url: https://{db-uri}.{aws-region}.api.lancedb.com/v1/table/{name}/query/
-      tags:
-        - Data
-      summary: Vector Query
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                vector:
-                  type: FixedSizeList
-                  description: |
-                    The targetted vector to search for. Required.
-                vector_column:
-                  type: string
-                  description: |
-                    The column to query, it can be inferred from the schema if there is only one vector column.
-                prefilter:
-                  type: boolean
-                  description: |
-                    Whether to prefilter the data. Optional.
-                k:
-                  type: integer
-                  description: |
-                    The number of search results to return. Default is 10.
-                distance_type:
-                  type: string
-                  description: |
-                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
-                bypass_vector_index:
-                  type: boolean
-                  description: |
-                    Whether to bypass vector index. Optional.
-                filter:
-                  type: string
-                  description: |
-                    A filter expression that specifies the rows to query. Optional.
-                columns:
-                  type: array
-                  items:
-                    type: string
-                  description: |
-                    The columns to return. Optional.
-                nprobe:
-                  type: integer
-                  description: |
-                    The number of probes to use for search. Optional.
-                refine_factor:
-                  type: integer
-                  description: |
-                    The refine factor to use for search. Optional.
-                  default: null
-                fast_search:
-                  type: boolean
-                  description: |
-                    Whether to use fast search. Optional.
-                  default: false
-              required:
-                - vector
-
-      responses:
-        "200":
-          description: top k results if query is successfully executed
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  results:
-                    type: array
-                    items:
-                      type: object
-                      properties:
-                        id:
-                          type: integer
-                        selected_col_1_to_return:
-                          type: col_1_type
-                        selected_col_n_to_return:
-                          type: col_n_type
-                        _distance:
-                          type: float
-
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/insert/:
-    post:
-      description: Insert new data to the Table.
-      tags:
-        - Data
-      operationId: insertData
-      summary: Insert new data.
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Insert successful
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/merge_insert/:
-    post:
-      description: Create a "merge insert" operation
-        This operation can add rows, update rows, and remove rows all in a single
-        transaction. See python method `lancedb.table.Table.merge_insert` for examples.
-      tags:
-        - Data
-      summary: Merge Insert
-      operationId: mergeInsert
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-        - name: on
-          in: query
-          description: |
-            The column to use as the primary key for the merge operation.
-          required: true
-          schema:
-            type: string
-        - name: when_matched_update_all
-          in: query
-          description: |
-            Rows that exist in both the source table (new data) and
-            the target table (old data) will be updated, replacing
-            the old row with the corresponding matching row.
-          required: false
-          schema:
-            type: boolean
-        - name: when_matched_update_all_filt
-          in: query
-          description: |
-            If present then only rows that satisfy the filter expression will
-            be updated
-          required: false
-          schema:
-            type: string
-        - name: when_not_matched_insert_all
-          in: query
-          description: |
-            Rows that exist only in the source table (new data) will be
-            inserted into the target table (old data).
-          required: false
-          schema:
-            type: boolean
-        - name: when_not_matched_by_source_delete
-          in: query
-          description: |
-            Rows that exist only in the target table (old data) will be
-            deleted. An optional condition (`when_not_matched_by_source_delete_filt`)
-            can be provided to limit what data is deleted.
-          required: false
-          schema:
-            type: boolean
-        - name: when_not_matched_by_source_delete_filt
-          in: query
-          description: |
-            The filter expression that specifies the rows to delete.
-          required: false
-          schema:
-            type: string
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Merge Insert successful
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/delete/:
-    post:
-      description: Delete rows from a table.
-      tags:
-        - Data
-      summary: Delete rows from a table
-      operationId: deleteData
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                predicate:
-                  type: string
-                  description: |
-                    A filter expression that specifies the rows to delete.
-      responses:
-        "200":
-          description: Delete successful
-        "401":
-          $ref: "#/components/responses/unauthorized"
-  /v1/table/{name}/drop/:
-    post:
-      description: Drop a table
-      tags:
-        - Tables
-      summary: Drop a table
-      operationId: dropTable
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        $ref: "#/components/requestBodies/arrow_stream_buffer"
-      responses:
-        "200":
-          description: Drop successful
-        "401":
-          $ref: "#/components/responses/unauthorized"
-
-  /v1/table/{name}/describe/:
-    post:
-      description: Describe a table and return Table Information.
-      tags:
-        - Tables
-      summary: Describe a table
-      operationId: describeTable
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      responses:
-        "200":
-          description: Table information
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  table:
-                    type: string
-                  version:
-                    type: integer
-                  schema:
-                    type: string
-                  stats:
-                    type: object
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-
-  /v1/table/{name}/index/list/:
-    post:
-      description: List indexes of a table
-      tags:
-        - Tables
-      summary: List indexes of a table
-      operationId: listIndexes
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      responses:
-        "200":
-          description: Available list of indexes on the table.
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  indexes:
-                    type: array
-                    items:
-                      type: object
-                      properties:
-                        columns:
-                          type: array
-                          items:
-                            type: string
-                        index_name:
-                          type: string
-                        index_uuid:
-                          type: string
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/create_index/:
-    post:
-      description: Create vector index on a Table
-      tags:
-        - Tables
-      summary: Create vector index on a Table
-      operationId: createIndex
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                column:
-                  type: string
-                metric_type:
-                  type: string
-                  nullable: false
-                  description: |
-                    The metric type to use for the index. L2, Cosine, Dot are supported.
-                index_type:
-                  type: string
-      responses:
-        "200":
-          description: Index successfully created
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/create_scalar_index/:
-    post:
-      description: Create a scalar index on a table
-      tags:
-        - Tables
-      summary: Create a scalar index on a table
-      operationId: createScalarIndex
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                column:
-                  type: string
-                index_type:
-                  type: string
-                  required: false
-      responses:
-        "200":
-          description: Scalar Index successfully created
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"

docs/package-lock.json

@@ -19,7 +19,7 @@
     },
     "../node": {
       "name": "vectordb",
-      "version": "0.12.0",
+      "version": "0.4.6",
       "cpu": [
         "x64",
         "arm64"
@@ -31,7 +31,9 @@
         "win32"
       ],
       "dependencies": {
+        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -44,7 +46,6 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
-        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -61,19 +62,15 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "^5.1.0",
+        "typescript": "*",
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.12.0",
-        "@lancedb/vectordb-darwin-x64": "0.12.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.12.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.12.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.12.0"
-      },
-      "peerDependencies": {
-        "@apache-arrow/ts": "^14.0.2",
-        "apache-arrow": "^14.0.2"
+        "@lancedb/vectordb-darwin-arm64": "0.4.6",
+        "@lancedb/vectordb-darwin-x64": "0.4.6",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
       }
     },
     "../node/node_modules/apache-arrow": {

docs/requirements.txt

@@ -1,7 +1,5 @@
 mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
-mkdocstrings[python]==0.25.2
-griffe
-mkdocs-render-swagger-plugin
-pydantic
+mkdocstrings[python]==0.20.0
+pydantic

docs/src/ann_indexes.md

@@ -18,46 +18,33 @@ See the [indexing](concepts/index_ivfpq.md) concepts guide for more information
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
-    === "Sync API"
 
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index"
-        ```
-    === "Async API"
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+    ```python
+    import lancedb
+    import numpy as np
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-ivfpq"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index_async"
-        ```
+    # Create 10,000 sample vectors
+    data = [{"vector": row, "item": f"item {i}"}
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
 
-=== "TypeScript"
+    # Add the vectors to a table
+    tbl = db.create_table("my_vectors", data=data)
 
-    === "@lancedb/lancedb"
+    # Create and train the index - you need to have enough data in the table for an effective training step
+    tbl.create_index(num_partitions=256, num_sub_vectors=96)
+    ```
 
-        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
+=== "Typescript"
 
-        ```typescript
-        --8<--- "nodejs/examples/ann_indexes.test.ts:import"
+    ```typescript
+    --8<--- "docs/src/ann_indexes.ts:import"
 
-        --8<-- "nodejs/examples/ann_indexes.test.ts:ingest"
-        ```
-
-    === "vectordb (deprecated)"
-
-        Creating indexes is done via the [lancedb.Table.createIndex](../javascript/interfaces/Table.md/#createIndex) method.
-
-        ```typescript
-        --8<--- "docs/src/ann_indexes.ts:import"
-
-        --8<-- "docs/src/ann_indexes.ts:ingest"
-        ```
+    --8<-- "docs/src/ann_indexes.ts:ingest"
+    ```
 
 === "Rust"
 
@@ -82,7 +69,6 @@ The following IVF_PQ paramters can be specified:
 - **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
   For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
   a single PQ code. The default is the dimension of the vector divided by 16.
-- **num_bits**: The number of bits used to encode each sub-vector. Only 4 and 8 are supported. The higher the number of bits, the higher the accuracy of the index, also the slower search. The default is 8.
 
 !!! note
 
@@ -105,30 +91,28 @@ You can specify the GPU device to train IVF partitions via
 
 === "Linux"
 
-    <!-- skip-test -->
-    ``` { .python .copy }
-    # Create index using CUDA on Nvidia GPUs.
-    tbl.create_index(
-        num_partitions=256,
-        num_sub_vectors=96,
-        accelerator="cuda"
-    )
-    ```
+     <!-- skip-test -->
+     ``` { .python .copy }
+     # Create index using CUDA on Nvidia GPUs.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="cuda"
+     )
+     ```
 
 === "MacOS"
 
-    <!-- skip-test -->
-    ```python
-    # Create index using MPS on Apple Silicon.
-    tbl.create_index(
-        num_partitions=256,
-        num_sub_vectors=96,
-        accelerator="mps"
-    )
-    ```
-!!! note
-    GPU based indexing is not yet supported with our asynchronous client.
-    
+     <!-- skip-test -->
+     ```python
+     # Create index using MPS on Apple Silicon.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="mps"
+     )
+     ```
+
 Troubleshooting:
 
 If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
@@ -142,27 +126,23 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 - **limit** (default: 10): The amount of results that will be returned
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
-    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
-
+  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
+  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
+  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
   A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
-!!! note
-    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
-
+  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
+  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async"
-        ```
+    ```python
+    tbl.search(np.random.random((1536))) \
+        .limit(2) \
+        .nprobes(20) \
+        .refine_factor(10) \
+        .to_pandas()
+    ```
 
     ```text
                                               vector       item       _distance
@@ -170,19 +150,11 @@ There are a couple of parameters that can be used to fine-tune the search:
     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
     ```
 
-=== "TypeScript"
+=== "Typescript"
 
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search1"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/ann_indexes.ts:search1"
-        ```
+    ```typescript
+    --8<-- "docs/src/ann_indexes.ts:search1"
+    ```
 
 === "Rust"
 
@@ -199,30 +171,16 @@ The search will return the data requested in addition to the distance of each it
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_filter"
-        ```
-    === "Async API"
+     ```python
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+     ```
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_filter"
-        ```
+=== "Typescript"
 
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search2"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```javascript
-        --8<-- "docs/src/ann_indexes.ts:search2"
-        ```
+     ```javascript
+     --8<-- "docs/src/ann_indexes.ts:search2"
+     ```
 
 ### Projections (select clause)
 
@@ -230,37 +188,23 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-    === "Sync API"
+     ```python
+     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+     ```
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_select"
-        ```
-    === "Async API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_select"
-        ```
+     ```text
+                                                   vector _distance
+     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+     ...
+     ```
 
-    ```text
-                                                vector _distance
-    0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-    1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-    ...
-    ```
+=== "Typescript"
 
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search3"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/ann_indexes.ts:search3"
-        ```
+     ```typescript
+     --8<-- "docs/src/ann_indexes.ts:search3"
+     ```
 
 ## FAQ
 
@@ -293,15 +237,7 @@ Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` t
 Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
 On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
 
-`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. The number should be a factor of the vector dimension. Because
+`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
 PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
-less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
-
-!!! note
-    if `num_sub_vectors` is set to be greater than the vector dimension, you will see errors like `attempt to divide by zero`
-
-### How to choose `m` and `ef_construction` for `IVF_HNSW_*` index?
-
-`m` determines the number of connections a new node establishes with its closest neighbors upon entering the graph. Typically, `m` falls within the range of 5 to 48. Lower `m` values are suitable for low-dimensional data or scenarios where recall is less critical. Conversely, higher `m` values are beneficial for high-dimensional data or when high recall is required. In essence, a larger `m` results in a denser graph with increased connectivity, but at the expense of higher memory consumption.
-
-`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase
+less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
+more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/api_reference.md

@@ -4,5 +4,5 @@ The API reference for the LanceDB client SDKs are available at the following loc
 
 - [Python](python/python.md)
 - [JavaScript (legacy vectordb package)](javascript/modules.md)
-- [JavaScript (newer @lancedb/lancedb package)](js/globals.md)
+- [JavaScript (newer @lancedb/lancedb package)](js/modules.md)
 - [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/assets/colab.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="117" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="117" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h30v20H0z"/><path fill="#007ec6" d="M30 0h87v20H30z"/><path fill="url(#b)" d="M0 0h117v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"><svg x="4px" y="0px" width="22px" height="20px" viewBox="-2 0 28 24" style="background-color: #fff;border-radius: 1px;"><path style="fill:#e8710a;" d="M1.977,16.77c-2.667-2.277-2.605-7.079,0-9.357C2.919,8.057,3.522,9.075,4.49,9.691c-1.152,1.6-1.146,3.201-0.004,4.803C3.522,15.111,2.918,16.126,1.977,16.77z"/><path style="fill:#f9ab00;" d="M12.257,17.114c-1.767-1.633-2.485-3.658-2.118-6.02c0.451-2.91,2.139-4.893,4.946-5.678c2.565-0.718,4.964-0.217,6.878,1.819c-0.884,0.743-1.707,1.547-2.434,2.446C18.488,8.827,17.319,8.435,16,8.856c-2.404,0.767-3.046,3.241-1.494,5.644c-0.241,0.275-0.493,0.541-0.721,0.826C13.295,15.939,12.511,16.3,12.257,17.114z"/><path style="fill:#e8710a;" d="M19.529,9.682c0.727-0.899,1.55-1.703,2.434-2.446c2.703,2.783,2.701,7.031-0.005,9.764c-2.648,2.674-6.936,2.725-9.701,0.115c0.254-0.814,1.038-1.175,1.528-1.788c0.228-0.285,0.48-0.552,0.721-0.826c1.053,0.916,2.254,1.268,3.6,0.83C20.502,14.551,21.151,11.927,19.529,9.682z"/><path style="fill:#f9ab00;" d="M4.49,9.691C3.522,9.075,2.919,8.057,1.977,7.413c2.209-2.398,5.721-2.942,8.476-1.355c0.555,0.32,0.719,0.606,0.285,1.128c-0.157,0.188-0.258,0.422-0.391,0.631c-0.299,0.47-0.509,1.067-0.929,1.371C8.933,9.539,8.523,8.847,8.021,8.746C6.673,8.475,5.509,8.787,4.49,9.691z"/><path style="fill:#f9ab00;" d="M1.977,16.77c0.941-0.644,1.545-1.659,2.509-2.277c1.373,1.152,2.85,1.433,4.45,0.499c0.332-0.194,0.503-0.088,0.673,0.19c0.386,0.635,0.753,1.285,1.181,1.89c0.34,0.48,0.222,0.715-0.253,1.006C7.84,19.73,4.205,19.188,1.977,16.77z"/></svg><text x="245" y="140" transform="scale(.1)" textLength="30"> </text><text x="725" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">Open in Colab</text><text x="725" y="140" transform="scale(.1)" textLength="770">Open in Colab</text></g> </svg>

docs/src/assets/ghost.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="88.25" height="28" role="img" aria-label="GHOST"><title>GHOST</title><g shape-rendering="crispEdges"><rect width="88.25" height="28" fill="#000"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSIjZjdkZjFlIiByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+R2hvc3Q8L3RpdGxlPjxwYXRoIGQ9Ik0xMiAwQzUuMzczIDAgMCA1LjM3MyAwIDEyczUuMzczIDEyIDEyIDEyIDEyLTUuMzczIDEyLTEyUzE4LjYyNyAwIDEyIDB6bS4yNTYgMi4zMTNjMi40Ny4wMDUgNS4xMTYgMi4wMDggNS44OTggMi45NjJsLjI0NC4zYzEuNjQgMS45OTQgMy41NjkgNC4zNCAzLjU2OSA2Ljk2NiAwIDMuNzE5LTIuOTggNS44MDgtNi4xNTggNy41MDgtMS40MzMuNzY2LTIuOTggMS41MDgtNC43NDggMS41MDgtNC41NDMgMC04LjM2Ni0zLjU2OS04LjM2Ni04LjExMiAwLS43MDYuMTctMS40MjUuMzQyLTIuMTUuMTIyLS41MTUuMjQ0LTEuMDMzLjMwNy0xLjU0OS41NDgtNC41MzkgMi45NjctNi43OTUgOC40MjItNy40MDhhNC4yOSA0LjI5IDAgMDEuNDktLjAyNloiLz48L3N2Zz4="/><text transform="scale(.1)" x="541.25" y="175" textLength="442.5" fill="#fff" font-weight="bold">GHOST</text></g></svg>

docs/src/assets/github.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="95.5" height="28" role="img" aria-label="GITHUB"><title>GITHUB</title><g shape-rendering="crispEdges"><rect width="95.5" height="28" fill="#121011"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgcm9sZT0iaW1nIiB2aWV3Qm94PSIwIDAgMjQgMjQiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PHRpdGxlPkdpdEh1YjwvdGl0bGU+PHBhdGggZD0iTTEyIC4yOTdjLTYuNjMgMC0xMiA1LjM3My0xMiAxMiAwIDUuMzAzIDMuNDM4IDkuOCA4LjIwNSAxMS4zODUuNi4xMTMuODItLjI1OC44Mi0uNTc3IDAtLjI4NS0uMDEtMS4wNC0uMDE1LTIuMDQtMy4zMzguNzI0LTQuMDQyLTEuNjEtNC4wNDItMS42MUM0LjQyMiAxOC4wNyAzLjYzMyAxNy43IDMuNjMzIDE3LjdjLTEuMDg3LS43NDQuMDg0LS43MjkuMDg0LS43MjkgMS4yMDUuMDg0IDEuODM4IDEuMjM2IDEuODM4IDEuMjM2IDEuMDcgMS44MzUgMi44MDkgMS4zMDUgMy40OTUuOTk4LjEwOC0uNzc2LjQxNy0xLjMwNS43Ni0xLjYwNS0yLjY2NS0uMy01LjQ2Ni0xLjMzMi01LjQ2Ni01LjkzIDAtMS4zMS40NjUtMi4zOCAxLjIzNS0zLjIyLS4xMzUtLjMwMy0uNTQtMS41MjMuMTA1LTMuMTc2IDAgMCAxLjAwNS0uMzIyIDMuMyAxLjIzLjk2LS4yNjcgMS45OC0uMzk5IDMtLjQwNSAxLjAyLjAwNiAyLjA0LjEzOCAzIC40MDUgMi4yOC0xLjU1MiAzLjI4NS0xLjIzIDMuMjg1LTEuMjMuNjQ1IDEuNjUzLjI0IDIuODczLjEyIDMuMTc2Ljc2NS44NCAxLjIzIDEuOTEgMS4yMyAzLjIyIDAgNC42MS0yLjgwNSA1LjYyNS01LjQ3NSA1LjkyLjQyLjM2LjgxIDEuMDk2LjgxIDIuMjIgMCAxLjYwNi0uMDE1IDIuODk2LS4wMTUgMy4yODYgMCAuMzE1LjIxLjY5LjgyNS41N0MyMC41NjUgMjIuMDkyIDI0IDE3LjU5MiAyNCAxMi4yOTdjMC02LjYyNy01LjM3My0xMi0xMi0xMiIvPjwvc3ZnPg=="/><text transform="scale(.1)" x="577.5" y="175" textLength="515" fill="#fff" font-weight="bold">GITHUB</text></g></svg>

docs/src/assets/open_hf_space.svg

@@ -1,22 +0,0 @@
-<svg width="147" height="20" viewBox="0 0 147 20" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect x="0.5" y="0.5" width="145.482" height="19" rx="9.5" fill="white" stroke="#EFEFEF"/>
-<path d="M14.1863 10.9251V12.7593H16.0205V10.9251H14.1863Z" fill="#FF3270"/>
-<path d="M17.8707 10.9251V12.7593H19.7049V10.9251H17.8707Z" fill="#861FFF"/>
-<path d="M14.1863 7.24078V9.07496H16.0205V7.24078H14.1863Z" fill="#097EFF"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M12.903 6.77179C12.903 6.32194 13.2676 5.95728 13.7175 5.95728C14.1703 5.95728 15.2556 5.95728 16.1094 5.95728C16.7538 5.95728 17.2758 6.47963 17.2758 7.12398V9.6698H19.8217C20.4661 9.6698 20.9884 10.1922 20.9884 10.8365C20.9884 11.6337 20.9884 12.4309 20.9884 13.2282C20.9884 13.678 20.6237 14.0427 20.1738 14.0427H17.3039H16.5874H13.7175C13.2676 14.0427 12.903 13.678 12.903 13.2282V9.71653V9.64174V6.77179ZM14.1863 7.24066V9.07485H16.0205V7.24066H14.1863ZM14.1863 12.7593V10.9251H16.0205V12.7593H14.1863ZM17.8708 12.7593V10.9251H19.705V12.7593H17.8708Z" fill="black"/>
-<path d="M18.614 8.35468L20.7796 6.18905M20.7796 6.18905V7.66073M20.7796 6.18905L19.2724 6.18905" stroke="black" stroke-width="0.686298" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M31.6082 13.9838C30.8546 13.9838 30.1895 13.802 29.6132 13.4385C29.0368 13.066 28.5846 12.5429 28.2565 11.869C27.9373 11.1862 27.7777 10.3749 27.7777 9.43501C27.7777 8.49511 27.9373 7.69265 28.2565 7.02762C28.5846 6.3626 29.0368 5.85275 29.6132 5.49807C30.1895 5.14339 30.8546 4.96605 31.6082 4.96605C32.3708 4.96605 33.0403 5.14339 33.6166 5.49807C34.193 5.85275 34.6408 6.3626 34.96 7.02762C35.2881 7.69265 35.4521 8.49511 35.4521 9.43501C35.4521 10.3749 35.2881 11.1862 34.96 11.869C34.6408 12.5429 34.193 13.066 33.6166 13.4385C33.0403 13.802 32.3708 13.9838 31.6082 13.9838ZM31.6082 12.6404C32.291 12.6404 32.8363 12.3523 33.2442 11.7759C33.6521 11.1907 33.856 10.4104 33.856 9.43501C33.856 8.45964 33.6521 7.69708 33.2442 7.14733C32.8363 6.58871 32.291 6.3094 31.6082 6.3094C30.9255 6.3094 30.3802 6.58871 29.9723 7.14733C29.5644 7.69708 29.3605 8.45964 29.3605 9.43501C29.3605 10.4104 29.5644 11.1907 29.9723 11.7759C30.3802 12.3523 30.9255 12.6404 31.6082 12.6404Z" fill="#2C3236"/>
-<path d="M37.0592 16.4045V7.29363H38.3227L38.4291 7.98526H38.4823C38.7572 7.75472 39.0631 7.55521 39.4 7.38674C39.7459 7.21826 40.0961 7.13403 40.4508 7.13403C41.2665 7.13403 41.8961 7.43551 42.3395 8.03846C42.7917 8.64142 43.0178 9.44831 43.0178 10.4591C43.0178 11.204 42.8848 11.8424 42.6188 12.3744C42.3528 12.8976 42.0069 13.2966 41.5813 13.5715C41.1646 13.8463 40.7124 13.9838 40.2247 13.9838C39.9409 13.9838 39.6572 13.9217 39.3734 13.7976C39.0897 13.6646 38.8148 13.4872 38.5488 13.2656L38.5887 14.3562V16.4045H37.0592ZM39.9055 12.7202C40.3399 12.7202 40.7035 12.5296 40.9961 12.1483C41.2887 11.767 41.435 11.2084 41.435 10.4724C41.435 9.81629 41.3242 9.30644 41.1025 8.94289C40.8808 8.57935 40.5217 8.39757 40.0252 8.39757C39.5641 8.39757 39.0853 8.64142 38.5887 9.1291V12.1749C38.8281 12.37 39.0587 12.5119 39.2803 12.6005C39.502 12.6803 39.7104 12.7202 39.9055 12.7202Z" fill="#2C3236"/>
-<path d="M47.3598 13.9838C46.7568 13.9838 46.2115 13.8508 45.7238 13.5848C45.2361 13.3099 44.8504 12.9197 44.5667 12.4143C44.2829 11.9 44.141 11.2838 44.141 10.5656C44.141 9.85619 44.2829 9.24437 44.5667 8.73009C44.8593 8.2158 45.2361 7.82122 45.6972 7.54634C46.1583 7.27147 46.6415 7.13403 47.147 7.13403C47.741 7.13403 48.2376 7.26703 48.6366 7.53304C49.0356 7.79018 49.3371 8.15373 49.541 8.62368C49.745 9.08476 49.847 9.62122 49.847 10.233C49.847 10.5523 49.8248 10.8005 49.7805 10.9779H45.6307C45.7016 11.5542 45.91 12.002 46.2558 12.3212C46.6016 12.6404 47.0361 12.8 47.5593 12.8C47.843 12.8 48.1046 12.7601 48.344 12.6803C48.5923 12.5917 48.8361 12.472 49.0755 12.3212L49.5942 13.2789C49.2839 13.4828 48.9381 13.6513 48.5568 13.7843C48.1755 13.9173 47.7765 13.9838 47.3598 13.9838ZM45.6174 9.94043H48.5169C48.5169 9.43501 48.4061 9.04043 48.1844 8.75669C47.9627 8.46408 47.6302 8.31777 47.1869 8.31777C46.8056 8.31777 46.4642 8.45964 46.1627 8.74339C45.8701 9.01826 45.6883 9.41728 45.6174 9.94043Z" fill="#2C3236"/>
-<path d="M51.3078 13.8242V7.29363H52.5714L52.6778 8.17147H52.731C53.0236 7.88772 53.3428 7.64388 53.6886 7.43994C54.0344 7.236 54.429 7.13403 54.8724 7.13403C55.5728 7.13403 56.0827 7.36014 56.4019 7.81235C56.7211 8.26457 56.8807 8.90299 56.8807 9.72762V13.8242H55.3512V9.92713C55.3512 9.38624 55.2714 9.00496 55.1118 8.78329C54.9522 8.56161 54.6906 8.45078 54.327 8.45078C54.0433 8.45078 53.7906 8.52171 53.5689 8.66358C53.3561 8.79659 53.1123 8.99609 52.8374 9.2621V13.8242H51.3078Z" fill="#2C3236"/>
-<path d="M61.4131 13.8242V7.29363H62.9426V13.8242H61.4131ZM62.1845 6.14979C61.9096 6.14979 61.6879 6.06999 61.5195 5.91038C61.351 5.75078 61.2668 5.53797 61.2668 5.27196C61.2668 5.01482 61.351 4.80644 61.5195 4.64684C61.6879 4.48723 61.9096 4.40743 62.1845 4.40743C62.4594 4.40743 62.6811 4.48723 62.8495 4.64684C63.018 4.80644 63.1022 5.01482 63.1022 5.27196C63.1022 5.53797 63.018 5.75078 62.8495 5.91038C62.6811 6.06999 62.4594 6.14979 62.1845 6.14979Z" fill="#2C3236"/>
-<path d="M64.8941 13.8242V7.29363H66.1576L66.264 8.17147H66.3172C66.6098 7.88772 66.929 7.64388 67.2748 7.43994C67.6207 7.236 68.0152 7.13403 68.4586 7.13403C69.1591 7.13403 69.6689 7.36014 69.9881 7.81235C70.3074 8.26457 70.467 8.90299 70.467 9.72762V13.8242H68.9374V9.92713C68.9374 9.38624 68.8576 9.00496 68.698 8.78329C68.5384 8.56161 68.2768 8.45078 67.9133 8.45078C67.6295 8.45078 67.3768 8.52171 67.1551 8.66358C66.9423 8.79659 66.6985 8.99609 66.4236 9.2621V13.8242H64.8941Z" fill="#2C3236"/>
-<path d="M75.1323 13.8242V5.12565H76.6752V8.62368H80.1998V5.12565H81.7427V13.8242H80.1998V9.96703H76.6752V13.8242H75.1323Z" fill="#2C3236"/>
-<path d="M83.9517 13.8242V5.12565H89.2054V6.4291H85.4945V8.88969H88.6601V10.1931H85.4945V13.8242H83.9517Z" fill="#2C3236"/>
-<path d="M95.9349 13.9838C95.3497 13.9838 94.7822 13.8729 94.2324 13.6513C93.6915 13.4296 93.2127 13.1148 92.796 12.7069L93.7004 11.6562C94.0108 11.9488 94.3654 12.1882 94.7645 12.3744C95.1635 12.5518 95.5625 12.6404 95.9615 12.6404C96.458 12.6404 96.8349 12.5385 97.092 12.3345C97.3492 12.1306 97.4778 11.8601 97.4778 11.5232C97.4778 11.1596 97.3492 10.8981 97.092 10.7385C96.8438 10.5789 96.5245 10.4148 96.1344 10.2463L94.9374 9.72762C94.6536 9.60348 94.3743 9.44388 94.0994 9.2488C93.8334 9.05373 93.6117 8.80546 93.4344 8.50398C93.2659 8.2025 93.1817 7.83895 93.1817 7.41334C93.1817 6.95225 93.3058 6.53994 93.5541 6.17639C93.8113 5.80398 94.1571 5.51137 94.5915 5.29856C95.0349 5.07689 95.5403 4.96605 96.1078 4.96605C96.6132 4.96605 97.1009 5.06802 97.5709 5.27196C98.0408 5.46703 98.4442 5.73304 98.7812 6.06999L97.9965 7.05423C97.7216 6.82368 97.429 6.64191 97.1186 6.5089C96.8172 6.3759 96.4802 6.3094 96.1078 6.3094C95.6999 6.3094 95.3674 6.4025 95.1103 6.58871C94.862 6.76605 94.7379 7.01432 94.7379 7.33353C94.7379 7.55521 94.7999 7.74142 94.9241 7.89215C95.0571 8.03403 95.23 8.15816 95.4428 8.26457C95.6556 8.36211 95.8817 8.45964 96.1211 8.55718L97.3048 9.0493C97.8191 9.27097 98.2403 9.56358 98.5684 9.92713C98.8965 10.2818 99.0605 10.7739 99.0605 11.4035C99.0605 11.8734 98.9364 12.3035 98.6881 12.6936C98.4398 13.0838 98.0807 13.3986 97.6108 13.638C97.1497 13.8685 96.591 13.9838 95.9349 13.9838Z" fill="#2C3236"/>
-<path d="M100.509 16.4045V7.29363H101.773L101.879 7.98526H101.932C102.207 7.75472 102.513 7.55521 102.85 7.38674C103.196 7.21826 103.546 7.13403 103.901 7.13403C104.717 7.13403 105.346 7.43551 105.79 8.03846C106.242 8.64142 106.468 9.44831 106.468 10.4591C106.468 11.204 106.335 11.8424 106.069 12.3744C105.803 12.8976 105.457 13.2966 105.031 13.5715C104.615 13.8463 104.162 13.9838 103.675 13.9838C103.391 13.9838 103.107 13.9217 102.824 13.7976C102.54 13.6646 102.265 13.4872 101.999 13.2656L102.039 14.3562V16.4045H100.509ZM103.356 12.7202C103.79 12.7202 104.154 12.5296 104.446 12.1483C104.739 11.767 104.885 11.2084 104.885 10.4724C104.885 9.81629 104.774 9.30644 104.553 8.94289C104.331 8.57935 103.972 8.39757 103.475 8.39757C103.014 8.39757 102.535 8.64142 102.039 9.1291V12.1749C102.278 12.37 102.509 12.5119 102.73 12.6005C102.952 12.6803 103.16 12.7202 103.356 12.7202Z" fill="#2C3236"/>
-<path d="M109.444 13.9838C108.876 13.9838 108.411 13.8064 108.047 13.4518C107.692 13.0971 107.515 12.636 107.515 12.0685C107.515 11.368 107.821 10.8271 108.433 10.4458C109.045 10.0557 110.02 9.78969 111.359 9.64782C111.35 9.30201 111.257 9.00496 111.08 8.75669C110.911 8.49954 110.605 8.37097 110.162 8.37097C109.843 8.37097 109.528 8.43304 109.218 8.55718C108.916 8.68132 108.619 8.83206 108.326 9.0094L107.768 7.98526C108.131 7.75472 108.539 7.55521 108.991 7.38674C109.452 7.21826 109.94 7.13403 110.454 7.13403C111.27 7.13403 111.878 7.37787 112.277 7.86555C112.685 8.34437 112.888 9.04043 112.888 9.95373V13.8242H111.625L111.518 13.1059H111.465C111.173 13.3542 110.858 13.5626 110.521 13.7311C110.193 13.8995 109.834 13.9838 109.444 13.9838ZM109.936 12.7867C110.202 12.7867 110.441 12.7247 110.654 12.6005C110.876 12.4675 111.111 12.2902 111.359 12.0685V10.6055C110.472 10.7207 109.856 10.8936 109.51 11.1242C109.164 11.3458 108.991 11.6207 108.991 11.9488C108.991 12.2414 109.08 12.4542 109.257 12.5872C109.435 12.7202 109.661 12.7867 109.936 12.7867Z" fill="#2C3236"/>
-<path d="M117.446 13.9838C116.851 13.9838 116.315 13.8508 115.836 13.5848C115.366 13.3099 114.989 12.9197 114.706 12.4143C114.431 11.9 114.293 11.2838 114.293 10.5656C114.293 9.83846 114.444 9.2222 114.746 8.71679C115.047 8.2025 115.446 7.81235 115.943 7.54634C116.448 7.27147 116.989 7.13403 117.565 7.13403C117.982 7.13403 118.346 7.20496 118.656 7.34684C118.966 7.48871 119.241 7.66161 119.48 7.86555L118.736 8.86309C118.567 8.71235 118.394 8.59708 118.217 8.51728C118.04 8.42861 117.849 8.38427 117.645 8.38427C117.122 8.38427 116.692 8.58378 116.355 8.98279C116.027 9.38181 115.863 9.9094 115.863 10.5656C115.863 11.2128 116.022 11.736 116.342 12.135C116.67 12.534 117.091 12.7335 117.605 12.7335C117.862 12.7335 118.102 12.6803 118.323 12.5739C118.554 12.4587 118.762 12.3256 118.948 12.1749L119.574 13.1857C119.272 13.4518 118.935 13.6513 118.563 13.7843C118.19 13.9173 117.818 13.9838 117.446 13.9838Z" fill="#2C3236"/>
-<path d="M123.331 13.9838C122.728 13.9838 122.183 13.8508 121.695 13.5848C121.207 13.3099 120.822 12.9197 120.538 12.4143C120.254 11.9 120.112 11.2838 120.112 10.5656C120.112 9.85619 120.254 9.24437 120.538 8.73009C120.83 8.2158 121.207 7.82122 121.668 7.54634C122.13 7.27147 122.613 7.13403 123.118 7.13403C123.712 7.13403 124.209 7.26703 124.608 7.53304C125.007 7.79018 125.308 8.15373 125.512 8.62368C125.716 9.08476 125.818 9.62122 125.818 10.233C125.818 10.5523 125.796 10.8005 125.752 10.9779H121.602C121.673 11.5542 121.881 12.002 122.227 12.3212C122.573 12.6404 123.007 12.8 123.53 12.8C123.814 12.8 124.076 12.7601 124.315 12.6803C124.563 12.5917 124.807 12.472 125.047 12.3212L125.565 13.2789C125.255 13.4828 124.909 13.6513 124.528 13.7843C124.147 13.9173 123.748 13.9838 123.331 13.9838ZM121.589 9.94043H124.488C124.488 9.43501 124.377 9.04043 124.156 8.75669C123.934 8.46408 123.601 8.31777 123.158 8.31777C122.777 8.31777 122.435 8.45964 122.134 8.74339C121.841 9.01826 121.66 9.41728 121.589 9.94043Z" fill="#2C3236"/>
-<path d="M129.101 13.9838C128.658 13.9838 128.215 13.8995 127.771 13.7311C127.328 13.5537 126.947 13.3365 126.627 13.0793L127.346 12.0951C127.638 12.3168 127.931 12.4941 128.223 12.6271C128.516 12.7601 128.826 12.8266 129.154 12.8266C129.509 12.8266 129.771 12.7513 129.939 12.6005C130.108 12.4498 130.192 12.2636 130.192 12.0419C130.192 11.8557 130.121 11.705 129.979 11.5897C129.846 11.4656 129.673 11.3591 129.46 11.2705C129.248 11.1729 129.026 11.0798 128.795 10.9912C128.512 10.8848 128.228 10.7562 127.944 10.6055C127.669 10.4458 127.443 10.2463 127.266 10.0069C127.088 9.75866 127 9.45274 127 9.0892C127 8.51284 127.213 8.04289 127.638 7.67935C128.064 7.3158 128.64 7.13403 129.367 7.13403C129.828 7.13403 130.241 7.21383 130.604 7.37344C130.968 7.53304 131.282 7.71482 131.548 7.91876L130.844 8.84979C130.613 8.68132 130.378 8.54831 130.139 8.45078C129.908 8.34437 129.664 8.29117 129.407 8.29117C129.079 8.29117 128.835 8.36211 128.676 8.50398C128.516 8.63698 128.436 8.80545 128.436 9.0094C128.436 9.26654 128.569 9.46161 128.835 9.59462C129.101 9.72762 129.412 9.85619 129.766 9.98033C130.068 10.0867 130.36 10.2197 130.644 10.3793C130.928 10.5301 131.163 10.7296 131.349 10.9779C131.544 11.2261 131.642 11.5542 131.642 11.9621C131.642 12.5207 131.424 12.9995 130.99 13.3986C130.555 13.7887 129.926 13.9838 129.101 13.9838Z" fill="#2C3236"/>
-</svg>

docs/src/assets/python.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="97.5" height="28" role="img" aria-label="PYTHON"><title>PYTHON</title><g shape-rendering="crispEdges"><rect width="97.5" height="28" fill="#3670a0"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSIjZmZkZDU0IiByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+UHl0aG9uPC90aXRsZT48cGF0aCBkPSJNMTQuMjUuMThsLjkuMi43My4yNi41OS4zLjQ1LjMyLjM0LjM0LjI1LjM0LjE2LjMzLjEuMy4wNC4yNi4wMi4yLS4wMS4xM1Y4LjVsLS4wNS42My0uMTMuNTUtLjIxLjQ2LS4yNi4zOC0uMy4zMS0uMzMuMjUtLjM1LjE5LS4zNS4xNC0uMzMuMS0uMy4wNy0uMjYuMDQtLjIxLjAySDguNzdsLS42OS4wNS0uNTkuMTQtLjUuMjItLjQxLjI3LS4zMy4zMi0uMjcuMzUtLjIuMzYtLjE1LjM3LS4xLjM1LS4wNy4zMi0uMDQuMjctLjAyLjIxdjMuMDZIMy4xN2wtLjIxLS4wMy0uMjgtLjA3LS4zMi0uMTItLjM1LS4xOC0uMzYtLjI2LS4zNi0uMzYtLjM1LS40Ni0uMzItLjU5LS4yOC0uNzMtLjIxLS44OC0uMTQtMS4wNS0uMDUtMS4yMy4wNi0xLjIyLjE2LTEuMDQuMjQtLjg3LjMyLS43MS4zNi0uNTcuNC0uNDQuNDItLjMzLjQyLS4yNC40LS4xNi4zNi0uMS4zMi0uMDUuMjQtLjAxaC4xNmwuMDYuMDFoOC4xNnYtLjgzSDYuMThsLS4wMS0yLjc1LS4wMi0uMzcuMDUtLjM0LjExLS4zMS4xNy0uMjguMjUtLjI2LjMxLS4yMy4zOC0uMi40NC0uMTguNTEtLjE1LjU4LS4xMi42NC0uMS43MS0uMDYuNzctLjA0Ljg0LS4wMiAxLjI3LjA1em0tNi4zIDEuOThsLS4yMy4zMy0uMDguNDEuMDguNDEuMjMuMzQuMzMuMjIuNDEuMDkuNDEtLjA5LjMzLS4yMi4yMy0uMzQuMDgtLjQxLS4wOC0uNDEtLjIzLS4zMy0uMzMtLjIyLS40MS0uMDktLjQxLjA5em0xMy4wOSAzLjk1bC4yOC4wNi4zMi4xMi4zNS4xOC4zNi4yNy4zNi4zNS4zNS40Ny4zMi41OS4yOC43My4yMS44OC4xNCAxLjA0LjA1IDEuMjMtLjA2IDEuMjMtLjE2IDEuMDQtLjI0Ljg2LS4zMi43MS0uMzYuNTctLjQuNDUtLjQyLjMzLS40Mi4yNC0uNC4xNi0uMzYuMDktLjMyLjA1LS4yNC4wMi0uMTYtLjAxaC04LjIydi44Mmg1Ljg0bC4wMSAyLjc2LjAyLjM2LS4wNS4zNC0uMTEuMzEtLjE3LjI5LS4yNS4yNS0uMzEuMjQtLjM4LjItLjQ0LjE3LS41MS4xNS0uNTguMTMtLjY0LjA5LS43MS4wNy0uNzcuMDQtLjg0LjAxLTEuMjctLjA0LTEuMDctLjE0LS45LS4yLS43My0uMjUtLjU5LS4zLS40NS0uMzMtLjM0LS4zNC0uMjUtLjM0LS4xNi0uMzMtLjEtLjMtLjA0LS4yNS0uMDItLjIuMDEtLjEzdi01LjM0bC4wNS0uNjQuMTMtLjU0LjIxLS40Ni4yNi0uMzguMy0uMzIuMzMtLjI0LjM1LS4yLjM1LS4xNC4zMy0uMS4zLS4wNi4yNi0uMDQuMjEtLjAyLjEzLS4wMWg1Ljg0bC42OS0uMDUuNTktLjE0LjUtLjIxLjQxLS4yOC4zMy0uMzIuMjctLjM1LjItLjM2LjE1LS4zNi4xLS4zNS4wNy0uMzIuMDQtLjI4LjAyLS4yMVY2LjA3aDIuMDlsLjE0LjAxem0tNi40NyAxNC4yNWwtLjIzLjMzLS4wOC40MS4wOC40MS4yMy4zMy4zMy4yMy40MS4wOC40MS0uMDguMzMtLjIzLjIzLS4zMy4wOC0uNDEtLjA4LS40MS0uMjMtLjMzLS4zMy0uMjMtLjQxLS4wOC0uNDEuMDh6Ii8+PC9zdmc+"/><text transform="scale(.1)" x="587.5" y="175" textLength="535" fill="#fff" font-weight="bold">PYTHON</text></g></svg>

docs/src/basic.md

@@ -16,60 +16,11 @@
       pip install lancedb
       ```
 
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
+=== "Typescript"
 
-        ```shell
-        npm install @lancedb/lancedb
-        ```
-        !!! note "Bundling `@lancedb/lancedb` apps with Webpack"
-
-            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-            ```javascript
-            /** @type {import('next').NextConfig} */
-            module.exports = ({
-            webpack(config) {
-                config.externals.push({ '@lancedb/lancedb': '@lancedb/lancedb' })
-                return config;
-            }
-            })
-            ```
-
-        !!! note "Yarn users"
-
-            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
-
-            ```shell
-            yarn add apache-arrow
-            ```
-
-    === "vectordb (deprecated)"
-
-        ```shell
-        npm install vectordb
-        ```
-        !!! note "Bundling `vectordb` apps with Webpack"
-
-            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-            ```javascript
-            /** @type {import('next').NextConfig} */
-            module.exports = ({
-            webpack(config) {
-                config.externals.push({ vectordb: 'vectordb' })
-                return config;
-            }
-            })
-            ```
-
-        !!! note "Yarn users"
-
-            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
-
-            ```shell
-            yarn add apache-arrow
-            ```
+      ```shell
+      npm install vectordb
+      ```
 
 === "Rust"
 
@@ -107,21 +58,14 @@ recommend switching to stable releases.
       pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
       ```
 
-=== "Typescript[^1]"
+=== "Typescript"
 
-    === "@lancedb/lancedb"
-
-        ```shell
-        npm install @lancedb/lancedb@preview
-        ```
-    === "vectordb (deprecated)"
-
-        ```shell
-        npm install vectordb@preview
-        ```
+      ```shell
+      npm install vectordb@preview
+      ```
 
 === "Rust"
-
+    
     We don't push preview releases to crates.io, but you can referent the tag
     in GitHub within your Cargo dependencies:
 
@@ -141,22 +85,31 @@ recommend switching to stable releases.
     --8<-- "python/python/tests/docs/test_basic.py:connect_async"
     ```
 
-=== "Typescript[^1]"
+    !!! note "Asynchronous Python API"
 
-    === "@lancedb/lancedb"
+        The asynchronous Python API is new and has some slight differences compared
+        to the synchronous API.  Feel free to start using the asynchronous version.
+        Once all features have migrated we will start to move the synchronous API to
+        use the same syntax as the asynchronous API.  To help with this migration we
+        have created a [migration guide](migration.md) detailing the differences.
 
-        ```typescript
-        import * as lancedb from "@lancedb/lancedb";
-        import * as arrow from "apache-arrow";
+=== "Typescript"
 
-        --8<-- "nodejs/examples/basic.test.ts:connect"
-        ```
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:import"
 
-    === "vectordb (deprecated)"
+    --8<-- "docs/src/basic_legacy.ts:open_db"
+    ```
 
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:open_db"
-        ```
+    !!! note "`@lancedb/lancedb` vs. `vectordb`"
+
+        The Javascript SDK was originally released as `vectordb`.  In an effort to
+        reduce maintenance we are aligning our SDKs.  The new, aligned, Javascript
+        API is being released as `lancedb`.  If you are starting new work we encourage
+        you to try out `lancedb`.  Once the new API is feature complete we will begin
+        slowly deprecating `vectordb` in favor of `lancedb`. There is a
+        [migration guide](migration.md) detailing the differences which will assist
+        you in this process.
 
 === "Rust"
 
@@ -199,23 +152,15 @@ table.
     --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
     ```
 
-=== "Typescript[^1]"
+=== "Typescript"
 
-    === "@lancedb/lancedb"
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_table"
+    ```
 
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:create_table"
-        ```
-
-        If the table already exists, LanceDB will raise an error by default.
-        If you want to overwrite the table, you can pass in `mode:"overwrite"`
-        to the `createTable` function.
+    If the table already exists, LanceDB will raise an error by default.
+    If you want to overwrite the table, you can pass in `mode="overwrite"`
+    to the `createTable` function.
 
 === "Rust"
 
@@ -255,19 +200,11 @@ similar to a `CREATE TABLE` statement in SQL.
     !!! note "You can define schema in Pydantic"
         LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
 
-=== "Typescript[^1]"
+=== "Typescript"
 
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
-        ```
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+    ```
 
 === "Rust"
 
@@ -286,19 +223,11 @@ Once created, you can open a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
     ```
 
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:open_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        const tbl = await db.openTable("myTable");
-        ```
+=== "Typescript"
 
+    ```typescript
+    const tbl = await db.openTable("myTable");
+    ```
 
 === "Rust"
 
@@ -315,18 +244,11 @@ If you forget the name of your table, you can always get a listing of all table
     --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
     ```
 
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
+=== "Javascript"
 
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:table_names"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        console.log(await db.tableNames());
-        ```
+    ```javascript
+    console.log(await db.tableNames());
+    ```
 
 === "Rust"
 
@@ -345,18 +267,11 @@ After a table has been created, you can always add more data to it as follows:
     --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
     ```
 
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
+=== "Typescript"
 
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:add_data"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:add"
-        ```
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:add"
+    ```
 
 === "Rust"
 
@@ -377,18 +292,11 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
 
     This returns a pandas DataFrame with the results.
 
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
+=== "Typescript"
 
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:vector_search"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:search"
-        ```
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:search"
+    ```
 
 === "Rust"
 
@@ -417,18 +325,11 @@ LanceDB allows you to create an ANN index on a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
     ```
 
-=== "Typescript[^1]"
-    === "@lancedb/lancedb"
+=== "Typescript"
 
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_index"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```{.typescript .ignore}
-        --8<-- "docs/src/basic_legacy.ts:create_index"
-        ```
+    ```{.typescript .ignore}
+    --8<-- "docs/src/basic_legacy.ts:create_index"
+    ```
 
 === "Rust"
 
@@ -456,19 +357,11 @@ This can delete any number of rows that match the filter.
     --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
     ```
 
-=== "Typescript[^1]"
+=== "Typescript"
 
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:delete_rows"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:delete"
-        ```
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:delete"
+    ```
 
 === "Rust"
 
@@ -485,15 +378,9 @@ simple or complex as needed. To see what expressions are supported, see the
 
       Read more: [lancedb.table.Table.delete][]
 
-=== "Typescript[^1]"
+=== "Javascript"
 
-    === "@lancedb/lancedb"
-
-        Read more: [lancedb.Table.delete](javascript/interfaces/Table.md#delete)
-
-    === "vectordb (deprecated)"
-
-        Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
 === "Rust"
 
@@ -505,31 +392,23 @@ Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
 
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-    ```
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      ```
 
-    This permanently removes the table and is not recoverable, unlike deleting rows.
-    By default, if the table does not exist an exception is raised. To suppress this,
-    you can pass in `ignore_missing=True`.
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
 
-=== "Typescript[^1]"
+=== "Typescript"
 
-    === "@lancedb/lancedb"
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
+      ```
 
-        ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:drop_table"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```typescript
-        --8<-- "docs/src/basic_legacy.ts:drop_table"
-        ```
-
-        This permanently removes the table and is not recoverable, unlike deleting rows.
-        If the table does not exist an exception is raised.
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised.
 
 === "Rust"
 
@@ -537,6 +416,19 @@ Use the `drop_table()` method on the database to remove a table.
     --8<-- "rust/lancedb/examples/simple.rs:drop_table"
     ```
 
+!!! note "Bundling `vectordb` apps with Webpack"
+
+    If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+    ```javascript
+    /** @type {import('next').NextConfig} */
+    module.exports = ({
+    webpack(config) {
+        config.externals.push({ vectordb: 'vectordb' })
+        return config;
+    }
+    })
+    ```
 
 ## Using the Embedding API
 You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
@@ -548,23 +440,7 @@ You can use the embedding API when working with embedding models. It automatical
     --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
     ```
 
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.test.ts:openai_embeddings"
-        ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/openai.rs:imports"
-    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/index.md).
+Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
 
 
 ## What's next
@@ -572,5 +448,3 @@ Learn about using the existing integrations and creating custom embedding functi
 This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
 
 If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.
-
-[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/basic_legacy.ts

@@ -1,14 +1,6 @@
 // --8<-- [start:import]
 import * as lancedb from "vectordb";
-import {
-  Schema,
-  Field,
-  Float32,
-  FixedSizeList,
-  Int32,
-  Float16,
-} from "apache-arrow";
-import * as arrow from "apache-arrow";
+import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
 // --8<-- [end:import]
 import * as fs from "fs";
 import { Table as ArrowTable, Utf8 } from "apache-arrow";
@@ -28,33 +20,9 @@ const example = async () => {
       { vector: [3.1, 4.1], item: "foo", price: 10.0 },
       { vector: [5.9, 26.5], item: "bar", price: 20.0 },
     ],
-    { writeMode: lancedb.WriteMode.Overwrite },
+    { writeMode: lancedb.WriteMode.Overwrite }
   );
   // --8<-- [end:create_table]
-  {
-    // --8<-- [start:create_table_with_schema]
-    const schema = new arrow.Schema([
-      new arrow.Field(
-        "vector",
-        new arrow.FixedSizeList(
-          2,
-          new arrow.Field("item", new arrow.Float32(), true),
-        ),
-      ),
-      new arrow.Field("item", new arrow.Utf8(), true),
-      new arrow.Field("price", new arrow.Float32(), true),
-    ]);
-    const data = [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ];
-    const tbl = await db.createTable({
-      name: "myTableWithSchema",
-      data,
-      schema,
-    });
-    // --8<-- [end:create_table_with_schema]
-  }
 
   // --8<-- [start:add]
   const newData = Array.from({ length: 500 }, (_, i) => ({
@@ -74,35 +42,33 @@ const example = async () => {
   // --8<-- [end:create_index]
 
   // --8<-- [start:create_empty_table]
-  const schema = new arrow.Schema([
-    new arrow.Field("id", new arrow.Int32()),
-    new arrow.Field("name", new arrow.Utf8()),
+  const schema = new Schema([
+    new Field("id", new Int32()),
+    new Field("name", new Utf8()),
   ]);
-
   const empty_tbl = await db.createTable({ name: "empty_table", schema });
   // --8<-- [end:create_empty_table]
-  {
-    // --8<-- [start:create_f16_table]
-    const dim = 16;
-    const total = 10;
-    const schema = new Schema([
-      new Field("id", new Int32()),
+
+  // --8<-- [start:create_f16_table]
+  const dim = 16
+  const total = 10
+  const f16_schema = new Schema([
+      new Field('id', new Int32()),
       new Field(
-        "vector",
-        new FixedSizeList(dim, new Field("item", new Float16(), true)),
-        false,
-      ),
-    ]);
-    const data = lancedb.makeArrowTable(
+        'vector',
+        new FixedSizeList(dim, new Field('item', new Float16(), true)),
+        false
+      )
+    ])
+  const data = lancedb.makeArrowTable(
       Array.from(Array(total), (_, i) => ({
         id: i,
-        vector: Array.from(Array(dim), Math.random),
+        vector: Array.from(Array(dim), Math.random)
       })),
-      { schema },
-    );
-    const table = await db.createTable("f16_tbl", data);
-    // --8<-- [end:create_f16_table]
-  }
+      { f16_schema }
+    )
+  const table = await db.createTable('f16_tbl', data)
+  // --8<-- [end:create_f16_table]
 
   // --8<-- [start:search]
   const query = await tbl.search([100, 100]).limit(2).execute();

docs/src/cloud/cloud_faq.md

@@ -1,34 +0,0 @@
-This section provides answers to the most common questions asked about LanceDB Cloud. By following these guidelines, you can ensure a smooth, performant experience with LanceDB Cloud.
-
-### Should I reuse the database connection?
-Yes! It is recommended to establish a single database connection and maintain it throughout your interaction with the tables within. 
-
-LanceDB uses HTTP connections to communicate with the servers. By re-using the Connection object, you avoid the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
-
-### Should I re-use the `Table` object?
-`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the **latest version** of the data. 
-
-### What should I do if I need to search for rows by `id`?
-LanceDB Cloud currently does not support an ID or primary key column. You are recommended to add a 
-user-defined ID column. To significantly improve the query performance with SQL causes, a scalar BITMAP/BTREE index should be created on this column. 
-
-### What are the vector indexing types supported by LanceDB Cloud?
-We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff between query latency and query quality.
-
-### When I add new rows to a table, do I need to manually update the index?
-No! LanceDB Cloud triggers an asynchronous background job to index the new vectors.
-
-Even though indexing is asynchronous, your vectors will still be immediately searchable. LanceDB uses brute-force search to search over unindexed rows. This makes you new data is immediately available, but does increase latency temporarily. To disable the brute-force part of search, set the `fast_search` flag in your query to `true`.
-
-### Do I need to reindex the whole dataset if only a small portion of the data is deleted or updated?
-No! Similar to adding data to the table, LanceDB Cloud triggers an asynchronous background job to update the existing indices. Therefore, no action is needed from users and there is absolutely no 
-downtime expected.
-
-### How do I know whether an index has been created?
-While index creation in LanceDB Cloud is generally fast, querying immediately after a `create_index` call may result in errors. It's recommended to use `list_indices` to verify index creation before querying.
-
-### Why is my query latency higher than expected?
-Multiple factors can impact query latency. To reduce query latency, consider the following:
-- Send pre-warm queries: send a few queries to warm up the cache before an actual user query.
-- Check network latency: LanceDB Cloud is hosted in AWS `us-east-1` region. It is recommended to run queries from an EC2 instance that is in the same region.
-- Create scalar indices: If you are filtering on metadata, it is recommended to create scalar indices on those columns. This will speedup searches with metadata filtering. See [here](../guides/scalar_index.md) for more details on creating a scalar index.

docs/src/cloud/rest.md

@@ -1 +0,0 @@
-!!swagger ../../openapi.yml!!

docs/src/concepts/index_hnsw.md

@@ -1,99 +0,0 @@
-
-# Understanding HNSW index
-
-Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. HNSW is one of the most accurate and fastest Approximate Nearest Neighbour search algorithms, It’s beneficial in high-dimensional spaces where finding the same nearest neighbor would be too slow and costly
-
-[Jump to usage](#usage)
-There are three main types of ANN search algorithms:
-
-* **Tree-based search algorithms**: Use a tree structure to organize and store data points.
-* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
-* **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. 
-
-HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below.  
-HNSW also combines this with the ideas behind a classic 1-dimensional search data structure: the skip list.
-
-## k-Nearest Neighbor Graphs and k-approximate Nearest neighbor Graphs
-The k-nearest neighbor graph actually predates its use for ANN search. Its construction is quite simple:
-
-* Each vector in the dataset is given an associated vertex.
-* Each vertex has outgoing edges to its k nearest neighbors. That is, the k closest other vertices by Euclidean distance between the two corresponding vectors. This can be thought of as a "friend list" for the vertex.
-* For some applications (including nearest-neighbor search), the incoming edges are also added.
-
-Eventually, it was realized that the following greedy search method over such a graph typically results in good approximate nearest neighbors:
-
-* Given a query vector, start at some fixed "entry point" vertex (e.g. the approximate center node).
-* Look at that vertex's neighbors. If any of them are closer to the query vector than the current vertex, then move to that vertex.
-* Repeat until a local optimum is found.
-
-The above algorithm also generalizes to e.g. top 10 approximate nearest neighbors.
-
-Computing a k-nearest neighbor graph is actually quite slow, taking quadratic time in the dataset size. It was quickly realized that near-identical performance can be achieved using a k-approximate nearest neighbor graph. That is, instead of obtaining the k-nearest neighbors for each vertex, an approximate nearest neighbor search data structure is used to build much faster.  
-In fact, another data structure is not needed: This can be done "incrementally".
-That is, if you start with a k-ANN graph for n-1 vertices, you can extend it to a k-ANN graph for n vertices as well by using the graph to obtain the k-ANN for the new vertex.
-
-One downside of k-NN and k-ANN graphs alone is that one must typically build them with a large value of k to get decent results, resulting in a large index.
-
-
-## HNSW: Hierarchical Navigable Small Worlds
-
-HNSW builds on k-ANN in two main ways:
-
-* Instead of getting the k-approximate nearest neighbors for a large value of k, it sparsifies the k-ANN graph using a carefully chosen "edge pruning" heuristic, allowing for the number of edges per vertex to be limited to a relatively small constant.
-* The "entry point" vertex is chosen dynamically using a recursively constructed data structure on a subset of the data, similarly to a skip list.
-
-This recursive structure can be thought of as separating into layers:
-
-* At the bottom-most layer, an k-ANN graph on the whole dataset is present.
-* At the second layer, a k-ANN graph on a fraction of the dataset (e.g. 10%) is present.
-* At the Lth layer, a k-ANN graph is present. It is over a (constant) fraction (e.g. 10%) of the vectors/vertices present in the L-1th layer.
-
-Then the greedy search routine operates as follows:
-
-* At the top layer (using an arbitrary vertex as an entry point), use the greedy local search routine on the k-ANN graph to get an approximate nearest neighbor at that layer.
-* Using the approximate nearest neighbor found in the previous layer as an entry point, find an approximate nearest neighbor in the next layer with the same method.
-* Repeat until the bottom-most layer is reached. Then use the entry point to find multiple nearest neighbors (e.g. top 10).
-
-
-## Usage
-
-There are three key parameters to set when constructing an HNSW index:
-
-* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
-* `m`: The number of neighbors to select for each vector in the HNSW graph.
-* `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph.
-
-
-We can combine the above concepts to understand how to build and query an HNSW index in LanceDB.
-
-### Construct index
-
-```python
-import lancedb
-import numpy as np
-uri = "/tmp/lancedb"
-db = lancedb.connect(uri)
-
-# Create 10,000 sample vectors
-data = [
-    {"vector": row, "item": f"item {i}"}
-    for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))
-]
-
-# Add the vectors to a table
-tbl = db.create_table("my_vectors", data=data)
-
-# Create and train the HNSW index for a 1536-dimensional vector
-# Make sure you have enough data in the table for an effective training step
-tbl.create_index(index_type=IVF_HNSW_SQ)
-
-```
-
-### Query the index
-
-```python
-# Search using a random 1536-dimensional embedding
-tbl.search(np.random.random((1536))) \
-    .limit(2) \
-    .to_pandas()
-```

docs/src/concepts/index_ivfpq.md

@@ -58,10 +58,8 @@ In Python, the index can be created as follows:
 # Make sure you have enough data in the table for an effective training step
 tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
 ```
-!!! note
-    `num_partitions`=256 and `num_sub_vectors`=96 does not work for every dataset. Those values needs to be adjusted for your particular dataset.
 
-The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See [here](../ann_indexes.md/#how-to-choose-num_partitions-and-num_sub_vectors-for-ivf_pq-index) for best practices on choosing these parameters.
+The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See the [FAQs](#faq) below for best practices on choosing these parameters.
 
 
 ### Query the index

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md

@@ -1,67 +0,0 @@
-# Imagebind embeddings
-We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
-
-This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"imagebind_huge"` | Name of the model. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
-
-Below is an example demonstrating how the API works:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry().get("imagebind").create()
-
-class ImageBindModel(LanceModel):
-    text: str
-    image_uri: str = func.SourceField()
-    audio_path: str
-    vector: Vector(func.ndims()) = func.VectorField()
-
-# add locally accessible image paths
-text_list=["A dog.", "A car", "A bird"]
-image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
-audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
-
-# Load data
-inputs = [
-    {"text": a, "audio_path": b, "image_uri": c}
-    for a, b, c in zip(text_list, audio_paths, image_paths)
-]
-
-#create table and add data
-table = db.create_table("img_bind", schema=ImageBindModel)
-table.add(inputs)
-```
-
-Now, we can search using any modality:
-
-#### image search
-```python
-query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
-actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "dog")
-```
-#### audio search
-
-```python
-query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
-actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "car")
-```
-#### Text search
-You can add any input query and fetch the result as follows:
-```python
-query = "an animal which flies and tweets" 
-actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "bird")
-```
-
-If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md

@@ -1,51 +0,0 @@
-# Jina Embeddings : Multimodal
-
-Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
-under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
-
-Usage Example:
-
-```python
-    import os
-    import requests
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-    import pandas as pd
-
-    os.environ['JINA_API_KEY'] = 'jina_*'
-
-    db = lancedb.connect("~/.lancedb")
-    func = get_registry().get("jina").create()
-
-
-    class Images(LanceModel):
-        label: str
-        image_uri: str = func.SourceField()  # image uri as the source
-        image_bytes: bytes = func.SourceField()  # image bytes as the source
-        vector: Vector(func.ndims()) = func.VectorField()  # vector column
-        vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
-
-
-    table = db.create_table("images", schema=Images)
-    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
-    uris = [
-        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
-        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
-        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
-        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
-        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
-        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
-    ]
-    # get each uri as bytes
-    image_bytes = [requests.get(uri).content for uri in uris]
-    table.add(
-      pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
-    )
-```

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md

@@ -1,82 +0,0 @@
-# OpenClip embeddings
-We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"ViT-B-32"` | The name of the model. |
-| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `batch_size` | `int` | `64` | The number of images to process in a batch. |
-| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
-
-This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
-
-!!! info
-    LanceDB supports ingesting images directly from accessible links.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry().get("open-clip").create()
-
-class Images(LanceModel):
-    label: str
-    image_uri: str = func.SourceField() # image uri as the source
-    image_bytes: bytes = func.SourceField() # image bytes as the source
-    vector: Vector(func.ndims()) = func.VectorField() # vector column 
-    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
-
-table = db.create_table("images", schema=Images)
-labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
-uris = [
-    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
-    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
-    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
-    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
-    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
-    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
-]
-# get each uri as bytes
-image_bytes = [requests.get(uri).content for uri in uris]
-table.add(
-    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
-)
-```
-Now we can search using text from both the default vector column and the custom vector column
-```python
-
-# text search
-actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
-print(actual.label) # prints "dog"
-
-frombytes = (
-    table.search("man's best friend", vector_column_name="vec_from_bytes")
-    .limit(1)
-    .to_pydantic(Images)[0]
-)
-print(frombytes.label)
-
-```
-
-Because we're using a multi-modal embedding function, we can also search using images
-
-```python
-# image search
-query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
-image_bytes = requests.get(query_image_uri).content
-query_image = Image.open(io.BytesIO(image_bytes))
-actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
-print(actual.label == "dog")
-
-# image search using a custom vector column
-other = (
-    table.search(query_image, vector_column_name="vec_from_bytes")
-    .limit(1)
-    .to_pydantic(Images)[0]
-)
-print(actual.label)
-
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md

@@ -1,51 +0,0 @@
-# AWS Bedrock Text Embedding Functions
-
-AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
-You can do so by using `awscli` and also add your session_token:
-```shell
-aws configure
-aws configure set aws_session_token "<your_session_token>"
-```
-to ensure that the credentials are set up correctly, you can run the following command:
-```shell
-aws sts get-caller-identity
-```
-
-Supported Embedding modelIDs are:
-* `amazon.titan-embed-text-v1`
-* `cohere.embed-english-v3`
-* `cohere.embed-multilingual-v3`
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
-| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
-| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
-| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
-| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
-| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
-| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
-
-Usage Example:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-import pandas as pd
-
-model = get_registry().get("bedrock-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("tmp_path")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md

@@ -1,63 +0,0 @@
-# Cohere Embeddings
-
-Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
-You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
-
-Supported models are:
-
-- embed-english-v3.0
-- embed-multilingual-v3.0
-- embed-english-light-v3.0
-- embed-multilingual-light-v3.0
-- embed-english-v2.0
-- embed-english-light-v2.0
-- embed-multilingual-v2.0
-
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|--------|---------|
-| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
-| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
-| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
-
-Cohere supports following input types:
-
-| Input Type               | Description                          |
-|-------------------------|---------------------------------------|
-| "`search_document`"     | Used for embeddings stored in a vector|
-|                         | database for search use-cases.        |
-| "`search_query`"        | Used for embeddings of search queries |
-|                         | run against a vector DB               |
-| "`semantic_similarity`" | Specifies the given text will be used |
-|                         | for Semantic Textual Similarity (STS) |
-| "`classification`"      | Used for embeddings passed through a  |
-|                         | text classifier.                      |
-| "`clustering`"          | Used for the embeddings run through a |
-|                         | clustering algorithm                  |
-
-Usage Example:
-    
-```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    cohere = EmbeddingFunctionRegistry
-        .get_instance()
-        .get("cohere")
-        .create(name="embed-multilingual-v2.0")
-
-    class TextModel(LanceModel):
-        text: str = cohere.SourceField()
-        vector: Vector(cohere.ndims()) =  cohere.VectorField()
-
-    data = [ { "text": "hello world" },
-            { "text": "goodbye world" }]
-
-    db = lancedb.connect("~/.lancedb")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md

@@ -1,35 +0,0 @@
-# Gemini Embeddings
-With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
-The Gemini Embedding Model API supports various task types:
-
-| Task Type               | Description                                                                                                                                                |
-|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
-| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically proided by Embeddings API |
-| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
-| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
-| "`clusering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
-
-
-Usage Example:
-
-```python
-import lancedb
-import pandas as pd
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-
-model = get_registry().get("gemini-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md

@@ -1,24 +0,0 @@
-# Huggingface embedding models
-We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. 
-
-Example usage - 
-```python
-import lancedb
-import pandas as pd
-
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-model = get_registry().get("huggingface").create(name='facebook/bart-base')
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
-table = db.create_table("greets", schema=Words)
-table.add(df)
-query = "old greeting"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md

@@ -1,75 +0,0 @@
-# IBM watsonx.ai Embeddings
-
-Generate text embeddings using IBM's watsonx.ai platform.
-
-## Supported Models
-
-You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are:
-
-- `ibm/slate-125m-english-rtrvr`
-- `ibm/slate-30m-english-rtrvr`
-- `sentence-transformers/all-minilm-l12-v2`
-- `intfloat/multilingual-e5-large`
-
-## Parameters
-
-The following parameters can be passed to the `create` method:
-
-| Parameter  | Type     | Default Value                    | Description                                               |
-|------------|----------|----------------------------------|-----------------------------------------------------------|
-| name       | str      | "ibm/slate-125m-english-rtrvr"   | The model ID of the watsonx.ai model to use               |
-| api_key    | str      | None                             | Optional IBM Cloud API key (or set `WATSONX_API_KEY`)     |
-| project_id | str      | None                             | Optional watsonx project ID (or set `WATSONX_PROJECT_ID`) |
-| url        | str      | None                             | Optional custom URL for the watsonx.ai instance           |
-| params     | dict     | None                             | Optional additional parameters for the embedding model    |
-
-## Usage Example
-
-First, the watsonx.ai library is an optional dependency, so must be installed seperately:
-
-```
-pip install ibm-watsonx-ai
-```
-
-Optionally set environment variables (if not passing credentials to `create` directly):
-
-```sh
-export WATSONX_API_KEY="YOUR_WATSONX_API_KEY"
-export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID"
-```
-
-```python
-import os
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import EmbeddingFunctionRegistry
-
-watsonx_embed = EmbeddingFunctionRegistry
-  .get_instance()
-  .get("watsonx")
-  .create(
-    name="ibm/slate-125m-english-rtrvr",
-    # Uncomment and set these if not using environment variables
-    # api_key="your_api_key_here",
-    # project_id="your_project_id_here",
-    # url="your_watsonx_url_here",
-    # params={...},
-  )
-
-class TextModel(LanceModel):
-    text: str = watsonx_embed.SourceField()
-    vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"},
-]
-
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
-
-tbl.add(data)
-
-rs = tbl.search("hello").limit(1).to_pandas()
-print(rs)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md

@@ -1,50 +0,0 @@
-# Instructor Embeddings
-[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
-
-If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
-
-!!! info
-    Represent the `domain` `text_type` for `task_objective`:
-
-    * `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
-    * `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
-    * `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
-
-More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
-
-| Argument | Type | Default | Description |
-|---|---|---|---|
-| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
-| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
-| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
-| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
-| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
-| `quantize` | `bool` | `False` | Whether to quantize the model |
-| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
-| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
-
-
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
-
-instructor = get_registry().get("instructor").create(
-                            source_instruction="represent the docuement for retreival",
-                            query_instruction="represent the document for retreiving the most similar documents"
-                            )
-
-class Schema(LanceModel):
-    vector: Vector(instructor.ndims()) = instructor.VectorField()
-    text: str = instructor.SourceField()
-
-db = lancedb.connect("~/.lancedb")
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-
-texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
-        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
-        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
-
-tbl.add(texts)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md

@@ -1,39 +0,0 @@
-# Jina Embeddings
-
-Jina embeddings are used to generate embeddings for text and image data.
-You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
-
-You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
-
-Usage Example:
-
-```python
-    import os
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    os.environ['JINA_API_KEY'] = 'jina_*'
-
-    jina_embed = EmbeddingFunctionRegistry.get_instance().get("jina").create(name="jina-embeddings-v2-base-en")
-
-
-    class TextModel(LanceModel):
-        text: str = jina_embed.SourceField()
-        vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
-
-
-    data = [{"text": "hello world"},
-            {"text": "goodbye world"}]
-
-    db = lancedb.connect("~/.lancedb-2")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md

@@ -1,37 +0,0 @@
-# Ollama embeddings
-
-Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
-
-- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
-- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
-
-| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
-|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
-| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
-| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
-| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
-| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
-| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("ollama").create(name="nomic-embed-text")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md

@@ -1,35 +0,0 @@
-# OpenAI embeddings
-
-LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
-| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
-| `use_azure` | bool | `False` | Set true to use Azure OpenAPI SDK |
-
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md

@@ -1,174 +0,0 @@
-# Sentence transformers
-Allows you to set parameters when registering a `sentence-transformers` object.
-
-!!! info
-    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
-| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
-| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
-| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository |
-
-
-??? "Check out available sentence-transformer models here!"
-    ```markdown
-    - sentence-transformers/all-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-mpnet-base-v2
-    - sentence-transformers/gtr-t5-base
-    - sentence-transformers/LaBSE
-    - sentence-transformers/all-MiniLM-L6-v2
-    - sentence-transformers/bert-base-nli-max-tokens
-    - sentence-transformers/bert-base-nli-mean-tokens
-    - sentence-transformers/bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
-    - sentence-transformers/bert-large-nli-cls-token
-    - sentence-transformers/bert-large-nli-max-tokens
-    - sentence-transformers/bert-large-nli-mean-tokens
-    - sentence-transformers/bert-large-nli-stsb-mean-tokens
-    - sentence-transformers/distilbert-base-nli-max-tokens
-    - sentence-transformers/distilbert-base-nli-mean-tokens
-    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
-    - sentence-transformers/distilroberta-base-msmarco-v1
-    - sentence-transformers/distilroberta-base-msmarco-v2
-    - sentence-transformers/nli-bert-base-cls-pooling
-    - sentence-transformers/nli-bert-base-max-pooling
-    - sentence-transformers/nli-bert-base
-    - sentence-transformers/nli-bert-large-cls-pooling
-    - sentence-transformers/nli-bert-large-max-pooling
-    - sentence-transformers/nli-bert-large
-    - sentence-transformers/nli-distilbert-base-max-pooling
-    - sentence-transformers/nli-distilbert-base
-    - sentence-transformers/nli-roberta-base
-    - sentence-transformers/nli-roberta-large
-    - sentence-transformers/roberta-base-nli-mean-tokens
-    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
-    - sentence-transformers/roberta-large-nli-mean-tokens
-    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
-    - sentence-transformers/stsb-bert-base
-    - sentence-transformers/stsb-bert-large
-    - sentence-transformers/stsb-distilbert-base
-    - sentence-transformers/stsb-roberta-base
-    - sentence-transformers/stsb-roberta-large
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
-    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
-    - sentence-transformers/bert-base-nli-cls-token
-    - sentence-transformers/all-distilroberta-v1
-    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
-    - sentence-transformers/multi-qa-distilbert-cos-v1
-    - sentence-transformers/multi-qa-distilbert-dot-v1
-    - sentence-transformers/multi-qa-mpnet-base-cos-v1
-    - sentence-transformers/multi-qa-mpnet-base-dot-v1
-    - sentence-transformers/nli-distilroberta-base-v2
-    - sentence-transformers/all-MiniLM-L6-v1
-    - sentence-transformers/all-mpnet-base-v1
-    - sentence-transformers/all-mpnet-base-v2
-    - sentence-transformers/all-roberta-large-v1
-    - sentence-transformers/allenai-specter
-    - sentence-transformers/average_word_embeddings_glove.6B.300d
-    - sentence-transformers/average_word_embeddings_glove.840B.300d
-    - sentence-transformers/average_word_embeddings_komninos
-    - sentence-transformers/average_word_embeddings_levy_dependency
-    - sentence-transformers/clip-ViT-B-32-multilingual-v1
-    - sentence-transformers/clip-ViT-B-32
-    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
-    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
-    - sentence-transformers/distilroberta-base-paraphrase-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v2
-    - sentence-transformers/distiluse-base-multilingual-cased
-    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
-    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
-    - sentence-transformers/gtr-t5-large
-    - sentence-transformers/gtr-t5-xl
-    - sentence-transformers/gtr-t5-xxl
-    - sentence-transformers/msmarco-MiniLM-L-12-v3
-    - sentence-transformers/msmarco-MiniLM-L-6-v3
-    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
-    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
-    - sentence-transformers/msmarco-bert-base-dot-v5
-    - sentence-transformers/msmarco-bert-co-condensor
-    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
-    - sentence-transformers/msmarco-distilbert-base-tas-b
-    - sentence-transformers/msmarco-distilbert-base-v2
-    - sentence-transformers/msmarco-distilbert-base-v3
-    - sentence-transformers/msmarco-distilbert-base-v4
-    - sentence-transformers/msmarco-distilbert-cos-v5
-    - sentence-transformers/msmarco-distilbert-dot-v5
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
-    - sentence-transformers/msmarco-distilroberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-ance-firstp
-    - sentence-transformers/msmarco-roberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-v3
-    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-    - sentence-transformers/nli-mpnet-base-v2
-    - sentence-transformers/nli-roberta-base-v2
-    - sentence-transformers/nq-distilbert-base-v1
-    - sentence-transformers/paraphrase-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-MiniLM-L3-v2
-    - sentence-transformers/paraphrase-MiniLM-L6-v2
-    - sentence-transformers/paraphrase-TinyBERT-L6-v2
-    - sentence-transformers/paraphrase-albert-base-v2
-    - sentence-transformers/paraphrase-albert-small-v2
-    - sentence-transformers/paraphrase-distilroberta-base-v1
-    - sentence-transformers/paraphrase-distilroberta-base-v2
-    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
-    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
-    - sentence-transformers/quora-distilbert-base
-    - sentence-transformers/quora-distilbert-multilingual
-    - sentence-transformers/sentence-t5-base
-    - sentence-transformers/sentence-t5-large
-    - sentence-transformers/sentence-t5-xxl
-    - sentence-transformers/sentence-t5-xl
-    - sentence-transformers/stsb-distilroberta-base-v2
-    - sentence-transformers/stsb-mpnet-base-v2
-    - sentence-transformers/stsb-roberta-base-v2
-    - sentence-transformers/stsb-xlm-r-multilingual
-    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
-    - sentence-transformers/clip-ViT-L-14
-    - sentence-transformers/clip-ViT-B-16
-    - sentence-transformers/use-cmlm-multilingual
-    - sentence-transformers/all-MiniLM-L12-v1
-    ```
-
-!!! info
-    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
-    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
-
-!!! note "BAAI Embeddings example"
-    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-    )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
-

docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md

@@ -1,51 +0,0 @@
-# VoyageAI Embeddings
-
-Voyage AI provides cutting-edge embedding and rerankers.
-
-
-Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
-You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API.
-
-Supported models are:
-
-- voyage-3
-- voyage-3-lite
-- voyage-finance-2
-- voyage-multilingual-2
-- voyage-law-2
-- voyage-code-2
-
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|--------|---------|
-| `name` | `str` | `None` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
-| `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. |
-| `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. |
-
-
-Usage Example:
-    
-```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import EmbeddingFunctionRegistry
-
-    voyageai = EmbeddingFunctionRegistry
-        .get_instance()
-        .get("voyageai")
-        .create(name="voyage-3")
-
-    class TextModel(LanceModel):
-        text: str = voyageai.SourceField()
-        vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
-
-    data = [ { "text": "hello world" },
-            { "text": "goodbye world" }]
-
-    db = lancedb.connect("~/.lancedb")
-    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-    tbl.add(data)
-```

docs/src/embeddings/custom_embedding_function.md

@@ -15,226 +15,198 @@ There is another optional layer of abstraction available: `TextEmbeddingFunction
 
 Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
 
+```python
+from lancedb.embeddings import register
+from lancedb.util import attempt_import_or_raise
 
-=== "Python"
+@register("sentence-transformers")
+class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+    name: str = "all-MiniLM-L6-v2"
+    # set more default instance vars like device, etc.
 
-    ```python
-    from lancedb.embeddings import register
-    from lancedb.util import attempt_import_or_raise
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._ndims = None
+    
+    def generate_embeddings(self, texts):
+        return self._embedding_model().encode(list(texts), ...).tolist()
 
-    @register("sentence-transformers")
-    class SentenceTransformerEmbeddings(TextEmbeddingFunction):
-        name: str = "all-MiniLM-L6-v2"
-        # set more default instance vars like device, etc.
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = len(self.generate_embeddings("foo")[0])
+        return self._ndims
 
-        def __init__(self, **kwargs):
-            super().__init__(**kwargs)
-            self._ndims = None
+    @cached(cache={}) 
+    def _embedding_model(self):
+        return sentence_transformers.SentenceTransformer(name)
+```
 
-        def generate_embeddings(self, texts):
-            return self._embedding_model().encode(list(texts), ...).tolist()
-
-        def ndims(self):
-            if self._ndims is None:
-                self._ndims = len(self.generate_embeddings("foo")[0])
-            return self._ndims
-
-        @cached(cache={})
-        def _embedding_model(self):
-            return sentence_transformers.SentenceTransformer(name)
-    ```
-
-=== "TypeScript"
-
-    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:imports"
-
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:embedding_impl"
-    ```
-
-
-This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and default settings.
+This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and defaul settings.
 
 Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
 
-=== "Python"
+```python
+from lancedb.pydantic import LanceModel, Vector
 
-    ```python
-    from lancedb.pydantic import LanceModel, Vector
+registry = EmbeddingFunctionRegistry.get_instance()
+stransformer = registry.get("sentence-transformers").create()
 
-    registry = EmbeddingFunctionRegistry.get_instance()
-    stransformer = registry.get("sentence-transformers").create()
+class TextModelSchema(LanceModel):
+    vector: Vector(stransformer.ndims) = stransformer.VectorField()
+    text: str = stransformer.SourceField()
 
-    class TextModelSchema(LanceModel):
-        vector: Vector(stransformer.ndims) = stransformer.VectorField()
-        text: str = stransformer.SourceField()
+tbl = db.create_table("table", schema=TextModelSchema)
 
-    tbl = db.create_table("table", schema=TextModelSchema)
+tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
+result = tbl.search("world").limit(5)
+```
 
-    tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
-    result = tbl.search("world").limit(5)
-    ```
+NOTE:
 
-=== "TypeScript"
-
-    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:call_custom_function"
-    ```
-
-!!! note
-
-    You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
+You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
 
 ## Multi-modal embedding function example
-You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support.
+You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support. LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
 
-=== "Python"
+```python
+@register("open-clip")
+class OpenClipEmbeddings(EmbeddingFunction):
+    name: str = "ViT-B-32"
+    pretrained: str = "laion2b_s34b_b79k"
+    device: str = "cpu"
+    batch_size: int = 64
+    normalize: bool = True
+    _model = PrivateAttr()
+    _preprocess = PrivateAttr()
+    _tokenizer = PrivateAttr()
 
-    LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            self.name, pretrained=self.pretrained
+        )
+        model.to(self.device)
+        self._model, self._preprocess = model, preprocess
+        self._tokenizer = open_clip.get_tokenizer(self.name)
+        self._ndims = None
 
-    ```python
-    @register("open-clip")
-    class OpenClipEmbeddings(EmbeddingFunction):
-        name: str = "ViT-B-32"
-        pretrained: str = "laion2b_s34b_b79k"
-        device: str = "cpu"
-        batch_size: int = 64
-        normalize: bool = True
-        _model = PrivateAttr()
-        _preprocess = PrivateAttr()
-        _tokenizer = PrivateAttr()
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = self.generate_text_embeddings("foo").shape[0]
+        return self._ndims
 
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
-            model, _, preprocess = open_clip.create_model_and_transforms(
-                self.name, pretrained=self.pretrained
-            )
-            model.to(self.device)
-            self._model, self._preprocess = model, preprocess
-            self._tokenizer = open_clip.get_tokenizer(self.name)
-            self._ndims = None
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
 
-        def ndims(self):
-            if self._ndims is None:
-                self._ndims = self.generate_text_embeddings("foo").shape[0]
-            return self._ndims
-
-        def compute_query_embeddings(
-            self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
-        ) -> List[np.ndarray]:
-            """
-            Compute the embeddings for a given user query
-
-            Parameters
-            ----------
-            query : Union[str, PIL.Image.Image]
-                The query to embed. A query can be either text or an image.
-            """
-            if isinstance(query, str):
-                return [self.generate_text_embeddings(query)]
-            else:
-                PIL = attempt_import_or_raise("PIL", "pillow")
-                if isinstance(query, PIL.Image.Image):
-                    return [self.generate_image_embedding(query)]
-                else:
-                    raise TypeError("OpenClip supports str or PIL Image as query")
-
-        def generate_text_embeddings(self, text: str) -> np.ndarray:
-            torch = attempt_import_or_raise("torch")
-            text = self.sanitize_input(text)
-            text = self._tokenizer(text)
-            text.to(self.device)
-            with torch.no_grad():
-                text_features = self._model.encode_text(text.to(self.device))
-                if self.normalize:
-                    text_features /= text_features.norm(dim=-1, keepdim=True)
-                return text_features.cpu().numpy().squeeze()
-
-        def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-            """
-            Sanitize the input to the embedding function.
-            """
-            if isinstance(images, (str, bytes)):
-                images = [images]
-            elif isinstance(images, pa.Array):
-                images = images.to_pylist()
-            elif isinstance(images, pa.ChunkedArray):
-                images = images.combine_chunks().to_pylist()
-            return images
-
-        def compute_source_embeddings(
-            self, images: IMAGES, *args, **kwargs
-        ) -> List[np.array]:
-            """
-            Get the embeddings for the given images
-            """
-            images = self.sanitize_input(images)
-            embeddings = []
-            for i in range(0, len(images), self.batch_size):
-                j = min(i + self.batch_size, len(images))
-                batch = images[i:j]
-                embeddings.extend(self._parallel_get(batch))
-            return embeddings
-
-        def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
-            """
-            Issue concurrent requests to retrieve the image data
-            """
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                futures = [
-                    executor.submit(self.generate_image_embedding, image)
-                    for image in images
-                ]
-                return [future.result() for future in futures]
-
-        def generate_image_embedding(
-            self, image: Union[str, bytes, "PIL.Image.Image"]
-        ) -> np.ndarray:
-            """
-            Generate the embedding for a single image
-
-            Parameters
-            ----------
-            image : Union[str, bytes, PIL.Image.Image]
-                The image to embed. If the image is a str, it is treated as a uri.
-                If the image is bytes, it is treated as the raw image bytes.
-            """
-            torch = attempt_import_or_raise("torch")
-            # TODO handle retry and errors for https
-            image = self._to_pil(image)
-            image = self._preprocess(image).unsqueeze(0)
-            with torch.no_grad():
-                return self._encode_and_normalize_image(image)
-
-        def _to_pil(self, image: Union[str, bytes]):
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query)]
+        else:
             PIL = attempt_import_or_raise("PIL", "pillow")
-            if isinstance(image, bytes):
-                return PIL.Image.open(io.BytesIO(image))
-            if isinstance(image, PIL.Image.Image):
-                return image
-            elif isinstance(image, str):
-                parsed = urlparse.urlparse(image)
-                # TODO handle drive letter on windows.
-                if parsed.scheme == "file":
-                    return PIL.Image.open(parsed.path)
-                elif parsed.scheme == "":
-                    return PIL.Image.open(image if os.name == "nt" else parsed.path)
-                elif parsed.scheme.startswith("http"):
-                    return PIL.Image.open(io.BytesIO(url_retrieve(image)))
-                else:
-                    raise NotImplementedError("Only local and http(s) urls are supported")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError("OpenClip supports str or PIL Image as query")
 
-        def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
-            """
-            encode a single image tensor and optionally normalize the output
-            """
-            image_features = self._model.encode_image(image_tensor)
+    def generate_text_embeddings(self, text: str) -> np.ndarray:
+        torch = attempt_import_or_raise("torch")
+        text = self.sanitize_input(text)
+        text = self._tokenizer(text)
+        text.to(self.device)
+        with torch.no_grad():
+            text_features = self._model.encode_text(text.to(self.device))
             if self.normalize:
-                image_features /= image_features.norm(dim=-1, keepdim=True)
-            return image_features.cpu().numpy().squeeze()
-    ```
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
 
-=== "TypeScript"
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
 
-    Coming Soon! See this [issue](https://github.com/lancedb/lancedb/issues/1482) to track the status!
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given images
+        """
+        images = self.sanitize_input(images)
+        embeddings = []
+        for i in range(0, len(images), self.batch_size):
+            j = min(i + self.batch_size, len(images))
+            batch = images[i:j]
+            embeddings.extend(self._parallel_get(batch))
+        return embeddings
+
+    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+        """
+        Issue concurrent requests to retrieve the image data
+        """
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_embedding, image)
+                for image in images
+            ]
+            return [future.result() for future in futures]
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        torch = attempt_import_or_raise("torch")
+        # TODO handle retry and errors for https
+        image = self._to_pil(image)
+        image = self._preprocess(image).unsqueeze(0)
+        with torch.no_grad():
+            return self._encode_and_normalize_image(image)
+
+    def _to_pil(self, image: Union[str, bytes]):
+        PIL = attempt_import_or_raise("PIL", "pillow")
+        if isinstance(image, bytes):
+            return PIL.Image.open(io.BytesIO(image))
+        if isinstance(image, PIL.Image.Image):
+            return image
+        elif isinstance(image, str):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            if parsed.scheme == "file":
+                return PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                return PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+
+    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+        """
+        encode a single image tensor and optionally normalize the output
+        """
+        image_features = self._model.encode_image(image_tensor)
+        if self.normalize:
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().squeeze()
+```

docs/src/embeddings/default_embedding_functions.md

@@ -1,86 +1,632 @@
-# 📚 Available Embedding Models
+There are various embedding functions available out of the box with LanceDB to manage your embeddings implicitly. We're actively working on adding other popular embedding APIs and models.
 
-There are various embedding functions available out of the box with LanceDB to manage your embeddings implicitly. We're actively working on adding other popular embedding APIs and models. 🚀
+## Text embedding functions
+Contains the text embedding functions registered by default.
 
-Before jumping on the list of available models, let's understand how to get an embedding model initialized and configured to use in our code: 
+* Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential backoff. 
+* Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
 
-!!! example "Example usage"
-    ```python
-    model = get_registry()
-              .get("openai")
-              .create(name="text-embedding-ada-002")
+### Sentence transformers
+Allows you to set parameters when registering a `sentence-transformers` object.
+
+!!! info
+    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
+| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
+| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
+
+
+??? "Check out available sentence-transformer models here!"
+    ```markdown
+    - sentence-transformers/all-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-mpnet-base-v2
+    - sentence-transformers/gtr-t5-base
+    - sentence-transformers/LaBSE
+    - sentence-transformers/all-MiniLM-L6-v2
+    - sentence-transformers/bert-base-nli-max-tokens
+    - sentence-transformers/bert-base-nli-mean-tokens
+    - sentence-transformers/bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
+    - sentence-transformers/bert-large-nli-cls-token
+    - sentence-transformers/bert-large-nli-max-tokens
+    - sentence-transformers/bert-large-nli-mean-tokens
+    - sentence-transformers/bert-large-nli-stsb-mean-tokens
+    - sentence-transformers/distilbert-base-nli-max-tokens
+    - sentence-transformers/distilbert-base-nli-mean-tokens
+    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
+    - sentence-transformers/distilroberta-base-msmarco-v1
+    - sentence-transformers/distilroberta-base-msmarco-v2
+    - sentence-transformers/nli-bert-base-cls-pooling
+    - sentence-transformers/nli-bert-base-max-pooling
+    - sentence-transformers/nli-bert-base
+    - sentence-transformers/nli-bert-large-cls-pooling
+    - sentence-transformers/nli-bert-large-max-pooling
+    - sentence-transformers/nli-bert-large
+    - sentence-transformers/nli-distilbert-base-max-pooling
+    - sentence-transformers/nli-distilbert-base
+    - sentence-transformers/nli-roberta-base
+    - sentence-transformers/nli-roberta-large
+    - sentence-transformers/roberta-base-nli-mean-tokens
+    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
+    - sentence-transformers/roberta-large-nli-mean-tokens
+    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
+    - sentence-transformers/stsb-bert-base
+    - sentence-transformers/stsb-bert-large
+    - sentence-transformers/stsb-distilbert-base
+    - sentence-transformers/stsb-roberta-base
+    - sentence-transformers/stsb-roberta-large
+    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
+    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
+    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
+    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
+    - sentence-transformers/bert-base-nli-cls-token
+    - sentence-transformers/all-distilroberta-v1
+    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
+    - sentence-transformers/multi-qa-distilbert-cos-v1
+    - sentence-transformers/multi-qa-distilbert-dot-v1
+    - sentence-transformers/multi-qa-mpnet-base-cos-v1
+    - sentence-transformers/multi-qa-mpnet-base-dot-v1
+    - sentence-transformers/nli-distilroberta-base-v2
+    - sentence-transformers/all-MiniLM-L6-v1
+    - sentence-transformers/all-mpnet-base-v1
+    - sentence-transformers/all-mpnet-base-v2
+    - sentence-transformers/all-roberta-large-v1
+    - sentence-transformers/allenai-specter
+    - sentence-transformers/average_word_embeddings_glove.6B.300d
+    - sentence-transformers/average_word_embeddings_glove.840B.300d
+    - sentence-transformers/average_word_embeddings_komninos
+    - sentence-transformers/average_word_embeddings_levy_dependency
+    - sentence-transformers/clip-ViT-B-32-multilingual-v1
+    - sentence-transformers/clip-ViT-B-32
+    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
+    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
+    - sentence-transformers/distilroberta-base-paraphrase-v1
+    - sentence-transformers/distiluse-base-multilingual-cased-v1
+    - sentence-transformers/distiluse-base-multilingual-cased-v2
+    - sentence-transformers/distiluse-base-multilingual-cased
+    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
+    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
+    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
+    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
+    - sentence-transformers/gtr-t5-large
+    - sentence-transformers/gtr-t5-xl
+    - sentence-transformers/gtr-t5-xxl
+    - sentence-transformers/msmarco-MiniLM-L-12-v3
+    - sentence-transformers/msmarco-MiniLM-L-6-v3
+    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
+    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
+    - sentence-transformers/msmarco-bert-base-dot-v5
+    - sentence-transformers/msmarco-bert-co-condensor
+    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
+    - sentence-transformers/msmarco-distilbert-base-tas-b
+    - sentence-transformers/msmarco-distilbert-base-v2
+    - sentence-transformers/msmarco-distilbert-base-v3
+    - sentence-transformers/msmarco-distilbert-base-v4
+    - sentence-transformers/msmarco-distilbert-cos-v5
+    - sentence-transformers/msmarco-distilbert-dot-v5
+    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
+    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
+    - sentence-transformers/msmarco-distilroberta-base-v2
+    - sentence-transformers/msmarco-roberta-base-ance-firstp
+    - sentence-transformers/msmarco-roberta-base-v2
+    - sentence-transformers/msmarco-roberta-base-v3
+    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
+    - sentence-transformers/nli-mpnet-base-v2
+    - sentence-transformers/nli-roberta-base-v2
+    - sentence-transformers/nq-distilbert-base-v1
+    - sentence-transformers/paraphrase-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-MiniLM-L3-v2
+    - sentence-transformers/paraphrase-MiniLM-L6-v2
+    - sentence-transformers/paraphrase-TinyBERT-L6-v2
+    - sentence-transformers/paraphrase-albert-base-v2
+    - sentence-transformers/paraphrase-albert-small-v2
+    - sentence-transformers/paraphrase-distilroberta-base-v1
+    - sentence-transformers/paraphrase-distilroberta-base-v2
+    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
+    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
+    - sentence-transformers/quora-distilbert-base
+    - sentence-transformers/quora-distilbert-multilingual
+    - sentence-transformers/sentence-t5-base
+    - sentence-transformers/sentence-t5-large
+    - sentence-transformers/sentence-t5-xxl
+    - sentence-transformers/sentence-t5-xl
+    - sentence-transformers/stsb-distilroberta-base-v2
+    - sentence-transformers/stsb-mpnet-base-v2
+    - sentence-transformers/stsb-roberta-base-v2
+    - sentence-transformers/stsb-xlm-r-multilingual
+    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
+    - sentence-transformers/clip-ViT-L-14
+    - sentence-transformers/clip-ViT-B-16
+    - sentence-transformers/use-cmlm-multilingual
+    - sentence-transformers/all-MiniLM-L12-v1
     ```
 
-Now let's understand the above syntax: 
+!!! info
+    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
+    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
+
+!!! note "BAAI Embeddings example"
+    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+    class Words(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    table = db.create_table("words", schema=Words)
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+    )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
+
+
+### Huggingface embedding models
+We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
+
+Example usage - 
 ```python
-model = get_registry().get("model_id").create(...params)
+import lancedb
+import pandas as pd
+
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+model = get_registry().get("huggingface").create(name='facebook/bart-base')
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
+table = db.create_table("greets", schema=Words)
+table.add()
+query = "old greeting"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
 ```
-**This👆 line effectively creates a configured instance of an `embedding function` with `model` of choice that is ready for use.**
 
-- `get_registry()` :  This function call returns an instance of a `EmbeddingFunctionRegistry` object. This registry manages the registration and retrieval of embedding functions.
 
-- `.get("model_id")` : This method call on the registry object and retrieves the **embedding models functions** associated with the `"model_id"` (1) .
-    { .annotate }
+### Ollama embeddings
+Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
 
-    1.  Hover over the names in table below to find out the `model_id` of different embedding functions.
+- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
+- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
 
-- `.create(...params)` : This method call is on the object returned by the `get` method. It instantiates an embedding model function using the **specified parameters**. 
+| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
+|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
+| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
+| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
+| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
+| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
 
-??? question "What parameters does the `.create(...params)` method accepts?"
-    **Checkout the documentation of specific embedding models (links in the table below👇) to know what parameters it takes**.
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
 
-!!! tip "Moving on"
-    Now that we know how to get the **desired embedding model** and use it in our code, let's explore the comprehensive **list** of embedding models **supported by LanceDB**, in the tables below.
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("ollama").create(name="nomic-embed-text")
 
-## Text Embedding Functions 📝 
-These functions are registered by default to handle text embeddings.
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
 
-- 🔄 **Embedding functions** have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with **exponential backoff**. 
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add([
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+])
 
-- 🌕 Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7. 
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
 
-🌟 **Available Text Embeddings**
 
-| **Embedding** :material-information-outline:{ title="Hover over the name to find out the model_id" } | **Description** | **Documentation** |
-|-----------|-------------|---------------|
-| [**Sentence Transformers**](available_embedding_models/text_embedding_functions/sentence_transformers.md "sentence-transformers")  | 🧠 **SentenceTransformers** is a Python framework for state-of-the-art sentence, text, and image embeddings. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/sbert_2.png" alt="Sentence Transformers Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/sentence_transformers.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Huggingface Models**](available_embedding_models/text_embedding_functions/huggingface_embedding.md "huggingface") |🤗 We offer support for all **Huggingface** models. The default model is `colbert-ir/colbertv2.0`. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/hugging_face.png" alt="Huggingface Icon" width="130" height="35">](available_embedding_models/text_embedding_functions/huggingface_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Ollama Embeddings**](available_embedding_models/text_embedding_functions/ollama_embedding.md "ollama") | 🔍 Generate embeddings via the **Ollama** python library. Ollama supports embedding models, making it possible to build RAG apps. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/Ollama.png" alt="Ollama Icon" width="110" height="35">](available_embedding_models/text_embedding_functions/ollama_embedding.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**OpenAI Embeddings**](available_embedding_models/text_embedding_functions/openai_embedding.md "openai")| 🔑 **OpenAI’s** text embeddings measure the relatedness of text strings. **LanceDB** supports state-of-the-art embeddings from OpenAI. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/openai.png" alt="OpenAI Icon" width="100" height="35">](available_embedding_models/text_embedding_functions/openai_embedding.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Instructor Embeddings**](available_embedding_models/text_embedding_functions/instructor_embedding.md "instructor") | 📚 **Instructor**: An instruction-finetuned text embedding model that can generate text embeddings tailored to any task and domains by simply providing the task instruction, without any finetuning. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/instructor_embedding.png" alt="Instructor Embedding Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/instructor_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Gemini Embeddings**](available_embedding_models/text_embedding_functions/gemini_embedding.md "gemini-text") | 🌌 Google’s Gemini API generates state-of-the-art embeddings for words, phrases, and sentences. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/gemini.png" alt="Gemini Icon" width="95" height="35">](available_embedding_models/text_embedding_functions/gemini_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Cohere Embeddings**](available_embedding_models/text_embedding_functions/cohere_embedding.md "cohere") | 💬 This will help you get started with **Cohere** embedding models using LanceDB. Using cohere API requires cohere package. Install it via `pip`. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/cohere.png" alt="Cohere Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/cohere_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**Jina Embeddings**](available_embedding_models/text_embedding_functions/jina_embedding.md "jina") | 🔗 World-class embedding models to improve your search and RAG systems. You will need **jina api key**. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="Jina Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/jina_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [ **AWS Bedrock Functions**](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md "bedrock-text") | ☁️ AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/aws_bedrock.png" alt="AWS Bedrock Icon" width="120" height="35">](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
-| [**IBM Watsonx.ai**](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md "watsonx") | 💡 Generate text embeddings using IBM's watsonx.ai platform. **Note**: watsonx.ai library is an optional dependency. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/watsonx.png" alt="Watsonx Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md) |
-| [**VoyageAI Embeddings**](available_embedding_models/text_embedding_functions/voyageai_embedding.md "voyageai") | 🌕 Voyage AI provides cutting-edge embedding and rerankers. This will help you get started with **VoyageAI** embedding models using LanceDB. Using voyageai API requires voyageai package. Install it via `pip`. | [<img src="https://www.voyageai.com/logo.svg" alt="VoyageAI Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/voyageai_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
+### OpenAI embeddings
+LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+    )
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### Instructor Embeddings
+[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+
+If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
+
+!!! info
+    Represent the `domain` `text_type` for `task_objective`:
+
+    * `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
+    * `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
+    * `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+
+More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
 
 
 
-[st-key]: "sentence-transformers"
-[hf-key]: "huggingface"
-[ollama-key]: "ollama"
-[openai-key]: "openai"
-[instructor-key]: "instructor"
-[gemini-key]: "gemini-text"
-[cohere-key]: "cohere"
-[jina-key]: "jina"
-[aws-key]: "bedrock-text"
-[watsonx-key]: "watsonx"
-[voyageai-key]: "voyageai"
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+instructor = get_registry().get("instructor").create(
+                            source_instruction="represent the docuement for retreival",
+                            query_instruction="represent the document for retreiving the most similar documents"
+                            )
+
+class Schema(LanceModel):
+    vector: Vector(instructor.ndims()) = instructor.VectorField()
+    text: str = instructor.SourceField()
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+tbl.add(texts)
+```
+
+### Gemini Embeddings
+With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
+The Gemini Embedding Model API supports various task types:
+
+| Task Type               | Description                                                                                                                                                |
+|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
+| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically proided by Embeddings API |
+| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
+| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
+| "`clusering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
 
 
-## Multi-modal Embedding Functions🖼️ 
+Usage Example:
 
-Multi-modal embedding functions allow you to query your table using both images and text. 💬🖼️
+```python
+import lancedb
+import pandas as pd
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
 
-🌐 **Available Multi-modal Embeddings**
 
-| Embedding :material-information-outline:{ title="Hover over the name to find out the model_id" }  | Description | Documentation  |
-|-----------|-------------|---------------|
-| [**OpenClip Embeddings**](available_embedding_models/multimodal_embedding_functions/openclip_embedding.md "open-clip") | 🎨 We support CLIP model embeddings using the open source alternative, **open-clip** which supports various customizations. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/openclip_github.png" alt="openclip Icon" width="150" height="35">](available_embedding_models/multimodal_embedding_functions/openclip_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    
-| [**Imagebind Embeddings**](available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md "imageind") | 🌌  We have support for **imagebind model embeddings**. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/imagebind_meta.png" alt="imagebind Icon" width="150" height="35">](available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md)|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    
-| [**Jina Multi-modal Embeddings**](available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md "jina") | 🔗 **Jina embeddings** can also be used to embed both **text** and **image** data, only some of the models support image data and you can check the detailed documentation. 👉 | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="jina Icon" width="90" height="35">](available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md) |
+model = get_registry().get("gemini-text").create()
 
-!!! note
-    If you'd like to request support for additional **embedding functions**, please feel free to open an issue on our LanceDB [GitHub issue page](https://github.com/lancedb/lancedb/issues).
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
+### Cohere Embeddings
+Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
+
+Supported models are:
+```
+    * embed-english-v3.0
+    * embed-multilingual-v3.0
+    * embed-english-light-v3.0
+    * embed-multilingual-light-v3.0
+    * embed-english-v2.0
+    * embed-english-light-v2.0
+    * embed-multilingual-v2.0
+```
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
+| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
+| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
+
+Cohere supports following input types:
+| Input Type               | Description                          |
+|-------------------------|---------------------------------------|
+| "`search_document`"     | Used for embeddings stored in a vector|
+|                         | database for search use-cases.        |
+| "`search_query`"        | Used for embeddings of search queries |
+|                         | run against a vector DB               |
+| "`semantic_similarity`" | Specifies the given text will be used |
+|                         | for Semantic Textual Similarity (STS) |
+| "`classification`"      | Used for embeddings passed through a  |
+|                         | text classifier.                      |
+| "`clustering`"          | Used for the embeddings run through a |
+|                         | clustering algorithm                  |
+
+Usage Example:
+    
+    ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import EmbeddingFunctionRegistry
+
+        cohere = EmbeddingFunctionRegistry
+            .get_instance()
+            .get("cohere")
+            .create(name="embed-multilingual-v2.0")
+
+        class TextModel(LanceModel):
+            text: str = cohere.SourceField()
+            vector: Vector(cohere.ndims()) =  cohere.VectorField()
+
+        data = [ { "text": "hello world" },
+                { "text": "goodbye world" }]
+
+        db = lancedb.connect("~/.lancedb")
+        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+        tbl.add(data)
+    ```
+
+### AWS Bedrock Text Embedding Functions
+AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
+You can do so by using `awscli` and also add your session_token:
+```shell
+aws configure
+aws configure set aws_session_token "<your_session_token>"
+```
+to ensure that the credentials are set up correctly, you can run the following command:
+```shell
+aws sts get-caller-identity
+```
+
+Supported Embedding modelIDs are:
+* `amazon.titan-embed-text-v1`
+* `cohere.embed-english-v3`
+* `cohere.embed-multilingual-v3`
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
+| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
+| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
+| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
+| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
+| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
+| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
+
+Usage Example:
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+model = get_registry().get("bedrock-text").create()
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("tmp_path")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
+## Multi-modal embedding functions
+Multi-modal embedding functions allow you to query your table using both images and text.
+
+### OpenClip embeddings
+We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"ViT-B-32"` | The name of the model. |
+| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `batch_size` | `int` | `64` | The number of images to process in a batch. |
+| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
+
+This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
+
+!!! info
+    LanceDB supports ingesting images directly from accessible links.
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect(tmp_path)
+func = get_registry.get("open-clip").create()
+
+class Images(LanceModel):
+    label: str
+    image_uri: str = func.SourceField() # image uri as the source
+    image_bytes: bytes = func.SourceField() # image bytes as the source
+    vector: Vector(func.ndims()) = func.VectorField() # vector column 
+    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
+
+table = db.create_table("images", schema=Images)
+labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+uris = [
+    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+]
+# get each uri as bytes
+image_bytes = [requests.get(uri).content for uri in uris]
+table.add(
+    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+)
+```
+Now we can search using text from both the default vector column and the custom vector column
+```python
+
+# text search
+actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
+print(actual.label) # prints "dog"
+
+frombytes = (
+    table.search("man's best friend", vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(frombytes.label)
+
+```
+
+Because we're using a multi-modal embedding function, we can also search using images
+
+```python
+# image search
+query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
+image_bytes = requests.get(query_image_uri).content
+query_image = Image.open(io.BytesIO(image_bytes))
+actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
+print(actual.label == "dog")
+
+# image search using a custom vector column
+other = (
+    table.search(query_image, vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(actual.label)
+
+```
+
+### Imagebind embeddings
+We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
+
+This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"imagebind_huge"` | Name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
+
+Below is an example demonstrating how the API works:
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect(tmp_path)
+func = get_registry.get("imagebind").create()
+
+class ImageBindModel(LanceModel):
+    text: str
+    image_uri: str = func.SourceField()
+    audio_path: str
+    vector: Vector(func.ndims()) = func.VectorField()
+
+# add locally accessible image paths
+text_list=["A dog.", "A car", "A bird"]
+image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
+audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
+
+# Load data
+inputs = [
+    {"text": a, "audio_path": b, "image_uri": c}
+    for a, b, c in zip(text_list, audio_paths, image_paths)
+]
+
+#create table and add data
+table = db.create_table("img_bind", schema=ImageBindModel)
+table.add(inputs)
+```
+
+Now, we can search using any modality:
+
+#### image search
+```python
+query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
+actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "dog")
+```
+#### audio search
+
+```python
+query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
+actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "car")
+```
+#### Text search
+You can add any input query and fetch the result as follows:
+```python
+query = "an animal which flies and tweets" 
+actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "bird")
+```
+
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/embedding_functions.md

@@ -2,12 +2,12 @@ Representing multi-modal data as vector embeddings is becoming a standard practi
 
 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
-!!! Note "Embedding functions on LanceDB cloud"
-    When using embedding functions with LanceDB cloud, the embeddings will be generated on the source device and sent to the cloud. This means that the source device must have the necessary resources to generate the embeddings.
+!!! Note "LanceDB cloud doesn't support embedding functions yet"
+    LanceDB Cloud does not support embedding functions yet. You need to generate embeddings before ingesting into the table or querying.
 
 !!! warning
-    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself.
-    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function
+    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
+    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
     and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
     table metadata and have LanceDB automatically take care of regenerating the embeddings.
 
@@ -16,7 +16,7 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
 === "Python"
     In the LanceDB python SDK, we define a global embedding function registry with
-    many different embedding models and even more coming soon.
+    many different embedding models and even more coming soon. 
     Here's let's an implementation of CLIP as example.
 
     ```python
@@ -26,35 +26,20 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
     clip = registry.get("open-clip").create()
     ```
 
-    You can also define your own embedding function by implementing the `EmbeddingFunction`
+    You can also define your own embedding function by implementing the `EmbeddingFunction` 
     abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
 
-=== "TypeScript"
+=== "JavaScript""
     In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
     embedding function is available.
 
     ```javascript
-    import * as lancedb from '@lancedb/lancedb'
-    import { getRegistry } from '@lancedb/lancedb/embeddings'
+    const lancedb = require("vectordb");
 
     // You need to provide an OpenAI API key
     const apiKey = "sk-..."
     // The embedding function will create embeddings for the 'text' column
-    const func = getRegistry().get("openai").create({apiKey})
-    ```
-=== "Rust"
-    In the Rust SDK, the choices are more limited. For now, only the OpenAI
-    embedding function is available. But unlike the Python and TypeScript SDKs, you need manually register the OpenAI embedding function.
-
-    ```toml
-    // Make sure to include the `openai` feature
-    [dependencies]
-    lancedb = {version = "*", features = ["openai"]}
-    ```
-
-    ```rust
-    --8<-- "rust/lancedb/examples/openai.rs:imports"
-    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
     ```
 
 ## 2. Define the data model or schema
@@ -70,14 +55,14 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
     `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-=== "TypeScript"
+=== "JavaScript"
 
     For the TypeScript SDK, a schema can be inferred from input data, or an explicit
     Arrow schema can be provided.
 
 ## 3. Create table and add data
 
-Now that we have chosen/defined our embedding function and the schema,
+Now that we have chosen/defined our embedding function and the schema, 
 we can create the table and ingest data without needing to explicitly generate
 the embeddings at all:
 
@@ -89,26 +74,17 @@ the embeddings at all:
     table.add([{"image_uri": u} for u in uris])
     ```
 
-=== "TypeScript"
+=== "JavaScript"
 
-    === "@lancedb/lancedb"
+    ```javascript
+    const db = await lancedb.connect("data/sample-lancedb");
+    const data = [
+    { text: "pepperoni"},
+    { text: "pineapple"}
+    ]
 
-        ```ts
-        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.test.ts:embedding_function"
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const db = await lancedb.connect("data/sample-lancedb");
-        const data = [
-            { text: "pepperoni"},
-            { text: "pineapple"}
-        ]
-
-        const table = await db.createTable("vectors", data, embedding)
-        ```
+    const table = await db.createTable("vectors", data, embedding)
+    ```
 
 ## 4. Querying your table
 Not only can you forget about the embeddings during ingestion, you also don't
@@ -121,8 +97,8 @@ need to worry about it when you query the table:
     ```python
     results = (
         table.search("dog")
-            .limit(10)
-            .to_pandas()
+        .limit(10)
+        .to_pandas()
     )
     ```
 
@@ -133,32 +109,22 @@ need to worry about it when you query the table:
     query_image = Image.open(p)
     results = (
         table.search(query_image)
-            .limit(10)
-            .to_pandas()
+        .limit(10)
+        .to_pandas()
     )
     ```
 
     Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        const results = await table.search("What's the best pizza topping?")
-            .limit(10)
-            .toArray()
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const results = await table
-            .search("What's the best pizza topping?")
-            .limit(10)
-            .execute()
-        ```
+=== "JavaScript"
 
+    ```javascript
+    const results = await table
+    .search("What's the best pizza topping?")
+    .limit(10)
+    .execute()
+    ```    
+    
     The above snippet returns an array of records with the top 10 nearest neighbors to the query.
 
 ---

docs/src/embeddings/index.md

@@ -1,13 +1,13 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio.
-This makes them a very powerful tool for machine learning practitioners.
-However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
+This makes them a very powerful tool for machine learning practitioners. 
+However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
 (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
 LanceDB supports 3 methods of working with embeddings.
 
 1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
 2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
-3. You can define your own [custom embedding function](./custom_embedding_function.md)
+3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
    that extends the default embedding functions.
 
 For python users, there is also a legacy [with_embeddings API](./legacy.md).
@@ -18,115 +18,57 @@ It is retained for compatibility and will be removed in a future version.
 To get started with embeddings, you can use the built-in embedding functions.
 
 ### OpenAI Embedding function
-
 LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
 
-=== "Python"
-
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-    class Words(LanceModel):
-        text: str = func.SourceField()
-        vector: Vector(func.ndims()) = func.VectorField()
-
-    table = db.create_table("words", schema=Words, mode="overwrite")
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-        )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-
-=== "TypeScript"
-
-    ```typescript
-    --8<--- "nodejs/examples/embedding.test.ts:imports"
-    --8<--- "nodejs/examples/embedding.test.ts:openai_embeddings"
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<--- "rust/lancedb/examples/openai.rs:imports"
-    --8<--- "rust/lancedb/examples/openai.rs:openai_embeddings"
-    ```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-=== "Python"
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
-        [
-            {"text": "hello world"},
-            {"text": "goodbye world"}
-        ]
-    )
-
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-
-=== "TypeScript"
-
-    Coming Soon!
-
-=== "Rust"
-
-    Coming Soon!
-
-### Embedding function with LanceDB cloud
-Embedding functions are now supported on LanceDB cloud. The embeddings will be generated on the source device and sent to the cloud. This means that the source device must have the necessary resources to generate the embeddings. Here's an example using the OpenAI embedding function:
-
 ```python
-import os
 import lancedb
 from lancedb.pydantic import LanceModel, Vector
 from lancedb.embeddings import get_registry
-os.environ['OPENAI_API_KEY'] = "..."
 
-db = lancedb.connect(
-  uri="db://....",
-  api_key="sk_...",
-  region="us-east-1"
-)
-func = get_registry().get("openai").create()
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("openai").create(name="text-embedding-ada-002")
 
 class Words(LanceModel):
     text: str = func.SourceField()
     vector: Vector(func.ndims()) = func.VectorField()
 
-table = db.create_table("words", schema=Words)
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+    )
 
 query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
+
+### Sentence Transformers Embedding function
+LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+class Words(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+table = db.create_table("words", schema=Words)
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+)
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```

docs/src/embeddings/understanding_embeddings.md

@@ -1,133 +0,0 @@
-# Understand Embeddings
-
-The term **dimension** is a synonym for the number of elements in a feature vector. Each feature can be thought of as a different axis in a geometric space. 
-
-High-dimensional data means there are many features(or attributes) in the data.
-
-!!! example
-     1. An image is a data point and it might have thousands of dimensions because each pixel could be considered as a feature. 
-
-     2. Text data, when represented by each word or character, can also lead to high dimensions, especially when considering all possible words in a language.
-
-Embedding captures **meaning and relationships** within data by mapping high-dimensional data into a lower-dimensional space. It captures it by placing inputs that are more **similar in meaning** closer together in the **embedding space**. 
-
-## What are Vector Embeddings?
-
-Vector embeddings is a way to convert complex data, like text, images, or audio into numerical coordinates (called vectors) that can be plotted in an n-dimensional space(embedding space). 
-
-The closer these data points are related in the real world, the closer their corresponding numerical coordinates (vectors) will be to each other in the embedding space. This proximity in the embedding space reflects their semantic similarities, allowing machines to intuitively understand and process the data in a way that mirrors human perception of relationships and meaning.
-
-In a way, it captures the most important aspects of the data while ignoring the less important ones. As a result, tasks like searching for related content or identifying patterns become more efficient and accurate, as the embeddings make it possible to quantify how **closely related** different **data points** are and **reduce** the **computational complexity**.
-
-??? question "Are vectors and embeddings the same thing?"
-
-    When we say “vectors” we mean - **list of numbers** that **represents the data**. 
-    When we say “embeddings” we mean - **list of numbers** that **capture important details and relationships**.
-
-    Although the terms are often used interchangeably, “embeddings” highlight how the data is represented with meaning and structure, while “vector” simply refers to the numerical form of that representation.
-
-## Embedding vs Indexing
-
-We already saw that creating **embeddings** on data is a method of creating **vectors** for a **n-dimensional embedding space** that captures the meaning and relationships inherent in the data.
-
-Once we have these **vectors**, indexing comes into play. Indexing is a method of organizing these vector embeddings, that allows us to quickly and efficiently locate and retrieve them from the entire dataset of vector embeddings.
-
-## What types of data/objects can be embedded?
-
-The following are common types of data that can be embedded:
-
-1. **Text**: Text data includes sentences, paragraphs, documents, or any written content.
-2. **Images**:  Image data encompasses photographs, illustrations, or any visual content.
-3. **Audio**: Audio data includes sounds, music, speech, or any auditory content.
-4. **Video**:  Video data consists of moving images and sound, which can convey complex information.
-
-Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model.
-
-!!! tip "LanceDB vs Other traditional Vector DBs"
-    While many vector databases primarily focus on the storage and retrieval of vector embeddings, **LanceDB** uses **Lance file format** (operates on a disk-based architecture), which allows for the storage and management of not just embeddings but also **raw file data (bytes)**. This capability means that users can integrate various types of data, including images and text, alongside their vector embeddings in a unified system.
-
-    With the ability to store both vectors and associated file data, LanceDB enhances the querying process. Users can perform semantic searches that not only retrieve similar embeddings but also access related files and metadata, thus streamlining the workflow.
-
-## How does embedding works?
-
-As mentioned, after creating embedding, each data point is represented as a vector in a n-dimensional space (embedding space). The dimensionality of this space can vary depending on the complexity of the data and the specific embedding technique used.
-
-Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar. To quantify this closeness, we use distance as a metric which can be measured in the  following way - 
-
-1. **Euclidean Distance (L2)**: It calculates the straight-line distance between two points (vectors) in a multidimensional space.
-2. **Cosine Similarity**: It measures the cosine of the angle between two vectors, providing a normalized measure of similarity based on their direction.
-3. **Dot product**: It is calculated as the sum of the products of their corresponding components. To measure relatedness it considers both the magnitude and direction of the vectors.
-
-## How do you create and store vector embeddings for your data?
-
-1. **Creating embeddings**: Choose an embedding model, it can be a pre-trained model (open-source or commercial) or you can train a custom embedding model for your scenario. Then feed your preprocessed data into the chosen model to obtain embeddings.
-
-??? question "Popular choices for embedding models"
-    For text data, popular choices are OpenAI’s text-embedding models, Google Gemini text-embedding models, Cohere’s Embed models, and SentenceTransformers, etc.
-
-    For image data, popular choices are CLIP (Contrastive Language–Image Pretraining), Imagebind embeddings by meta (supports audio, video, and image), and Jina multi-modal embeddings, etc.
-
-2. **Storing vector embeddings**: This effectively requires **specialized databases** that can handle the complexity of vector data, as traditional databases often struggle with this task. Vector databases are designed specifically for storing and querying vector embeddings. They optimize for efficient nearest-neighbor searches and provide built-in indexing mechanisms.
-
-!!! tip "Why LanceDB"
-    LanceDB **automates** the entire process of creating and storing embeddings for your data. LanceDB allows you to define and use **embedding functions**, which can be **pre-trained models** or **custom models**. 
-    
-    This enables you to **generate** embeddings tailored to the nature of your data (e.g., text, images) and **store** both the **original data** and **embeddings** in a **structured schema** thus providing efficient querying capabilities for similarity searches.
-
-Let's quickly [get started](./index.md) and learn how to manage embeddings in LanceDB. 
-
-## Bonus: As a developer, what you can create using embeddings?
-
-As a developer, you can create a variety of innovative applications using vector embeddings. Check out the following - 
-
-<div class="grid cards" markdown>
-
--   __Chatbots__
-
-    ---
-
-    Develop chatbots that utilize embeddings to retrieve relevant context and generate coherent, contextually aware responses to user queries.
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/chatbot.md)
-
--   __Recommendation Systems__
-
-    ---
-
-    Develop systems that recommend content (such as articles, movies, or products) based on the similarity of keywords and descriptions, enhancing user experience.
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/recommendersystem.md)
-
--   __Vector Search__
-
-    ---
-
-    Build powerful applications that harness the full potential of semantic search, enabling them to retrieve relevant data quickly and effectively. 
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/vector_search.md)
-
--   __RAG Applications__
-
-    ---
-
-    Combine the strengths of large language models (LLMs) with retrieval-based approaches to create more useful applications.
-
-    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/rag.md)
-
--   __Many more examples__
-
-    ---
-
-    Explore applied examples available as Colab notebooks or Python scripts to integrate into your applications.
-
-    [:octicons-arrow-right-24: More](../examples/examples_python.md)
-
-</div>
-
-
-
-
-
-
-
-

docs/src/examples/examples_python.md

@@ -1,22 +1,17 @@
-# Overview : Python Examples 
+# Examples: Python
 
-To help you get started, we provide some examples, projects, and applications that use the LanceDB Python API. These examples are designed to get you right into the code with minimal introduction, enabling you to move from an idea to a proof of concept in minutes. 
+To help you get started, we provide some examples, projects and applications that use the LanceDB Python API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
 
-You can find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository. 
-
-**Introduction**
-
-Explore applied examples available as Colab notebooks or Python scripts to integrate into your applications. You can also checkout our blog posts related to the particular example for deeper understanding.
-
-| Explore                                         | Description                                                                                                                                                      |
-|----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [**Build from Scratch with LanceDB** 🛠️🚀](python_examples/build_from_scratch.md)               | Start building your **GenAI applications** from the **ground up** using **LanceDB's** efficient vector-based document retrieval capabilities! Get started quickly with a solid foundation.      |
-| [**Multimodal Search with LanceDB** 🤹‍♂️🔍](python_examples/multimodal.md)               | Combine **text** and **image queries** to find the most relevant results using **LanceDB’s multimodal** capabilities. Leverage the efficient vector-based similarity search.   |
-| [**RAG (Retrieval-Augmented Generation) with LanceDB** 🔓🧐](python_examples/rag.md) | Build RAG (Retrieval-Augmented Generation) with **LanceDB** for efficient **vector-based information retrieval** and more accurate responses from AI.                    |
-| [**Vector Search: Efficient Retrieval** 🔓👀](python_examples/vector_search.md) | Use **LanceDB's** vector search capabilities to perform efficient and accurate **similarity searches**, enabling rapid discovery and retrieval of relevant documents in Large datasets.                                                 |
-| [**Chatbot applications with LanceDB** 🤖](python_examples/chatbot.md)                | Create **chatbots** that retrieves relevant context for **coherent and context-aware replies**, enhancing user experience through advanced conversational AI. |
-| [**Evaluation: Assessing Text Performance with Precision** 📊💡](python_examples/evaluations.md) | Develop **evaluation** applications that allows you to input reference and candidate texts to **measure** their performance across various metrics.                                                      |
-| [**AI Agents: Intelligent Collaboration** 🤖](python_examples/aiagent.md)              | Enable **AI agents** to communicate and collaborate efficiently through dense vector representations, achieving shared goals seamlessly.                                         |
-| [**Recommender Systems: Personalized Discovery** 🍿📺](python_examples/recommendersystem.md)      | Deliver **personalized experiences** by efficiently storing and querying item embeddings with **LanceDB's** powerful vector database capabilities.                        |
-| **Miscellaneous Examples🌟** | Find other **unique examples** and **creative solutions** using **LanceDB**, showcasing the flexibility and broad applicability of the platform. |
+| Example | Interactive Envs | Scripts  |
+|-------- | ---------------- | ------   |
+| | | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
+| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
+| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
+| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
+| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
+| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
+| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
 

docs/src/examples/index.md

@@ -8,15 +8,9 @@ LanceDB provides language APIs, allowing you to embed a database in your languag
 * 👾 [JavaScript](examples_js.md) examples
 * 🦀 Rust examples (coming soon)
 
-## Python Applications powered by LanceDB
+## Applications powered by LanceDB
 
-| Project Name | Description |
-| --- | --- |
-| **Ultralytics Explorer 🚀**<br>[![Ultralytics](https://img.shields.io/badge/Ultralytics-Docs-green?labelColor=0f3bc4&style=flat-square&logo=https://cdn.prod.website-files.com/646dd1f1a3703e451ba81ecc/64994922cf2a6385a4bf4489_UltralyticsYOLO_mark_blue.svg&link=https://docs.ultralytics.com/datasets/explorer/)](https://docs.ultralytics.com/datasets/explorer/)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ultralytics/ultralytics/blob/main/docs/en/datasets/explorer/explorer.ipynb) |  - 🔍 **Explore CV Datasets**: Semantic search, SQL queries, vector similarity, natural language.<br>- 🖥️ **GUI & Python API**: Seamless dataset interaction.<br>- ⚡ **Efficient & Scalable**: Leverages LanceDB for large datasets.<br>- 📊 **Detailed Analysis**: Easily analyze data patterns.<br>- 🌐 **Browser GUI Demo**: Create embeddings, search images, run queries. |
-| **Website Chatbot🤖**<br>[![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/lancedb-vercel-chatbot)<br>[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&amp;env=OPENAI_API_KEY&amp;envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&amp;project-name=lancedb-vercel-chatbot&amp;repository-name=lancedb-vercel-chatbot&amp;demo-title=LanceDB%20Chatbot%20Demo&amp;demo-description=Demo%20website%20chatbot%20with%20LanceDB.&amp;demo-url=https%3A%2F%2Flancedb.vercel.app&amp;demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png) | - 🌐 **Chatbot from Sitemap/Docs**: Create a chatbot using site or document context.<br>- 🚀 **Embed LanceDB in Next.js**: Lightweight, on-prem storage.<br>- 🧠 **AI-Powered Context Retrieval**: Efficiently access relevant data.<br>- 🔧 **Serverless & Native JS**: Seamless integration with Next.js.<br>- ⚡ **One-Click Deploy on Vercel**: Quick and easy setup.. |
-
-## Nodejs Applications powered by LanceDB
-
-| Project Name | Description |
-| --- | --- |
-| **Langchain Writing Assistant✍️ **<br>[![Github](../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/applications/node/lanchain_writing_assistant) |  - **📂 Data Source Integration**:  Use your own data by specifying data source file, and the app instantly processes it to provide insights. <br>- **🧠 Intelligent Suggestions**:  Powered by LangChain.js and LanceDB, it improves writing productivity and accuracy.  <br>- **💡 Enhanced Writing Experience**: It delivers real-time contextual insights and factual suggestions while the user writes. |
+| Project Name                                        | Description                                                                                                          | Screenshot                                |
+|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
+| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
+| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/python_examples/aiagent.md

@@ -1,27 +0,0 @@
-# AI Agents: Intelligent Collaboration🤖
-
-Think of a platform where AI Agents can seamlessly exchange information, coordinate over tasks, and achieve shared targets with great efficiency💻📈.
-
-## Vector-Based Coordination: The Technical Advantage
-Leveraging LanceDB's vector-based capabilities, we can enable **AI agents 🤖** to communicate and collaborate through dense vector representations. AI agents can exchange information, coordinate on a task or work towards a common goal, just by giving queries📝.
-
-| **AI Agents** | **Description** | **Links** |
-|:--------------|:----------------|:----------|
-| **AI Agents: Reducing Hallucinationt📊** | 🤖💡 **Reduce AI hallucinations** using Critique-Based Contexting! Learn by Simplifying and Automating tedious workflows by going through fitness trainer agent example.💪 | [![Github](../../assets/github.svg)][hullucination_github] <br>[![Open In Collab](../../assets/colab.svg)][hullucination_colab] <br>[![Python](../../assets/python.svg)][hullucination_python] <br>[![Ghost](../../assets/ghost.svg)][hullucination_ghost] |                                                                                                                                                                                                                                                
-| **AI Trends Searcher: CrewAI🔍️** | 🔍️ Learn about **CrewAI Agents** ! Utilize the features of CrewAI - Role-based Agents, Task Management, and Inter-agent Delegation ! Make AI agents work together to do tricky stuff 😺| [![Github](../../assets/github.svg)][trend_github] <br>[![Open In Collab](../../assets/colab.svg)][trend_colab] <br>[![Ghost](../../assets/ghost.svg)][trend_ghost] |                                                                                                                                                                                                                                                
-| **SuperAgent Autogen🤖** | 💻 AI interactions with the Super Agent! Integrating **Autogen**, **LanceDB**, **LangChain**, **LiteLLM**, and **Ollama** to create AI agent that excels in understanding and processing complex queries.🤖 | [![Github](../../assets/github.svg)][superagent_github] <br>[![Open In Collab](../../assets/colab.svg)][superagent_colab] |
-
-
-[hullucination_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents
-[hullucination_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb
-[hullucination_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.py
-[hullucination_ghost]: https://blog.lancedb.com/how-to-reduce-hallucinations-from-llm-powered-agents-using-long-term-memory-72f262c3cc1f/
-
-[trend_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/AI-Trends-with-CrewAI
-[trend_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/AI-Trends-with-CrewAI/CrewAI_AI_Trends.ipynb 
-[trend_ghost]: https://blog.lancedb.com/track-ai-trends-crewai-agents-rag/
-
-[superagent_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/SuperAgent_Autogen
-[superagent_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/SuperAgent_Autogen/main.ipynb
-
-

docs/src/examples/python_examples/build_from_scratch.md

@@ -1,13 +0,0 @@
-# **Build from Scratch with LanceDB 🛠️🚀**
-
-Start building your GenAI applications from the ground up using **LanceDB's** efficient vector-based document retrieval capabilities! 📑
-
-**Get Started in Minutes ⏱️**
-
-These examples provide a solid foundation for building your own GenAI applications using LanceDB. Jump from idea to **proof of concept** quickly with applied examples. Get started and see what you can create! 💻
-
-| **Build From Scratch**                     | **Description**                                                                                              | **Links**                                                                                                                                                                                                                                                                                                                                                                                                                |
-|:-------------------------------------------|:-------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Build RAG from Scratch🚀💻**             | 📝 Create a **Retrieval-Augmented Generation** (RAG) model from scratch using LanceDB.                       | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/RAG-from-Scratch)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)]()                                                                                                                                  |
-| **Local RAG from Scratch with Llama3🔥💡** | 🐫 Build a local RAG model using **Llama3** and **LanceDB** for fast and efficient text generation.          | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Local-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Local-RAG-from-Scratch/rag.py)    |
-| **Multi-Head RAG from Scratch📚💻**        | 🤯 Develop a **Multi-Head RAG model** from scratch, enabling generation of text based on multiple documents. | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch) |

docs/src/examples/python_examples/chatbot.md

@@ -1,41 +0,0 @@
-**Chatbot applications with LanceDB 🤖**
-====================================================================
-
- Create innovative chatbot applications that utilizes LanceDB for efficient vector-based response generation! 🌐✨ 
- 
-**Introduction 👋✨**
-
- Users can input their queries, allowing the chatbot to retrieve relevant context seamlessly. 🔍📚 This enables the generation of coherent and context-aware replies that enhance user experience. 🌟🤝 Dive into the world of advanced conversational AI and streamline interactions with powerful data management! 🚀💡
-
-
-| **Chatbot**  | **Description**  | **Links**  |
-|:----------------|:-----------------|:-----------|
-| **Databricks DBRX Website Bot ⚡️** | Engage with the **Hogwarts chatbot**, that uses Open-source RAG with **DBRX**, **LanceDB** and **LLama-index with Hugging Face Embeddings**, to provide interactive and engaging user experiences. ✨ | [![GitHub](../../assets/github.svg)][databricks_github] <br>[![Python](../../assets/python.svg)][databricks_python] |
-| **CLI SDK Manual Chatbot Locally 💻** | CLI chatbot for SDK/hardware documents using **Local RAG** with **LLama3**, **Ollama**, **LanceDB**, and **Openhermes Embeddings**, built with **Phidata** Assistant and Knowledge Base 🤖 | [![GitHub](../../assets/github.svg)][clisdk_github] <br>[![Python](../../assets/python.svg)][clisdk_python] |
-| **Youtube Transcript Search QA Bot 📹** |  Search through **youtube transcripts** using natural language with a Q&A bot, leveraging **LanceDB** for effortless data storage and management 💬 | [![GitHub](../../assets/github.svg)][youtube_github] <br>[![Open In Collab](../../assets/colab.svg)][youtube_colab] <br>[![Python](../../assets/python.svg)][youtube_python] |
-| **Code Documentation Q&A Bot with LangChain 🤖** | Query your own documentation easily using questions in natural language with a Q&A bot, powered by **LangChain** and **LanceDB**, demonstrated with **Numpy 1.26 docs** 📚 | [![GitHub](../../assets/github.svg)][docs_github] <br>[![Open In Collab](../../assets/colab.svg)][docs_colab] <br>[![Python](../../assets/python.svg)][docs_python] |
-| **Context-aware Chatbot using Llama 2 & LanceDB 🤖** | Build **conversational AI** with a **context-aware chatbot**, powered by **Llama 2**, **LanceDB**, and **LangChain**, that enables intuitive and meaningful conversations with your data 📚💬 | [![GitHub](../../assets/github.svg)][aware_github] <br>[![Open In Collab](../../assets/colab.svg)][aware_colab] <br>[![Ghost](../../assets/ghost.svg)][aware_ghost] |
-| **Chat with csv using Hybrid Search 📊** | **Chat** application that interacts with **CSV** and **Excel files** using **LanceDB’s** hybrid search capabilities, performing direct operations on large-scale columnar data efficiently 🚀 | [![GitHub](../../assets/github.svg)][csv_github] <br>[![Open In Collab](../../assets/colab.svg)][csv_colab] <br>[![Ghost](../../assets/ghost.svg)][csv_ghost] |
-
-
-[databricks_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/databricks_DBRX_website_bot
-[databricks_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/databricks_DBRX_website_bot/main.py
-
-[clisdk_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/CLI-SDK-Manual-Chatbot-Locally
-[clisdk_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/CLI-SDK-Manual-Chatbot-Locally/assistant.py
-
-[youtube_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Youtube-Search-QA-Bot
-[youtube_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Youtube-Search-QA-Bot/main.ipynb
-[youtube_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Youtube-Search-QA-Bot/main.py
-
-[docs_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot
-[docs_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb
-[docs_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.py
-
-[aware_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/chatbot_using_Llama2_&_lanceDB
-[aware_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/chatbot_using_Llama2_&_lanceDB/main.ipynb
-[aware_ghost]: https://blog.lancedb.com/context-aware-chatbot-using-llama-2-lancedb-as-vector-database-4d771d95c755
-
-[csv_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Chat_with_csv_file
-[csv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Chat_with_csv_file/main.ipynb
-[csv_ghost]: https://blog.lancedb.com/p/d8c71df4-e55f-479a-819e-cde13354a6a3/

docs/src/examples/python_examples/evaluations.md

@@ -1,21 +0,0 @@
-**Evaluation: Assessing Text Performance with Precision 📊💡**
-====================================================================
-
-Evaluation is a comprehensive tool designed to measure the performance of text-based inputs, enabling data-driven optimization and improvement 📈. 
-
-**Text Evaluation 101 📚**
-
-Using robust framework for assessing reference and candidate texts across various metrics📊, ensure that the text outputs are high-quality and meet specific requirements and standards📝.
-
-| **Evaluation** | **Description** | **Links** |
-| -------------- | --------------- | --------- |
-| **Evaluating Prompts with Prompttools 🤖** | Compare, visualize & evaluate **embedding functions** (incl. OpenAI) across metrics like latency & custom evaluation 📈📊 | [![Github](../../assets/github.svg)][prompttools_github] <br>[![Open In Collab](../../assets/colab.svg)][prompttools_colab] |
-| **Evaluating RAG with RAGAs and GPT-4o 📊** | Evaluate **RAG pipelines** with cutting-edge metrics and tools, integrate with CI/CD for continuous performance checks, and generate responses with GPT-4o 🤖📈 | [![Github](../../assets/github.svg)][RAGAs_github] <br>[![Open In Collab](../../assets/colab.svg)][RAGAs_colab] |
-
-
-
-[prompttools_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts
-[prompttools_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb
-
-[RAGAs_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Evaluating_RAG_with_RAGAs
-[RAGAs_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Evaluating_RAG_with_RAGAs/Evaluating_RAG_with_RAGAs.ipynb

docs/src/examples/python_examples/multimodal.md

@@ -1,28 +0,0 @@
-# **Multimodal Search with LanceDB 🤹‍♂️🔍**
-
-Using LanceDB's multimodal capabilities, combine text and image queries to find the most relevant results in your corpus ! 🔓💡
-
-**Explore the Future of Search 🚀**
-
-LanceDB supports multimodal search by indexing and querying vector representations of text and image data 🤖. This enables efficient retrieval of relevant documents and images using vector-based similarity search 📊. The platform facilitates cross-modal search, allowing for text-image and image-text retrieval, and supports scalable indexing of high-dimensional vector spaces 💻.
-
-
-
-| **Multimodal**  | **Description**  | **Links**  |
-|:----------------|:-----------------|:-----------|
-| **Multimodal CLIP: DiffusionDB 🌐💥**    | Multi-Modal Search with **CLIP** and **LanceDB** Using **DiffusionDB** Data for Combined Text and Image Understanding ! 🔓 | [![GitHub](../../assets/github.svg)][Clip_diffusionDB_github] <br>[![Open In Collab](../../assets/colab.svg)][Clip_diffusionDB_colab] <br>[![Python](../../assets/python.svg)][Clip_diffusionDB_python] <br>[![Ghost](../../assets/ghost.svg)][Clip_diffusionDB_ghost] |
-| **Multimodal CLIP: Youtube Videos 📹👀** | Search **Youtube videos** using Multimodal CLIP, finding relevant content with ease and accuracy! 🎯                                                                                                                                                                | [![Github](../../assets/github.svg)][Clip_youtube_github]                                                                                                    <br>[![Open In Collab](../../assets/colab.svg)][Clip_youtube_colab]                                                                                           <br> [![Python](../../assets/python.svg)][Clip_youtube_python]         <br>[![Ghost](../../assets/ghost.svg)][Clip_youtube_python] |
-| **Multimodal Image + Text Search 📸🔍**               | Find **relevant documents** and **images** with a single query using **LanceDB's** multimodal search capabilities, to seamlessly integrate text and visuals ! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
-| **Cambrian-1: Vision-Centric Image Exploration 🔍👀** | Learn how **Cambrian-1** works, using an example of **Vision-Centric** exploration on images found through vector search ! Work on **Flickr-8k** dataset 🔎                        | [![Kaggle](https://img.shields.io/badge/Kaggle-035a7d?style=for-the-badge&logo=kaggle&logoColor=white)](https://www.kaggle.com/code/prasantdixit/cambrian-1-vision-centric-exploration-of-images/)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/cambrian-1-vision-centric-exploration/)                                                                                                                                                                                                                   |
-
-
-[Clip_diffusionDB_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb
-[Clip_diffusionDB_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.ipynb
-[Clip_diffusionDB_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.py
-[Clip_diffusionDB_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/
-
-
-[Clip_youtube_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search
-[Clip_youtube_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb
-[Clip_youtube_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.py
-[Clip_youtube_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/

docs/src/examples/python_examples/rag.md

@@ -1,83 +0,0 @@
-**RAG (Retrieval-Augmented Generation) with LanceDB 🔓🧐**
-====================================================================
-
-Build RAG (Retrieval-Augmented Generation) with  LanceDB, a powerful solution for efficient vector-based information retrieval 📊. 
-
-**Experience the Future of Search 🔄**
-
-🤖 RAG enables AI to **retrieve** relevant information from external sources and use it to **generate** more accurate and context-specific responses. 💻 LanceDB provides a robust framework for integrating LLMs with external knowledge sources 📝.
-
-| **RAG**                                      | **Description**                                                                                                                                                  | **Links**                  |
-|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|                                                                                                                                                     
-| **RAG with Matryoshka Embeddings and LlamaIndex** 🪆🔗  | Utilize **Matryoshka embeddings** and **LlamaIndex** to improve the efficiency and accuracy of your RAG models. 📈✨ | [![Github](../../assets/github.svg)][matryoshka_github] <br>[![Open In Collab](../../assets/colab.svg)][matryoshka_colab]  |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with Re-ranking** 📈🔄 | Enhance your RAG applications by implementing **re-ranking strategies** for more relevant document retrieval. 📚🔍 | [![Github](../../assets/github.svg)][rag_reranking_github] <br>[![Open In Collab](../../assets/colab.svg)][rag_reranking_colab] <br>[![Ghost](../../assets/ghost.svg)][rag_reranking_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Instruct-Multitask** 🧠🎯 |  Integrate the **Instruct Embedding Model** with LanceDB to streamline your embedding API, reducing redundant code and overhead. 🌐📊 | [![Github](../../assets/github.svg)][instruct_multitask_github] <br>[![Open In Collab](../../assets/colab.svg)][instruct_multitask_colab] <br>[![Python](../../assets/python.svg)][instruct_multitask_python] <br>[![Ghost](../../assets/ghost.svg)][instruct_multitask_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with HyDE** 🌌🔍 | Use **Hypothetical Document Embeddings** for efficient, accurate, and unsupervised dense retrieval. 📄🔍 | [![Github](../../assets/github.svg)][hyde_github] <br>[![Open In Collab](../../assets/colab.svg)][hyde_colab]<br>[![Ghost](../../assets/ghost.svg)][hyde_ghost]                 |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with LOTR** 🧙‍♂️📜 | Enhance RAG with **Lord of the Retriever (LOTR)** to address 'Lost in the Middle' challenges, especially in medical data. 🌟📜 | [![Github](../../assets/github.svg)][lotr_github] <br>[![Open In Collab](../../assets/colab.svg)][lotr_colab] <br>[![Ghost](../../assets/ghost.svg)][lotr_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Advanced RAG: Parent Document Retriever** 📑🔗 | Use **Parent Document & Bigger Chunk Retriever** to maintain context and relevance when generating related content. 🎵📄 | [![Github](../../assets/github.svg)][parent_doc_retriever_github] <br>[![Open In Collab](../../assets/colab.svg)][parent_doc_retriever_colab] <br>[![Ghost](../../assets/ghost.svg)][parent_doc_retriever_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Corrective RAG with Langgraph** 🔧📊 | Enhance RAG reliability with **Corrective RAG (CRAG)** by self-reflecting and fact-checking for accurate and trustworthy results. ✅🔍 |[![Github](../../assets/github.svg)][corrective_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][corrective_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][corrective_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Contextual Compression with RAG** 🗜️🧠 | Apply **contextual compression techniques** to condense large documents while retaining essential information. 📄🗜️                                               | [![Github](../../assets/github.svg)][compression_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][compression_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][compression_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Improve RAG with FLARE** 🔥|  Enable users to ask questions directly to **academic papers**, focusing on **ArXiv papers**, with **F**orward-**L**ooking **A**ctive **RE**trieval augmented generation.🚀🌟                                                                       | [![Github](../../assets/github.svg)][flare_github] <br>[![Open In Collab](../../assets/colab.svg)][flare_colab] <br>[![Ghost](../../assets/ghost.svg)][flare_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Query Expansion and Reranker** 🔍🔄 | Enhance RAG with query expansion using Large Language Models and advanced **reranking methods** like **Cross Encoders**, **ColBERT v2**, and **FlashRank** for improved document retrieval precision and recall 🔍📈                                                         | [![Github](../../assets/github.svg)][query_github] <br>[![Open In Collab](../../assets/colab.svg)][query_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **RAG Fusion** ⚡🌐 | Build RAG Fusion, utilize the **RRF algorithm** to rerank documents based on user queries ! Use **LanceDB** as vector database to store and retrieve documents related to queries via **OPENAI Embeddings**⚡🌐                                                                          | [![Github](../../assets/github.svg)][fusion_github] <br>[![Open In Collab](../../assets/colab.svg)][fusion_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
-| **Agentic RAG** 🤖📚 | Build autonomous information retrieval with **Agentic RAG**, a framework of **intelligent agents** that collaborate to synthesize, summarize, and compare data across sources, that enables proactive and informed decision-making 🤖📚                                                              | [![Github](../../assets/github.svg)][agentic_github] <br>[![Open In Collab](../../assets/colab.svg)][agentic_colab] |
-
-
-
-
-
-
-
-
-
-
-
-
-[matryoshka_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex
-[matryoshka_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex/RAG_with_MatryoshkaEmbedding_and_Llamaindex.ipynb
-
-[rag_reranking_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking
-[rag_reranking_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking/main.ipynb
-[rag_reranking_ghost]: https://blog.lancedb.com/simplest-method-to-improve-rag-pipeline-re-ranking-cf6eaec6d544
-
-
-[instruct_multitask_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask
-[instruct_multitask_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.ipynb
-[instruct_multitask_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.py
-[instruct_multitask_ghost]: https://blog.lancedb.com/multitask-embedding-with-lancedb-be18ec397543
-
-[hyde_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE
-[hyde_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb
-[hyde_ghost]: https://blog.lancedb.com/advanced-rag-precise-zero-shot-dense-retrieval-with-hyde-0946c54dfdcb
-
-[lotr_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR
-[lotr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR/main.ipynb
-[lotr_ghost]: https://blog.lancedb.com/better-rag-with-lotr-lord-of-retriever-23c8336b9a35
-
-[parent_doc_retriever_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever
-[parent_doc_retriever_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever/main.ipynb
-[parent_doc_retriever_ghost]: https://blog.lancedb.com/modified-rag-parent-document-bigger-chunk-retriever-62b3d1e79bc6
-
-[corrective_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph
-[corrective_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb
-[corrective_rag_ghost]: https://blog.lancedb.com/implementing-corrective-rag-in-the-easiest-way-2/
-
-[compression_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG
-[compression_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG/main.ipynb
-[compression_rag_ghost]: https://blog.lancedb.com/enhance-rag-integrate-contextual-compression-and-filtering-for-precision-a29d4a810301/
-
-[flare_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR
-[flare_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb
-[flare_ghost]: https://blog.lancedb.com/better-rag-with-active-retrieval-augmented-generation-flare-3b66646e2a9f/
-
-[query_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/QueryExpansion%26Reranker
-[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/QueryExpansion&Reranker/main.ipynb
-
-
-[fusion_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/RAG_Fusion
-[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/RAG_Fusion/main.ipynb
-
-[agentic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG
-[agentic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb
-
-

docs/src/examples/python_examples/recommendersystem.md

@@ -1,37 +0,0 @@
-**Recommender Systems: Personalized Discovery🍿📺**
-==============================================================
-Deliver personalized experiences with Recommender Systems. 🎁
-
-**Technical Overview📜**
-
-🔍️ LanceDB's powerful vector database capabilities can efficiently store and query item embeddings. Recommender Systems can utilize it and provide personalized recommendations based on user preferences 🤝 and item features 📊 and therefore enhance the user experience.🗂️ 
-
-| **Recommender System** | **Description** | **Links** |
-| ---------------------- | --------------- | --------- |
-| **Movie Recommender System🎬** | 🤝 Use **collaborative filtering** to predict user preferences, assuming similar users will like similar movies, and leverage **Singular Value Decomposition** (SVD) from Numpy for precise matrix factorization and accurate recommendations📊 | [![Github](../../assets/github.svg)][movie_github] <br>[![Open In Collab](../../assets/colab.svg)][movie_colab] <br>[![Python](../../assets/python.svg)][movie_python] |
-| **🎥 Movie Recommendation with Genres** | 🔍 Creates movie embeddings using **Doc2Vec**, capturing genre and characteristic nuances, and leverages VectorDB for efficient storage and querying, enabling accurate genre classification and personalized movie recommendations through **similarity searches**🎥 | [![Github](../../assets/github.svg)][genre_github] <br>[![Open In Collab](../../assets/colab.svg)][genre_colab] <br>[![Ghost](../../assets/ghost.svg)][genre_ghost] |
-| **🛍️ Product Recommender using Collaborative Filtering and LanceDB** | 📈 Using **Collaborative Filtering** and **LanceDB** to analyze your past purchases, recommends products based on user's past purchases. Demonstrated with the Instacart dataset in our example🛒 | [![Github](../../assets/github.svg)][product_github] <br>[![Open In Collab](../../assets/colab.svg)][product_colab] <br>[![Python](../../assets/python.svg)][product_python] |
-| **🔍 Arxiv Search with OpenCLIP and LanceDB** | 💡 Build a semantic search engine for **Arxiv papers** using **LanceDB**, and benchmarks its performance against traditional keyword-based search on **Nomic's Atlas**, to demonstrate the power of semantic search in finding relevant research papers📚 | [![Github](../../assets/github.svg)][arxiv_github] <br>[![Open In Collab](../../assets/colab.svg)][arxiv_colab] <br>[![Python](../../assets/python.svg)][arxiv_python] |
-| **Food Recommendation System🍴** | 🍔 Build a food recommendation system with **LanceDB**, featuring vector-based recommendations, full-text search, hybrid search, and reranking model integration for personalized and accurate food suggestions👌 | [![Github](../../assets/github.svg)][food_github] <br>[![Open In Collab](../../assets/colab.svg)][food_colab] |
-
-[movie_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommender
-[movie_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb
-[movie_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.py
-
-
-[genre_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/movie-recommendation-with-genres
-[genre_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/movie-recommendation-with-genres/movie_recommendation_with_doc2vec_and_lancedb.ipynb
-[genre_ghost]: https://blog.lancedb.com/movie-recommendation-system-using-lancedb-and-doc2vec/
-
-[product_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/product-recommender
-[product_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/product-recommender/main.ipynb
-[product_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/product-recommender/main.py
-
-
-[arxiv_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender
-[arxiv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender/main.ipynb
-[arxiv_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender/main.py
- 
-
-[food_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Food_recommendation
-[food_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Food_recommendation/main.ipynb

docs/src/examples/python_examples/vector_search.md

@@ -1,80 +0,0 @@
-**Vector Search: Efficient Retrieval 🔓👀**
-====================================================================
-
-Vector search with LanceDB, is a solution for  efficient and accurate similarity searches in large datasets 📊. 
-
-**Vector Search Capabilities in LanceDB🔝**
-
-LanceDB implements vector search algorithms for efficient document retrieval and analysis 📊. This enables fast and accurate discovery of relevant documents, leveraging dense vector representations 🤖. The platform supports scalable indexing and querying of high-dimensional vector spaces, facilitating precise document matching and retrieval 📈.
-
-| **Vector Search** | **Description** | **Links** |
-|:-----------------|:---------------|:---------|
-| **Inbuilt Hybrid Search 🔄** | Perform hybrid search in **LanceDB** by combining the results of semantic and full-text search via a reranking algorithm of your choice 📊 | [![Github](../../assets/github.svg)][inbuilt_hybrid_search_github] <br>[![Open In Collab](../../assets/colab.svg)][inbuilt_hybrid_search_colab] |                                                                                                                                                                                                                                                                                                    
-| **Hybrid Search with BM25 and LanceDB 💡** | Use **Synergizes BM25's** keyword-focused precision (term frequency, document length normalization, bias-free retrieval) with **LanceDB's** semantic understanding (contextual analysis, query intent alignment) for nuanced search results in complex datasets 📈 | [![Github](../../assets/github.svg)][BM25_github] <br>[![Open In Collab](../../assets/colab.svg)][BM25_colab] <br>[![Ghost](../../assets/ghost.svg)][BM25_ghost] |                                                                                                                                                                                                                                                                                                    
-| **NER-powered Semantic Search 🔎** | Extract and identify essential information from text with Named Entity Recognition **(NER)** methods: Dictionary-Based, Rule-Based, and Deep Learning-Based, to accurately extract and categorize entities, enabling precise semantic search results 🗂️ | [![Github](../../assets/github.svg)][NER_github] <br>[![Open In Collab](../../assets/colab.svg)][NER_colab] <br>[![Ghost](../../assets/ghost.svg)][NER_ghost]|                                                                                                                                                                                                                                                                                                    
-| **Audio Similarity Search using Vector Embeddings 🎵** | Create vector **embeddings of audio files** to find similar audio content, enabling efficient audio similarity search and retrieval in **LanceDB's** vector store 📻 |[![Github](../../assets/github.svg)][audio_search_github] <br>[![Open In Collab](../../assets/colab.svg)][audio_search_colab] <br>[![Python](../../assets/python.svg)][audio_search_python]|                                                                                                                                                                                                                                                                                                    
-| **LanceDB Embeddings API: Multi-lingual Semantic Search 🌎** | Build a universal semantic search table with **LanceDB's Embeddings API**, supporting multiple languages (e.g., English, French) using **cohere's** multi-lingual model, for accurate cross-lingual search results 📄 | [![Github](../../assets/github.svg)][mls_github] <br>[![Open In Collab](../../assets/colab.svg)][mls_colab] <br>[![Python](../../assets/python.svg)][mls_python] |                                                                                                                                                                                                                                                                                                    
-| **Facial Recognition: Face Embeddings 🤖** | Detect, crop, and embed faces using Facenet, then store and query face embeddings in **LanceDB** for efficient facial recognition and top-K matching results 👥 | [![Github](../../assets/github.svg)][fr_github] <br>[![Open In Collab](../../assets/colab.svg)][fr_colab] |                                                                                                                                                                                                                                                                                                    
-| **Sentiment Analysis: Hotel Reviews 🏨** | Analyze customer sentiments towards the hotel industry using **BERT models**, storing sentiment labels, scores, and embeddings in **LanceDB**, enabling queries on customer opinions and potential areas for improvement 💬 | [![Github](../../assets/github.svg)][sentiment_analysis_github] <br>[![Open In Collab](../../assets/colab.svg)][sentiment_analysis_colab] <br>[![Ghost](../../assets/ghost.svg)][sentiment_analysis_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Vector Arithmetic with LanceDB ⚖️** | Perform **vector arithmetic** on embeddings, enabling complex relationships and nuances in data to be captured, and simplifying the process of retrieving semantically similar results 📊 | [![Github](../../assets/github.svg)][arithmetic_github] <br>[![Open In Collab](../../assets/colab.svg)][arithmetic_colab] <br>[![Ghost](../../assets/ghost.svg)][arithmetic_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Imagebind Demo 🖼️** | Explore the multi-modal capabilities of **Imagebind** through a Gradio app, use **LanceDB API** for seamless image search and retrieval experiences 📸 | [![Github](../../assets/github.svg)][imagebind_github] <br> [![Open in Spaces](../../assets/open_hf_space.svg)][imagebind_huggingface] |                                                                                                                                                                                                                                                                                                    
-| **Search Engine using SAM & CLIP 🔍** | Build a search engine within an image using **SAM** and **CLIP** models, enabling object-level search and retrieval, with LanceDB indexing and search capabilities to find the closest match between image embeddings and user queries 📸 | [![Github](../../assets/github.svg)][swi_github] <br>[![Open In Collab](../../assets/colab.svg)][swi_colab] <br>[![Ghost](../../assets/ghost.svg)][swi_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Zero Shot Object Localization and Detection with CLIP 🔎** | Perform object detection on images using **OpenAI's CLIP**, enabling zero-shot localization and detection of objects, with capabilities to split images into patches, parse with CLIP, and plot bounding boxes 📊 | [![Github](../../assets/github.svg)][zsod_github] <br>[![Open In Collab](../../assets/colab.svg)][zsod_colab] |                                                                                                                                                                                                                                                                                                    
-| **Accelerate Vector Search with OpenVINO 🚀** | Boost vector search applications using **OpenVINO**, achieving significant speedups with **CLIP** for text-to-image and image-to-image searching, through PyTorch model optimization, FP16 and INT8 format conversion, and quantization with **OpenVINO NNCF** 📈 | [![Github](../../assets/github.svg)][openvino_github] <br>[![Open In Collab](../../assets/colab.svg)][openvino_colab] <br>[![Ghost](../../assets/ghost.svg)][openvino_ghost] |                                                                                                                                                                                                                                                                                                    
-| **Zero-Shot Image Classification with CLIP and LanceDB 📸** | Achieve zero-shot image classification using **CLIP** and **LanceDB**, enabling models to classify images without prior training on specific use cases, unlocking flexible and adaptable image classification capabilities 🔓 | [![Github](../../assets/github.svg)][zsic_github] <br>[![Open In Collab](../../assets/colab.svg)][zsic_colab] <br>[![Ghost](../../assets/ghost.svg)][zsic_ghost] |                                                                                                                                                                                                                                                                                                    
-
-
-
-
-[inbuilt_hybrid_search_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Inbuilt-Hybrid-Search
-[inbuilt_hybrid_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Inbuilt-Hybrid-Search/Inbuilt_Hybrid_Search_with_LanceDB.ipynb
-
-[BM25_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Hybrid_search_bm25_lancedb
-[BM25_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Hybrid_search_bm25_lancedb/main.ipynb
-[BM25_ghost]: https://blog.lancedb.com/hybrid-search-combining-bm25-and-semantic-search-for-better-results-with-lan-1358038fe7e6
-
-[NER_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/NER-powered-Semantic-Search
-[NER_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/NER-powered-Semantic-Search/NER_powered_Semantic_Search_with_LanceDB.ipynb
-[NER_ghost]: https://blog.lancedb.com/ner-powered-semantic-search-using-lancedb-51051dc3e493
-
-[audio_search_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/audio_search
-[audio_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.ipynb
-[audio_search_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.py
-
-[mls_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multi-lingual-wiki-qa
-[mls_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.ipynb
-[mls_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.py
-
-[fr_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/facial_recognition
-[fr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/facial_recognition/main.ipynb
-
-[sentiment_analysis_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews
-[sentiment_analysis_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews/Sentiment_Analysis_using_LanceDB.ipynb
-[sentiment_analysis_ghost]: https://blog.lancedb.com/sentiment-analysis-using-lancedb-2da3cb1e3fa6
-
-[arithmetic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Vector-Arithmetic-with-LanceDB
-[arithmetic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Vector-Arithmetic-with-LanceDB/main.ipynb
-[arithmetic_ghost]: https://blog.lancedb.com/vector-arithmetic-with-lancedb-an-intro-to-vector-embeddings/
-
-[imagebind_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/imagebind_demo
-[imagebind_huggingface]: https://huggingface.co/spaces/raghavd99/imagebind2
-
-[swi_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/search-within-images-with-sam-and-clip
-[swi_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/search-within-images-with-sam-and-clip/main.ipynb
-[swi_ghost]: https://blog.lancedb.com/search-within-an-image-331b54e4285e
-
-[zsod_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/zero-shot-object-detection-CLIP
-[zsod_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/zero-shot-object-detection-CLIP/zero_shot_object_detection_clip.ipynb
-
-[openvino_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Accelerate-Vector-Search-Applications-Using-OpenVINO
-[openvino_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Accelerate-Vector-Search-Applications-Using-OpenVINO/clip_text_image_search.ipynb
-[openvino_ghost]: https://blog.lancedb.com/accelerate-vector-search-applications-using-openvino-lancedb/
-
-[zsic_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/zero-shot-image-classification
-[zsic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/zero-shot-image-classification/main.ipynb
-[zsic_ghost]: https://blog.lancedb.com/zero-shot-image-classification-with-vector-search/
-
-
-
-
-

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -25,8 +25,8 @@ s3://eto-public/datasets/sift/vec_data.lance
 Then, we can write a quick Python script to populate our LanceDB Table:
 
 ```python
-import lance
-sift_dataset = lance.dataset("/path/to/local/vec_data.lance")
+import pylance
+sift_dataset = pylance.dataset("/path/to/local/vec_data.lance")
 df = sift_dataset.to_table().to_pandas()
 
 import lancedb

docs/src/fts.md

@@ -1,258 +1,173 @@
-# Full-text search (Native FTS)
+# Full-text search
 
-LanceDB provides support for full-text search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
-!!! note
-    The Python SDK uses tantivy-based FTS by default, need to pass `use_tantivy=False` to use native FTS.
+
+## Installation
+
+To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
+
+```sh
+# Say you want to use tantivy==0.20.1
+pip install tantivy==0.20.1
+```
 
 ## Example
 
-Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
+Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search.
 
-=== "Python"
-    === "Sync API"
+```python
+import lancedb
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts"
-        ```
-    === "Async API"
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts_async"
-        ```
+table = db.create_table(
+    "my_table",
+    data=[
+        {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+        {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+    ],
+)
+```
 
-=== "TypeScript"
+## Create FTS index on single column
 
-    ```typescript
-    import * as lancedb from "@lancedb/lancedb";
-    const uri = "data/sample-lancedb"
-    const db = await lancedb.connect(uri);
+The FTS index must be created before you can search via keywords.
 
-    const data = [
-    { vector: [3.1, 4.1], text: "Frodo was a happy puppy" },
-    { vector: [5.9, 26.5], text: "There are several kittens playing" },
-    ];
-    const tbl = await db.createTable("my_table", data, { mode: "overwrite" });
-    await tbl.createIndex("text", {
-        config: lancedb.Index.fts(),
-    });
+```python
+table.create_fts_index("text")
+```
 
-    await tbl
-        .search("puppy", "fts")
-        .select(["text"])
-        .limit(10)
-        .toArray();
-    ```
+To search an FTS index via keywords, LanceDB's `table.search` accepts a string as input:
 
-=== "Rust"
+```python
+table.search("puppy").limit(10).select(["text"]).to_list()
+```
 
-    ```rust
-    let uri = "data/sample-lancedb";
-    let db = connect(uri).execute().await?;
-    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    let tbl = db
-        .create_table("my_table", initial_data)
-        .execute()
-        .await?;
-    tbl
-        .create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
-        .execute()
-        .await?;
+This returns the result as a list of dictionaries as follows.
 
-    tbl
-        .query()
-        .full_text_search(FullTextSearchQuery::new("puppy".to_owned()))
-        .select(lancedb::query::Select::Columns(vec!["text".to_owned()]))
-        .limit(10)
-        .execute()
-        .await?;
-    ```
-
-It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
-
-Passing `fts_columns="text"` if you want to specify the columns to search.
+```python
+[{'text': 'Frodo was a happy puppy', 'score': 0.6931471824645996}]
+```
 
 !!! note
     LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
 
 ## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are with length greater than 40, and lowercase all words.
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
 
-Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.
+```python
+table.create_fts_index("text", tokenizer_name="en_stem")
+```
 
-For example, to enable stemming for English:
-=== "Sync API"
+The following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem"
-    ```
-=== "Async API"
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem_async"
-    ```
+## Index multiple columns
 
-the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
 
-The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.
+```python
+table.create_fts_index(["text1", "text2"])
+```
 
-For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding_async"
-    ```
+Note that the search API call does not change - you can search over all indexed columns at once.
 
 ## Filtering
 
-LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported.
+Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
+applied on top of the full text search results. This can be invoked via the familiar
+`where` syntax:
 
-This can be invoked via the familiar `where` syntax.
- 
-With pre-filtering:
-=== "Python"
+```python
+table.search("puppy").limit(10).where("meta='foo'").to_list()
+```
 
-    === "Sync API"
+## Sorting
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering"
-        ```
-    === "Async API"
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example, 
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering_async"
-        ```
+```
+table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
 
-=== "TypeScript"
+(table.search("terms", ordering_field_name="sort_by_field")
+ .limit(20)
+ .to_list())
+```
 
-    ```typescript
-    await tbl
-    .search("puppy")
-    .select(["id", "doc"])
-    .limit(10)
-    .where("meta='foo'")
-    .prefilter(true)
-    .toArray();
-    ```
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
 
-=== "Rust"
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see 
+    an error during indexing that looks like 
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
 
-    ```rust
-    table
-        .query()
-        .full_text_search(FullTextSearchQuery::new("puppy".to_owned()))
-        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
-        .limit(10)
-        .only_if("meta='foo'")
-        .execute()
-        .await?;
-    ```
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
 
-With post-filtering:
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl
-    .search("apple")
-    .select(["id", "doc"])
-    .limit(10)
-    .where("meta='foo'")
-    .prefilter(false)
-    .toArray();
-    ```
-
-=== "Rust"
-
-    ```rust
-    table
-        .query()
-        .full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
-        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
-        .postfilter()
-        .limit(10)
-        .only_if("meta='foo'")
-        .execute()
-        .await?;
-    ```
 
 ## Phrase queries vs. terms queries
 
-!!! warning "Warn"
-    Lance-based FTS doesn't support queries using boolean operators `OR`, `AND`.
-
 For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `old man sea`. For more details on the terms
+or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
 query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
 
-To search for a phrase, the index must be created with `with_position=True`:
-=== "Sync API"
+!!! tip "Note"
+    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position_async"
-    ```
-This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
-
-
-## Incremental indexing
-
-LanceDB supports incremental indexing, which means you can add new records to the table without reindexing the entire table.
-
-This can make the query more efficient, especially when the table is large and the new records are relatively small.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [3.1, 4.1], text: "Frodo was a happy puppy" }]);
-    await tbl.optimize();
+    ```py
+    # This raises a syntax error
+    table.search("they could have been dogs OR cats")
     ```
 
-=== "Rust"
+    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+    the query is treated as a phrase query.
 
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
+    ```py
+    # This works!
+    table.search("they could have been dogs or cats")
     ```
-!!! note
 
-    New data added after creating the FTS index will appear in search results while incremental index is still progress, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 
+It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+query you want to perform. To make this simpler, when you want to perform a phrase query, you can
+enforce it in one of two ways:
+
+1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
+a phrase query.
+2. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
+itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
+is treated as a phrase query.
+
+In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
+double quotes replaced by single quotes.
+
+
+## Configurations
+
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can
+reduce this if running on a smaller node, or increase this for faster performance while
+indexing a larger corpus.
+
+```python
+# configure a 512MB heap size
+heap = 1024 * 1024 * 512
+table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
+```
+
+## Current limitations
+
+1. Currently we do not yet support incremental writes.
+   If you add data after FTS index creation, it won't be reflected
+   in search results until you do a full reindex.
+
+2. We currently only support local filesystem paths for the FTS index.
+   This is a tantivy limitation. We've implemented an object store plugin
+   but there's no way in tantivy-py to specify to use it.

docs/src/fts_tantivy.md

@@ -1,160 +0,0 @@
-# Full-text search (Tantivy-based FTS)
-
-LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
-
-The tantivy-based FTS is only available in Python synchronous APIs and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
-
-## Installation
-
-To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
-
-```sh
-# Say you want to use tantivy==0.20.1
-pip install tantivy==0.20.1
-```
-
-## Example
-
-Consider that we have a LanceDB table named `my_table`, whose string column `content` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
-
-```python
-import lancedb
-
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
-table = db.create_table(
-    "my_table",
-    data=[
-        {"id": 1, "vector": [3.1, 4.1], "title": "happy puppy", "content": "Frodo was a happy puppy", "meta": "foo"},
-        {"id": 2, "vector": [5.9, 26.5], "title": "playing kittens", "content": "There are several kittens playing around the puppy", "meta": "bar"},
-    ],
-)
-
-# passing `use_tantivy=False` to use lance FTS index
-# `use_tantivy=True` by default
-table.create_fts_index("content", use_tantivy=True)
-table.search("puppy").limit(10).select(["content"]).to_list()
-# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
-# ...
-```
-
-It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
-
-!!! note
-    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
-
-## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
-
-```python
-table.create_fts_index("content", use_tantivy=True, tokenizer_name="en_stem", replace=True)
-```
-
-the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
-
-## Index multiple columns
-
-If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
-
-```python
-table.create_fts_index(["title", "content"], use_tantivy=True, replace=True)
-```
-
-Note that the search API call does not change - you can search over all indexed columns at once.
-
-## Filtering
-
-Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
-applied on top of the full text search results (see [native FTS](fts.md) if you need pre-filtering). This can be invoked via the familiar
-`where` syntax:
-
-```python
-table.search("puppy").limit(10).where("meta='foo'").to_list()
-```
-
-## Sorting
-
-You can pre-sort the documents by specifying `ordering_field_names` when
-creating the full-text search index. Once pre-sorted, you can then specify
-`ordering_field_name` while searching to return results sorted by the given
-field. For example,
-
-```python
-table.create_fts_index(["content"], use_tantivy=True, ordering_field_names=["id"], replace=True)
-
-(table.search("puppy", ordering_field_name="id")
- .limit(20)
- .to_list())
-```
-
-!!! note
-    If you wish to specify an ordering field at query time, you must also
-    have specified it during indexing time. Otherwise at query time, an
-    error will be raised that looks like `ValueError: The field does not exist: xxx`
-
-!!! note
-    The fields to sort on must be of typed unsigned integer, or else you will see
-    an error during indexing that looks like
-    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
-
-!!! note
-    You can specify multiple fields for ordering at indexing time.
-    But at query time only one ordering field is supported.
-
-
-## Phrase queries vs. terms queries
-
-For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
-query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
-
-!!! tip "Note"
-    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
-
-    ```py
-    # This raises a syntax error
-    table.search("they could have been dogs OR cats")
-    ```
-
-    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
-    the query is treated as a phrase query.
-
-    ```py
-    # This works!
-    table.search("they could have been dogs or cats")
-    ```
-
-It can be cumbersome to have to remember what will cause a syntax error depending on the type of
-query you want to perform. To make this simpler, when you want to perform a phrase query, you can
-enforce it in one of two ways:
-
-1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
-a phrase query.
-1. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
-itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
-is treated as a phrase query.
-
-In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
-double quotes replaced by single quotes.
-
-
-## Configurations
-
-By default, LanceDB configures a 1GB heap size limit for creating the index. You can
-reduce this if running on a smaller node, or increase this for faster performance while
-indexing a larger corpus.
-
-```python
-# configure a 512MB heap size
-heap = 1024 * 1024 * 512
-table.create_fts_index(["title", "content"], use_tantivy=True, writer_heap_size=heap, replace=True)
-```
-
-## Current limitations
-
-1. New data added after creating the FTS index will appear in search results, but with increased latency due to a flat search on the unindexed portion. Re-indexing with `create_fts_index` will reduce latency. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 
-
-2. We currently only support local filesystem paths for the FTS index.
-   This is a tantivy limitation. We've implemented an object store plugin
-   but there's no way in tantivy-py to specify to use it.

docs/src/guides/scalar_index.md

@@ -1,156 +0,0 @@
-# Building a Scalar Index
-
-Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
-
-Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
-over scalar columns.
-
-- `BTREE`: The most common type is BTREE. The index stores a copy of the
-  column in sorted order. This sorted copy allows a binary search to be used to
-  satisfy queries.
-- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
-  uses a series of bits to indicate whether a value is present in a row of a table
-- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
-  support queries with `array_contains_all` and `array_contains_any`
-  using an underlying bitmap index.
-  For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.
-
-!!! tips "How to choose the right scalar index type"
-
-    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
-    
-    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
-    
-    `LABEL_LIST`: This index should be used for columns containing list-type data.
-
-| Data Type                                                       | Filter                                    | Index Type   |
-| --------------------------------------------------------------- | ----------------------------------------- | ------------ |
-| Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
-| Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
-| List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |
-
-### Create a scalar index
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```js
-        const db = await lancedb.connect("data");
-        const tbl = await db.openTable("my_vectors");
-
-        await tbl.create_index("book_id");
-        await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
-        ```
-
-The following scan will be faster if the column `book_id` has a scalar index:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```js
-        const db = await lancedb.connect("data");
-        const tbl = await db.openTable("books");
-
-        await tbl
-          .query()
-          .where("book_id = 2")
-          .limit(10)
-          .toArray();
-        ```
-
-Scalar indices can also speed up scans containing a vector search or full text search, and a prefilter:
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```js
-        const db = await lancedb.connect("data/lance");
-        const tbl = await db.openTable("book_with_embeddings");
-
-        await tbl.search(Array(1536).fill(1.2))
-          .where("book_id != 3")  // prefilter is default behavior.
-          .limit(10)
-          .toArray();
-        ```
-### Update a scalar index
-Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
-    await tbl.optimize();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
-    ```
-
-!!! note
-
-    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.

docs/src/guides/storage.md

@@ -12,101 +12,48 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 === "Python"
 
     AWS S3:
-    === "Sync API"
 
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("s3://bucket/path")
+    ```
 
     Google Cloud Storage:
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("gs://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("gs://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("gs://bucket/path")
+    ```
 
     Azure Blob Storage:
 
-    <!-- skip-test -->
-    === "Sync API"
+    ```python
+    import lancedb
+    db = lancedb.connect("az://bucket/path")
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect("az://bucket/path")
-        ```
-    <!-- skip-test -->
-    === "Async API"
+=== "JavaScript"
 
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("az://bucket/path")
-        ```
-    Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
+    AWS S3:
 
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path");
+    ```
 
-=== "TypeScript"
+    Google Cloud Storage:
 
-    === "@lancedb/lancedb"
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("gs://bucket/path");
+    ```
 
-        AWS S3:
+    Azure Blob Storage:
 
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("s3://bucket/path");
-        ```
-
-        Google Cloud Storage:
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("gs://bucket/path");
-        ```
-
-        Azure Blob Storage:
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("az://bucket/path");
-        ```
-
-
-    === "vectordb (deprecated)"
-
-        AWS S3:
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("s3://bucket/path");
-        ```
-
-        Google Cloud Storage:
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("gs://bucket/path");
-        ```
-
-        Azure Blob Storage:
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("az://bucket/path");
-        ```
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("az://bucket/path");
+    ```
 
 In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
 
@@ -114,106 +61,58 @@ In most cases, when running in the respective cloud and permissions are set up c
 export TIMEOUT=60s
 ```
 
+!!! note "`storage_options` availability"
+
+    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
+    It is not yet supported in the Python synchronous API.
+
 If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={"timeout": "60s"}
+    )
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    === "Async API"
+=== "JavaScript"
 
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-
-        const db = await lancedb.connect("s3://bucket/path", {
-            storageOptions: {timeout: "60s"}
-        });
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("s3://bucket/path", {
-            storageOptions: {timeout: "60s"}
-        });
-        ```
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path",
+                                     {storageOptions: {timeout: "60s"}});
+    ```
 
 Getting even more specific, you can set the `timeout` for only a particular table:
 
 === "Python"
 
     <!-- skip-test -->
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async("s3://bucket/path")
+    table = await db.create_table(
+        "table",
+        [{"a": 1, "b": 2}],
+        storage_options={"timeout": "60s"}
+    )
+    ```
+
+=== "JavaScript"
 
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        table = db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
     <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        async_table = await async_db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        <!-- skip-test -->
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect("s3://bucket/path");
-        const table = db.createTable(
-            "table",
-            [{ a: 1, b: 2}],
-            {storageOptions: {timeout: "60s"}}
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        <!-- skip-test -->
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect("s3://bucket/path");
-        const table = db.createTable(
-            "table",
-            [{ a: 1, b: 2}],
-            {storageOptions: {timeout: "60s"}}
-        );
-        ```
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path");
+    const table = db.createTable(
+        "table",
+        [{ a: 1, b: 2}],
+        {storageOptions: {timeout: "60s"}}
+    );
+    ```
 
 !!! info "Storage option casing"
 
@@ -236,6 +135,7 @@ There are several options that can be set for all object stores, mostly related
 | `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
 | `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
 
+
 ### AWS S3
 
 To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
@@ -243,66 +143,33 @@ These can be set as environment variables or passed in the `storage_options` par
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "aws_access_key_id": "my-access-key",
+            "aws_secret_access_key": "my-secret-key",
+            "aws_session_token": "my-session-token",
+        }
+    )
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "s3://bucket/path",
+        {
+            storageOptions: {
+                awsAccessKeyId: "my-access-key",
+                awsSecretAccessKey: "my-secret-key",
+                awsSessionToken: "my-session-token",
             }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    awsAccessKeyId: "my-access-key",
-                    awsSecretAccessKey: "my-secret-key",
-                    awsSessionToken: "my-session-token",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    awsAccessKeyId: "my-access-key",
-                    awsSecretAccessKey: "my-secret-key",
-                    awsSessionToken: "my-session-token",
-                }
-            }
-        );
-        ```
+        }
+    );
+    ```
 
 Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
 
@@ -321,6 +188,7 @@ The following keys can be used as both environment variables or keys in the `sto
 | `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
 | `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
 
+
 !!! tip "Automatic cleanup for failed writes"
 
     LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
@@ -397,180 +265,37 @@ For **read-only access**, LanceDB will need a policy such as:
 }
 ```
 
-#### DynamoDB Commit Store for concurrent writes
-
-By default, S3 does not support concurrent writes. Having two or more processes
-writing to the same table at the same time can lead to data corruption. This is
-because S3, unlike other object stores, does not have any atomic put or copy
-operation.
-
-To enable concurrent writes, you can configure LanceDB to use a DynamoDB table
-as a commit store. This table will be used to coordinate writes between
-different processes. To enable this feature, you must modify your connection
-URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the
-name of the table to use.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )    
-        ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-
-    const db = await lancedb.connect(
-        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-    );
-    ```
-
-The DynamoDB table must be created with the following schema:
-
-- Hash key: `base_uri` (string)
-- Range key: `version` (number)
-
-You can create this programmatically with:
-
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import boto3
-
-    dynamodb = boto3.client("dynamodb")
-    table = dynamodb.create_table(
-        TableName=table_name,
-        KeySchema=[
-            {"AttributeName": "base_uri", "KeyType": "HASH"},
-            {"AttributeName": "version", "KeyType": "RANGE"},
-        ],
-        AttributeDefinitions=[
-            {"AttributeName": "base_uri", "AttributeType": "S"},
-            {"AttributeName": "version", "AttributeType": "N"},
-        ],
-        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
-    )
-    ```
-
-=== "JavaScript"
-
-    <!-- skip-test -->
-    ```javascript
-    import {
-      CreateTableCommand,
-      DynamoDBClient,
-    } from "@aws-sdk/client-dynamodb";
-
-    const dynamodb = new DynamoDBClient({
-      region: CONFIG.awsRegion,
-      credentials: {
-        accessKeyId: CONFIG.awsAccessKeyId,
-        secretAccessKey: CONFIG.awsSecretAccessKey,
-      },
-      endpoint: CONFIG.awsEndpoint,
-    });
-    const command = new CreateTableCommand({
-      TableName: table_name,
-      AttributeDefinitions: [
-        {
-          AttributeName: "base_uri",
-          AttributeType: "S",
-        },
-        {
-          AttributeName: "version",
-          AttributeType: "N",
-        },
-      ],
-      KeySchema: [
-        { AttributeName: "base_uri", KeyType: "HASH" },
-        { AttributeName: "version", KeyType: "RANGE" },
-      ],
-      ProvisionedThroughput: {
-        ReadCapacityUnits: 1,
-        WriteCapacityUnits: 1,
-      },
-    });
-    await client.send(command);
-    ```
-
-
 #### S3-compatible stores
 
 LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "region": "us-east-1",
+            "endpoint": "http://minio:9000",
+        }
+    )
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "s3://bucket/path",
+        {
+            storageOptions: {
+                region: "us-east-1",
+                endpoint: "http://minio:9000",
             }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )    
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    endpoint: "http://minio:9000",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "s3://bucket/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    endpoint: "http://minio:9000",
-                }
-            }
-        );
-        ```
+        }
+    );
+    ```
 
 This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
 
@@ -584,68 +309,38 @@ This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` envir
 
 #### S3 Express
 
-LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional infrastructure configuration for the compute service, such as EC2 or Lambda. Please refer to [Networking requirements for S3 Express One Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html).
+LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
 
 To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        storage_options={
+            "region": "us-east-1",
+            "s3_express": "true",
+        }
+    )
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        {
+            storageOptions: {
+                region: "us-east-1",
+                s3Express: "true",
             }
-        )
-        ```
-    === "Async API"
+        }
+    );
+    ```
 
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )    
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    s3Express: "true",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            {
-                storageOptions: {
-                    region: "us-east-1",
-                    s3Express: "true",
-                }
-            }
-        );
-        ```
 
 ### Google Cloud Storage
 
@@ -654,64 +349,36 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
 === "Python"
 
     <!-- skip-test -->
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "gs://my-bucket/my-database",
+        storage_options={
+            "service_account": "path/to/service-account.json",
+        }
+    )
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "gs://my-bucket/my-database",
+        {
+            storageOptions: {
+                serviceAccount: "path/to/service-account.json",
             }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
+        }
+    );
+    ```
 
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )    
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "gs://my-bucket/my-database",
-            {
-                storageOptions: {
-                    serviceAccount: "path/to/service-account.json",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "gs://my-bucket/my-database",
-            {
-                storageOptions: {
-                    serviceAccount: "path/to/service-account.json",
-                }
-            }
-        );
-        ```
 
 !!! info "HTTP/2 support"
 
     By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
 
+
 The following keys can be used as both environment variables or keys in the `storage_options` parameter:
 <!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
 
@@ -721,6 +388,7 @@ The following keys can be used as both environment variables or keys in the `sto
 | ``google_service_account_key``        | The serialized service account key.          |
 | ``google_application_credentials``    | Path to the application credentials.         |
 
+
 ### Azure Blob Storage
 
 Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
@@ -728,63 +396,31 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
 === "Python"
 
     <!-- skip-test -->
-    === "Sync API"
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "az://my-container/my-database",
+        storage_options={
+            account_name: "some-account",
+            account_key: "some-key",
+        }
+    )
+    ```
 
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "az://my-container/my-database",
+        {
+            storageOptions: {
+                accountName: "some-account",
+                accountKey: "some-key",
             }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )   
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        import * as lancedb from "@lancedb/lancedb";
-        const db = await lancedb.connect(
-            "az://my-container/my-database",
-            {
-                storageOptions: {
-                    accountName: "some-account",
-                    accountKey: "some-key",
-                }
-            }
-        );
-        ```
-
-    === "vectordb (deprecated)"
-
-        ```ts
-        const lancedb = require("lancedb");
-        const db = await lancedb.connect(
-            "az://my-container/my-database",
-            {
-                storageOptions: {
-                    accountName: "some-account",
-                    accountKey: "some-key",
-                }
-            }
-        );
-        ```
+        }
+    );
+    ```
 
 These keys can be used as both environment variables or keys in the `storage_options` parameter:
 
@@ -809,4 +445,4 @@ These keys can be used as both environment variables or keys in the `storage_opt
 | ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
 | ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
 
-<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->
+<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,8 +1,5 @@
 ## Improving retriever performance
-
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-
-VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retrievers are a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
 
 There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
 
@@ -19,7 +16,7 @@ Using different embedding models is something that's very specific to the use ca
 
 
 ## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv).
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
 
 ### Using different query types
 Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
@@ -45,14 +42,14 @@ table.add(df[["context"]].to_dict(orient="records"))
 queries = df["query"].tolist()
 ```
 
-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset:
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
 
 * <b> Vector Search: </b>
 
     ```python
     table.search(quries[0], query_type="vector").limit(5).to_pandas()
     ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement:
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
 
     ```python
     table.search(quries[0]).limit(5).to_pandas()
@@ -77,7 +74,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
 * <b> Hybrid Search: </b>
 
-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset:
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
     ```python
     table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
     ```
@@ -87,7 +84,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
     !!! note "Note"
         By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/).
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
 
     
 

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -1,6 +1,4 @@
-Continuing from the previous section, we can now rerank the results using more complex rerankers.
-
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Continuing from the previous example, we can now rerank the results using more complex rerankers.
 
 ## Reranking search results
 You can rerank any search results using a reranker. The syntax for reranking is as follows:
@@ -62,6 +60,9 @@ Let us take a look at the same datasets from the previous sections, using the sa
 | Reranked fts  | 0.672    |
 | Hybrid | 0.759 |
 
+### SQuAD Dataset
+
+
 ### Uber10K sec filing Dataset
 
 | Query Type | Hit-rate@5 |

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -1,82 +0,0 @@
-## Finetuning the Embedding Model
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-
-Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
-
-We'll use the same dataset as in the previous sections. Start off by splitting the dataset into training and validation sets:
-```python
-from sklearn.model_selection import train_test_split
-
-train_df, validation_df = train_test_split("data_qa.csv", test_size=0.2, random_state=42)
-
-train_df.to_csv("data_train.csv", index=False)
-validation_df.to_csv("data_val.csv", index=False)
-```
-
-You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
-
-
-We parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node:
-```python
-from llama_index.core.node_parser import SentenceSplitter
-from llama_index.readers.file import PagedCSVReader
-from llama_index.finetuning import generate_qa_embedding_pairs
-from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
-
-def load_corpus(file):
-    loader = PagedCSVReader(encoding="utf-8")
-    docs = loader.load_data(file=Path(file))
-
-    parser = SentenceSplitter()
-    nodes = parser.get_nodes_from_documents(docs)
-
-    return nodes
-
-from llama_index.llms.openai import OpenAI
-
-
-train_dataset = generate_qa_embedding_pairs(
-    llm=OpenAI(model="gpt-3.5-turbo"), nodes=train_nodes, verbose=False
-)
-val_dataset = generate_qa_embedding_pairs(
-    llm=OpenAI(model="gpt-3.5-turbo"), nodes=val_nodes, verbose=False
-)
-```
-
-Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model:
-
-```python
-from llama_index.finetuning import SentenceTransformersFinetuneEngine
-
-finetune_engine = SentenceTransformersFinetuneEngine(
-    train_dataset,
-    model_id="BAAI/bge-small-en-v1.5",
-    model_output_path="tuned_model",
-    val_dataset=val_dataset,
-)
-finetune_engine.finetune()
-embed_model = finetune_engine.get_finetuned_model()
-```
-This saves the fine tuned embedding model in `tuned_model` folder.
-
-# Evaluation results
-In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 
-On performing the same hit-rate evaluation as before, we see a significant improvement in the hit-rate across all query types.
-
-### Baseline
-| Query Type | Hit-rate@5 |
-| --- | --- |
-| Vector Search | 0.640 |
-| Full-text Search | 0.595 |
-| Reranked Vector Search | 0.677 |
-| Reranked Full-text Search | 0.672 |
-| Hybrid Search (w/ CohereReranker) | 0.759|
-
-### Fine-tuned model ( 2 iterations )
-| Query Type | Hit-rate@5 |
-| --- | --- |
-| Vector Search | 0.672 |
-| Full-text Search | 0.595 |
-| Reranked Vector Search | 0.754 |
-| Reranked Full-text Search | 0.672|
-| Hybrid Search (w/ CohereReranker) | 0.768 |

docs/src/hybrid_search/eval.md

@@ -3,22 +3,22 @@
 Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
 
 ## The challenge of (re)ranking search results
-Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step: reranking.
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
 There are two approaches for reranking search results from multiple sources.
 
-* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example: Weighted linear combination of semantic search & keyword-based search results.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
 
-* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
 
-Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset or application specific so it's hard to generalize.
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
 
 ### Example evaluation of hybrid search with Reranking
 
-Here's some evaluation numbers from an experiment comparing these rerankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
 
 <b> With OpenAI ada2 embedding </b>
 
-Vector Search baseline: `0.64`
+Vector Search baseline - `0.64`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |
@@ -33,7 +33,7 @@ Vector Search baseline: `0.64`
 
 <b> With OpenAI embedding-v3-small </b>
 
-Vector Search baseline: `0.59`
+Vector Search baseline - `0.59`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |

docs/src/hybrid_search/hybrid_search.md

@@ -5,59 +5,238 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
 
-=== "Sync API"
+```python
+import os
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:import-os"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
-    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search"
-    ```
-=== "Async API"
+import lancedb
+import openai
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:import-os"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
-    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search_async"
-    ```
+db = lancedb.connect("~/.lancedb")
 
-!!! Note
-    You can also pass the vector and text query manually. This is useful if you're not using the embedding API or if you're using a separate embedder service.
-### Explicitly passing the vector and text query
-=== "Sync API"
+# Ingest embedding function in LanceDB table
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+    openai.api_key = "sk-..."
+embeddings = get_registry().get("openai").create()
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text"
-    ```
-=== "Async API"
+class Documents(LanceModel):
+    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+    text: str = embeddings.SourceField()
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text_async"
-    ```
+table = db.create_table("documents", schema=Documents)
 
-By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
+data = [
+    { "text": "rebel spaceships striking from a hidden base"},
+    { "text": "have won their first victory against the evil Galactic Empire"},
+    { "text": "during the battle rebel spies managed to steal secret plans"},
+    { "text": "to the Empire's ultimate weapon the Death Star"}
+]
+
+# ingest docs with auto-vectorization
+table.add(data)
+
+# Create a fts index before the hybrid search
+table.create_fts_index("text")
+# hybrid search with default re-ranker
+results = table.search("flower moon", query_type="hybrid").to_pandas()
+```
+
+By default, LanceDB uses `LinearCombinationReranker(weight=0.7)` to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
 
 
 ### `rerank()` arguments
 * `normalize`: `str`, default `"score"`:
     The method to normalize the scores. Can be "rank" or "score". If "rank", the scores are converted to ranks and then normalized. If "score", the scores are normalized directly.
-* `reranker`: `Reranker`, default `RRF()`.
+* `reranker`: `Reranker`, default `LinearCombinationReranker(weight=0.7)`.
     The reranker to use. If not specified, the default reranker is used.
 
 
 ## Available Rerankers
-LanceDB provides a number of rerankers out of the box. You can use any of these rerankers by passing them to the `rerank()` method. 
-Go to [Rerankers](../reranking/index.md) to learn more about using the available rerankers and implementing custom rerankers.
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Here's a list of available re-rankers:
+
+### Linear Combination Reranker
+This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
 
 
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search
+
+results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `weight`: `float`, default `0.7`:
+    The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
+* `fill`: `float`, default `1.0`:
+        The score to give to results that are only in one of the two result sets.This is treated as penalty, so a higher value means a lower score.
+        TODO: We should just hardcode this-- its pretty confusing as we invert scores to calculate final score
+* `return_score` : str, default `"relevance"`
+        options are "relevance" or "all"
+        The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score.
+
+### Cohere Reranker
+This re-ranker uses the [Cohere](https://cohere.ai/) API to combine the results of semantic and full-text search. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll need to set the `COHERE_API_KEY` environment variable to use this re-ranker.
+
+```python
+from lancedb.rerankers import CohereReranker
+
+reranker = CohereReranker()
+
+results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : str, default `"rerank-english-v2.0"`
+        The name of the cross encoder model to use. Available cohere models are:
+        - rerank-english-v2.0
+        - rerank-multilingual-v2.0
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `top_n` : str, default `None`
+        The number of results to return. If None, will return all results.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### Cross Encoder Reranker
+This reranker uses the [Sentence Transformers](https://www.sbert.net/) library to combine the results of semantic and full-text search. You can use it by passing `CrossEncoderReranker()` to the `rerank()` method.
+
+```python
+from lancedb.rerankers import CrossEncoderReranker
+
+reranker = CrossEncoderReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+
+### Arguments
+----------------
+* `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
+        The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `device` : str, default `None`
+        The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu".
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+
+### ColBERT Reranker
+This reranker uses the ColBERT model to combine the results of semantic and full-text search. You can use it by passing `ColbertrReranker()` to the `rerank()` method. 
+
+ColBERT reranker model calculates relevance of given docs against the query and don't take existing fts and vector search scores into account, so it currently only supports `return_score="relevance"`. By default, it looks for `text` column to rerank the results. But you can specify the column name to use as input to the cross encoder model as described below.
+
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
+        The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+        options are `"relevance"` or `"all"`. Only `"relevance"` is supported for now.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### OpenAI Reranker
+This reranker uses the OpenAI API to combine the results of semantic and full-text search. You can use it by passing `OpenaiReranker()` to the `rerank()` method.
+
+!!! Note
+    This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
+
+!!! Tip
+    - You might run out of token limit so set the search `limits` based on your token limit.
+    - It is recommended to use gpt-4-turbo-preview, the default model, older models might lead to undesired behaviour
+
+```python
+from lancedb.rerankers import OpenaiReranker
+
+reranker = OpenaiReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : `str`, default `"gpt-4-turbo-preview"`
+    The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+    The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+    options are "relevance" or "all". Only "relevance" is supported for now.
+* `api_key` : `str`, default `None`
+    The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+
+
+## Building Custom Rerankers
+You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
+
+The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
+
+```python
+
+from lancedb.rerankers import Reranker
+import pyarrow as pa
+
+class MyReranker(Reranker):
+    def __init__(self, param1, param2, ..., return_score="relevance"):
+        super().__init__(return_score)
+        self.param1 = param1
+        self.param2 = param2
+    
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
+        # Use the built-in merging function
+        combined_result = self.merge_results(vector_results, fts_results)
+        
+        # Do something with the combined results
+        # ...
+
+        # Return the combined results
+        return combined_result
+
+```
+
+### Example of a Custom Reranker
+For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
+
+```python
+
+from typing import List, Union
+import pandas as pd
+from lancedb.rerankers import CohereReranker
+
+class MofidifiedCohereReranker(CohereReranker):
+    def __init__(self, filters: Union[str, List[str]], **kwargs):
+        super().__init__(**kwargs)
+        filters = filters if isinstance(filters, list) else [filters]
+        self.filters = filters
+
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
+        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
+        df = combined_result.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
+
+```
+
+!!! tip
+    The `vector_results` and `fts_results` are pyarrow tables. You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/index.md

@@ -49,8 +49,7 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Working with tables](guides/tables.md): Learn how to work with tables and their associated functions
 * [Indexing](ann_indexes.md): Understand how to create indexes
 * [Vector search](search.md): Learn how to perform vector similarity search
-* [Full-text search (native)](fts.md): Learn how to perform full-text search
-* [Full-text search (tantivy-based)](fts_tantivy.md): Learn how to perform full-text search using Tantivy
+* [Full-text search](fts.md): Learn how to perform full-text search
 * [Managing embeddings](embeddings/index.md): Managing embeddings and the embedding functions API in LanceDB
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references

docs/src/integrations/dlt.md

@@ -1,142 +0,0 @@
-# dlt  
-
-[dlt](https://dlthub.com/docs/intro) is an open-source library that you can add to your Python scripts to load data from various and often messy data sources into well-structured, live datasets. dlt's [integration with LanceDB](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb) lets you ingest data from any source (databases, APIs, CSVs, dataframes, JSONs, and more) into LanceDB with a few lines of simple python code. The integration enables automatic normalization of nested data, schema inference, incremental loading and embedding the data. dlt also has integrations with several other tools like dbt, airflow, dagster etc. that can be inserted into your LanceDB workflow.
-
-## How to ingest data into LanceDB  
-
-In this example, we will be fetching movie information from the [Open Movie Database (OMDb) API](https://www.omdbapi.com/) and loading it into a local LanceDB instance. To implement it, you will need an API key for the OMDb API (which can be created freely [here](https://www.omdbapi.com/apikey.aspx)).
-  
-1. **Install `dlt` with LanceDB extras:**  
-    ```sh 
-    pip install dlt[lancedb]
-    ```
-
-2. **Inside an empty directory, initialize a `dlt` project with:**  
-    ```sh
-    dlt init rest_api lancedb
-    ```
-    This will add all the files necessary to create a `dlt` pipeline that can ingest data from any REST API (ex: OMDb API) and load into LanceDB.
-    ```text
-    ├── .dlt
-    │   ├── config.toml
-    │   └── secrets.toml
-    ├── rest_api
-    ├── rest_api_pipeline.py
-    └── requirements.txt
-    ```
-  
-    dlt has a list of pre-built [sources](https://dlthub.com/docs/dlt-ecosystem/verified-sources/) like [SQL databases](https://dlthub.com/docs/dlt-ecosystem/verified-sources/sql_database), [REST APIs](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api), [Google Sheets](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_sheets), [Notion](https://dlthub.com/docs/dlt-ecosystem/verified-sources/notion) etc., that can be used out-of-the-box by running `dlt init <source_name> lancedb`. Since dlt is a python library, it is also very easy to modify these pre-built sources or to write your own custom source from scratch.
-
-
-3. **Specify necessary credentials and/or embedding model details:**  
-    
-    In order to fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB cloud, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/sercrets.toml`. 
-
-    dlt's LanceDB integration also allows you to automatically embed the data during ingestion. Depending on the embedding model chosen, you may need to paste the necessary credentials inside `.dlt/sercrets.toml`:
-    ```toml
-    [sources.rest_api]
-    api_key = "api_key" # Enter the API key for the OMDb API
-
-    [destination.lancedb]
-    embedding_model_provider = "sentence-transformers"
-    embedding_model = "all-MiniLM-L6-v2"
-    [destination.lancedb.credentials]
-    uri = ".lancedb"
-    api_key = "api_key" # API key to connect to LanceDB Cloud. Leave out if you are using LanceDB OSS.
-    embedding_model_provider_api_key = "embedding_model_provider_api_key" # Not needed for providers that don't need authentication (ollama, sentence-transformers).
-    ```
-    See [here](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb#configure-the-destination) for more information and for a list of available models and model providers.  
-
-
-4. **Write the pipeline code inside `rest_api_pipeline.py`:**  
-
-    The following code shows how you can configure dlt's REST API source to connect to the [OMDb API](https://www.omdbapi.com/), fetch all movies with the word "godzilla" in the title, and load it into a LanceDB table. The REST API source allows you to pull data from any API with minimal code, to learn more read the [dlt docs](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api).
-
-    ```python   
-
-    # Import necessary modules
-    import dlt
-    from rest_api import rest_api_source
-
-    # Configure the REST API source
-    movies_source = rest_api_source(
-        {
-            "client": {
-                "base_url": "https://www.omdbapi.com/",
-                "auth": { # authentication strategy for the OMDb API
-                    "type": "api_key",
-                    "name": "apikey",
-                    "api_key": dlt.secrets["sources.rest_api.api_token"], # read API credentials directly from secrets.toml
-                    "location": "query"
-                },
-                "paginator": { # pagination strategy for the OMDb API 
-                    "type": "page_number",
-                    "base_page": 1,
-                    "total_path": "totalResults",
-                    "maximum_page": 5
-                }
-            },
-            "resources": [ # list of API endpoints to request
-                {
-                    "name": "movie_search",
-                    "endpoint": {
-                        "path": "/",
-                        "params": {
-                            "s": "godzilla",
-                            "type": "movie"
-                        }
-                    }
-                }
-            ]
-        })
-
-
-    if __name__ == "__main__":
-        # Create a pipeline object
-        pipeline = dlt.pipeline(
-            pipeline_name='movies_pipeline',
-            destination='lancedb', # this tells dlt to load the data into LanceDB
-            dataset_name='movies_data_pipeline',
-        )
-
-        # Run the pipeline
-        load_info = pipeline.run(movies_source)
-
-        # pretty print the information on data that was loaded
-        print(load_info)
-    ```
-
-    The script above will ingest the data into LanceDB as it is, i.e. without creating any embeddings. If we want to embed one of the fields (for example, `"Title"` that contains the movie titles), then we will use dlt's `lancedb_adapter` and modify the script as follows:  
-    
-    - Add the following import statement:
-        ```python
-        from dlt.destinations.adapters import lancedb_adapter
-        ```
-    - Modify the pipeline run like this:
-        ```python
-        load_info = pipeline.run(
-            lancedb_adapter(
-                movies_source,
-                embed="Title",
-            )
-        )
-        ```
-    This will use the embedding model specified inside `.dlt/secrets.toml` to embed the field `"Title"`.
-
-5. **Install necessary dependencies:**  
-    ```sh
-    pip install -r requirements.txt
-    ```  
-
-    Note: You may need to install the dependencies for your embedding models separately.
-    ```sh
-    pip install sentence-transformers
-    ```
-
-6. **Run the pipeline:**
-    Finally, running the following command will ingest the data into your LanceDB instance.
-    ```sh
-    python custom_source.py
-    ```
-
-For more information and advanced usage of dlt's LanceDB integration, read [the dlt documentation](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb).

docs/src/integrations/langchain.md

@@ -1,13 +1,8 @@
-**LangChain** is a framework designed for building applications with large language models (LLMs) by chaining together various components. It supports a range of functionalities including memory, agents, and chat models, enabling developers to create context-aware applications.
-
-![Illustration](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/langchain_rag.png)
-
-LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities.
-
-Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs.
+# Langchain
+![Illustration](../assets/langchain.png)
 
 ## Quick Start
-You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. Checkout Complete example here - [LangChain demo](../notebooks/langchain_example.ipynb)
+You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model.
 ```python
 import os
 from langchain.document_loaders import TextLoader
@@ -31,28 +26,18 @@ print(docs[0].page_content)
 
 ## Documentation
 In the above example `LanceDB` vector store class object is created using `from_documents()` method  which is a `classmethod` and returns the initialized class object. 
-
 You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method.  
 
-The exhaustive list of parameters for `LanceDB` vector store are : 
-
-|Name|type|Purpose|default|
-|:----|:----|:----|:----|
-|`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.|`None`|
-|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.|
-|`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`|
-|`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`|
-|`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`|
-|`text_key` |(Optional) `str` |Column name to use for text in the table.|`'text'`|
-|`table_name` |(Optional) `str`| Name of your table in the database.|`'vectorstore'`|
-|`api_key` |(Optional `str`) |API key to use for LanceDB cloud database.|`None`|
-|`region` |(Optional) `str`| Region to use for LanceDB cloud database.|Only for LanceDB Cloud : `None`.|
-|`mode` |(Optional) `str` |Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`|
-|`table`| (Optional) `Any`|You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`|
-|`distance`|(Optional) `str`|The choice of distance metric used to calculate the similarity between vectors.|`'l2'`|
-|`reranker` |(Optional) `Any`|The reranker to use for LanceDB.|`None`|
-|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`|
-|`limit`|`int`|Set the maximum number of results to return.|`DEFAULT_K` (it is 4)|
+The exhaustive list of parameters for `LanceDB` vector store are :  
+- `connection`: (Optional) `lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.  
+- `embedding`: Langchain embedding model.  
+- `vector_key`: (Optional) Column name to use for vector's in the table. Defaults to `'vector'`.   
+- `id_key`: (Optional) Column name to use for id's in the table. Defaults to `'id'`.  
+- `text_key`: (Optional) Column name to use for text in the table. Defaults to `'text'`.  
+- `table_name`: (Optional) Name of your table in the database. Defaults to `'vectorstore'`.  
+- `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
+- `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
+- `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
 
 ```python
 db_url = "db://lang_test" # url of db you created
@@ -64,24 +49,17 @@ vector_store = LanceDB(
     api_key=api_key, #(dont include for local API)
     region=region, #(dont include for local API)
     embedding=embeddings,
-    table_name='langchain_test' # Optional
+    table_name='langchain_test' #Optional
     )
 ```
 
 ### Methods 
-
+To add texts and store respective embeddings automatically:   
 ##### add_texts()
+- `texts`: `Iterable` of strings to add to the vectorstore.
+- `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
+- `ids`: Optional `list` of ids to associate with the texts. 
 
-This method turn texts into embedding and add it to the database.
-
-|Name|Purpose|defaults|
-|:---|:---|:---|
-|`texts`|`Iterable` of strings to add to the vectorstore.|Provided by user|
-|`metadatas`|Optional `list[dict()]` of metadatas associated with the texts.|`None`|
-|`ids`|Optional `list` of ids to associate with the texts.|`None`|
-|`kwargs`| Other keyworded arguments provided by the user. |-|
-
-It returns list of ids of the added texts.
 
 ```python
 vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
@@ -96,25 +74,13 @@ pd_df.to_csv("docsearch.csv", index=False)
 # you can also create a new vector store object using an older connection object:
 vector_store = LanceDB(connection=tbl, embedding=embeddings)
 ```
-
-------
-
-
-##### create_index() 
-
-This method creates a scalar(for non-vector cols) or a vector index on a table.
-
-|Name|type|Purpose|defaults|
-|:---|:---|:---|:---|
-|`vector_col`|`Optional[str]`| Provide if you want to create index on a vector column. |`None`|
-|`col_name`|`Optional[str]`| Provide if you want to create index on a non-vector column. |`None`|
-|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'L2', 'dot', 'cosine'. |`L2`|
-|`num_partitions`|`Optional[int]`|Number of partitions to use for the index.|`256`|
-|`num_sub_vectors`|`Optional[int]` |Number of sub-vectors to use for the index.|`96`|
-|`index_cache_size`|`Optional[int]` |Size of the index cache.|`None`|
-|`name`|`Optional[str]` |Name of the table to create index on.|`None`|
-
 For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+##### create_index() 
+- `col_name`: `Optional[str] = None`
+- `vector_col`: `Optional[str] = None`
+- `num_partitions`: `Optional[int] = 256`
+- `num_sub_vectors`: `Optional[int] = 96`
+- `index_cache_size`: `Optional[int] = None`
 
 ```python
 # for creating vector index
@@ -123,144 +89,4 @@ vector_store.create_index(vector_col='vector', metric = 'cosine')
 # for creating scalar index(for non-vector columns)
 vector_store.create_index(col_name='text')
 
-```
-
-------
-
-##### similarity_search()
-
-This method performs similarity search based on **text query**.
-
-| Name    | Type                 | Purpose | Default |
-|---------|----------------------|---------|---------|
-| `query` | `str`                |  A `str` representing the text query that you want to search for in the vector store.         | N/A     |
-| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
-| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
-| `fts`   | `Optional[bool]`     |   It indicates whether to perform a full-text search (FTS).      | `False`   |
-| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
-| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
-
-Return documents most similar to the query **without relevance scores**.
-
-```python
-docs = docsearch.similarity_search(query)
-print(docs[0].page_content)
-```
-
-------
-
-##### similarity_search_by_vector()
-
-The method returns documents that are most similar to the specified **embedding (query) vector**. 
-
-| Name        | Type                      | Purpose | Default |
-|-------------|---------------------------|---------|---------|
-| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.         | N/A     |
-| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
-| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
-| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
-| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
-
-**It does not provide relevance scores.**
-
-```python
-docs = docsearch.similarity_search_by_vector(query)
-print(docs[0].page_content)
-```
-
-------
-
-##### similarity_search_with_score()
-
-Returns documents most similar to the **query string** along with their relevance scores.
-
-| Name     | Type                      | Purpose | Default |
-|----------|---------------------------|---------|---------|
-| `query`  | `str`                     |A `str` representing the text query you want to search for in the vector store. This query will be converted into an embedding using the specified embedding function.         | N/A     |
-| `k`      | `Optional[int]`           | It specifies the number of documents to return.  | `None`    |
-| `filter` | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria. This allows you to narrow down the search results based on certain metadata attributes associated with the documents.       | `None`    |
-| `kwargs` | `Any`                     |  Other keyworded arguments provided by the user.       | N/A     |
-
-It gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
-
-```python
-docs = docsearch.similarity_search_with_relevance_scores(query)
-print("relevance score - ", docs[0][1])
-print("text- ", docs[0][0].page_content[:1000])
-```
-
-------
-
-##### similarity_search_by_vector_with_relevance_scores()
-
-Similarity search using **query vector**.
-
-| Name        | Type                      | Purpose | Default |
-|-------------|---------------------------|---------|---------|
-| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.        | N/A     |
-| `k`         | `Optional[int]`           |   It specifies the number of documents to return.       | `None`    |
-| `filter`    | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria.       | `None`    |
-| `name`      | `Optional[str]`           |   It is used for specifying the name of the table to query.       | `None`    |
-| `kwargs`    | `Any`                     |   Other keyworded arguments provided by the user.      | N/A     |
-
-The method returns documents most similar to the specified embedding (query) vector, along with their relevance scores.
-
-```python
-docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)
-print("relevance score - ", docs[0][1])
-print("text- ", docs[0][0].page_content[:1000])
-```
-
-------
-
-##### max_marginal_relevance_search()
-
-This method returns docs selected using the maximal marginal relevance(MMR).
-Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.
-
-| Name | Type | Purpose | Default |
-|---------------|-----------------|-----------|---------|
-| `query`       | `str` | Text to look up documents similar to. | N/A     |
-| `k`   | `Optional[int]` | Number of Documents to return.| `4`       |
-| `fetch_k`| `Optional[int]`| Number of Documents to fetch to pass to MMR algorithm.| `None`    |
-| `lambda_mult` | `float`                   | Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. | `0.5`     |
-| `filter`| `Optional[Dict[str, str]]`| Filter by metadata. | `None`    |
-|`kwargs`| Other keyworded arguments provided by the user. |-|
-
-Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 
-
-```python
-result = docsearch.max_marginal_relevance_search(
-        query="text"
-    )
-result_texts = [doc.page_content for doc in result]
-print(result_texts)
-
-## search by vector :
-result = docsearch.max_marginal_relevance_search_by_vector(
-        embeddings.embed_query("text")
-    )
-result_texts = [doc.page_content for doc in result]
-print(result_texts)
-```
-
-------
-
-##### add_images()
-
-This method ddds images by automatically creating their embeddings and adds them to the vectorstore.
-
-| Name       | Type                          | Purpose                        | Default |
-|------------|-------------------------------|--------------------------------|---------|
-| `uris`     | `List[str]`                   | File path to the image         | N/A     |
-| `metadatas`| `Optional[List[dict]]`        | Optional list of metadatas     | `None`    |
-| `ids`      | `Optional[List[str]]`         | Optional list of IDs           | `None`    |
-
-It returns list of IDs of the added images.
-
-```python
-vec_store.add_images(uris=image_uris) 
-# here image_uris are local fs paths to the images.
-```
-
-
+```

docs/src/integrations/llamaIndex.md

@@ -2,8 +2,7 @@
 ![Illustration](../assets/llama-index.jpg)
 
 ## Quick start
-You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. 
-You can run the below script to try it out :
+You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. You can run the below script to try it out :
 ```python
 import logging
 import sys
@@ -44,8 +43,6 @@ retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter})
 response = retriever.retrieve("What did the author do growing up?")
 ```
 
-Checkout Complete example here - [LlamaIndex demo](../notebooks/LlamaIndex_example.ipynb)
-
 ### Filtering
 For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
 ```python

docs/src/integrations/phidata.md

@@ -1,383 +0,0 @@
-**phidata** is a framework for building **AI Assistants** with long-term memory, contextual knowledge, and the ability to take actions using function calling. It helps turn general-purpose LLMs into specialized assistants tailored to your use case by extending its capabilities using **memory**, **knowledge**, and **tools**. 
-
-- **Memory**: Stores chat history in a **database** and enables LLMs to have long-term conversations.
-- **Knowledge**: Stores information in a **vector database** and provides LLMs with business context. (Here we will use LanceDB)
-- **Tools**: Enable LLMs to take actions like pulling data from an **API**, **sending emails** or **querying a database**, etc.
-
-![example](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/phidata_assistant.png)
-
-Memory & knowledge make LLMs smarter while tools make them autonomous.
-
-LanceDB is a vector database and its integration into phidata makes it easy for us to provide a **knowledge base** to LLMs. It enables us to store information as [embeddings](../embeddings/understanding_embeddings.md) and search for the **results** similar to ours using **query**. 
-
-??? Question "What is Knowledge Base?"
-    Knowledge Base is a database of information that the Assistant can search to improve its responses. This information is stored in a vector database and provides LLMs with business context, which makes them respond in a context-aware manner.
-
-    While any type of storage can act as a knowledge base, vector databases offer the best solution for retrieving relevant results from dense information quickly.
-
-Let's see how using LanceDB inside phidata helps in making LLM more useful:
-
-## Prerequisites: install and import necessary dependencies
-
-**Create a virtual environment**
-
-1. install virtualenv package
-    ```python
-    pip install virtualenv
-    ```
-2. Create a directory for your project and go to the directory and create a virtual environment inside it.
-    ```python
-    mkdir phi
-    ```
-    ```python
-    cd phi
-    ```
-    ```python
-    python -m venv phidata_
-    ```
-
-**Activating virtual environment**
-
-1. from inside the project directory, run the following command to activate the virtual environment.
-    ```python
-    phidata_/Scripts/activate
-    ```
-
-**Install the following packages in the virtual environment**
-```python
-pip install lancedb phidata youtube_transcript_api openai ollama numpy pandas
-```
-
-**Create python files and import necessary libraries**
-
-You need to create two files - `transcript.py` and `ollama_assistant.py` or `openai_assistant.py`
-
-=== "openai_assistant.py"
-
-    ```python
-    import os, openai
-    from rich.prompt import Prompt
-    from phi.assistant import Assistant
-    from phi.knowledge.text import TextKnowledgeBase
-    from phi.vectordb.lancedb import LanceDb
-    from phi.llm.openai import OpenAIChat
-    from phi.embedder.openai import OpenAIEmbedder
-    from transcript import extract_transcript
-
-    if "OPENAI_API_KEY" not in os.environ:
-    # OR set the key here as a variable
-        openai.api_key = "sk-..."
-
-    # The code below creates a file "transcript.txt" in the directory, the txt file will be used below
-    youtube_url = "https://www.youtube.com/watch?v=Xs33-Gzl8Mo" 
-    segment_duration = 20
-    transcript_text,dict_transcript = extract_transcript(youtube_url,segment_duration)
-    ```
-
-=== "ollama_assistant.py"
-
-    ```python
-    from rich.prompt import Prompt
-    from phi.assistant import Assistant
-    from phi.knowledge.text import TextKnowledgeBase
-    from phi.vectordb.lancedb import LanceDb
-    from phi.llm.ollama import Ollama
-    from phi.embedder.ollama import OllamaEmbedder
-    from transcript import extract_transcript
-
-    # The code below creates a file "transcript.txt" in the directory, the txt file will be used below
-    youtube_url = "https://www.youtube.com/watch?v=Xs33-Gzl8Mo"
-    segment_duration = 20
-    transcript_text,dict_transcript = extract_transcript(youtube_url,segment_duration)
-    ```
-
-=== "transcript.py"
-
-    ``` python
-    from youtube_transcript_api import YouTubeTranscriptApi
-    import re
-
-    def smodify(seconds):
-        hours, remainder = divmod(seconds, 3600)
-        minutes, seconds = divmod(remainder, 60)
-        return f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}"
-        
-    def extract_transcript(youtube_url,segment_duration):
-        # Extract video ID from the URL
-        video_id = re.search(r'(?<=v=)[\w-]+', youtube_url)
-        if not video_id:
-            video_id = re.search(r'(?<=be/)[\w-]+', youtube_url)
-        if not video_id:
-            return None
-
-        video_id = video_id.group(0)
-
-        # Attempt to fetch the transcript
-        try:
-            # Try to get the official transcript
-            transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=['en'])
-        except Exception:
-            # If no official transcript is found, try to get auto-generated transcript
-            try:
-                transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-                for transcript in transcript_list:
-                    transcript = transcript.translate('en').fetch()
-            except Exception:
-                return None
-
-        # Format the transcript into 120s chunks
-        transcript_text,dict_transcript = format_transcript(transcript,segment_duration)
-        # Open the file in write mode, which creates it if it doesn't exist
-        with open("transcript.txt", "w",encoding="utf-8") as file:
-            file.write(transcript_text)
-        return transcript_text,dict_transcript
-
-    def format_transcript(transcript,segment_duration):
-        chunked_transcript = []
-        chunk_dict = []
-        current_chunk = []
-        current_time = 0
-        # 2 minutes in seconds
-        start_time_chunk = 0  # To track the start time of the current chunk
-
-        for segment in transcript:
-            start_time = segment['start']
-            end_time_x = start_time + segment['duration']
-            text = segment['text']
-            
-            # Add text to the current chunk
-            current_chunk.append(text)
-            
-            # Update the current time with the duration of the current segment
-            # The duration of the current segment is given by segment['start'] - start_time_chunk
-            if current_chunk:
-                current_time = start_time - start_time_chunk
-            
-            # If current chunk duration reaches or exceeds 2 minutes, save the chunk
-            if current_time >= segment_duration:
-                # Use the start time of the first segment in the current chunk as the timestamp
-                chunked_transcript.append(f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}] " + " ".join(current_chunk))
-                current_chunk = re.sub(r'[\xa0\n]', lambda x: '' if x.group() == '\xa0' else ' ', "\n".join(current_chunk))
-                chunk_dict.append({"timestamp":f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}]", "text": "".join(current_chunk)})
-                current_chunk = []  # Reset the chunk
-                start_time_chunk = start_time + segment['duration'] # Update the start time for the next chunk
-                current_time = 0  # Reset current time
-
-        # Add any remaining text in the last chunk
-        if current_chunk:
-            chunked_transcript.append(f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}] " + " ".join(current_chunk))
-            current_chunk = re.sub(r'[\xa0\n]', lambda x: '' if x.group() == '\xa0' else ' ', "\n".join(current_chunk))
-            chunk_dict.append({"timestamp":f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}]", "text": "".join(current_chunk)})
-
-        return "\n\n".join(chunked_transcript), chunk_dict
-    ```
-
-!!! warning
-    If creating Ollama assistant, download and install Ollama [from here](https://ollama.com/) and then run the Ollama instance in the background. Also, download the required models using `ollama pull <model-name>`. Check out the models [here](https://ollama.com/library) 
-
-
-**Run the following command to deactivate the virtual environment if needed**
-```python
-deactivate
-```
-
-## **Step 1** - Create a Knowledge Base for AI Assistant using LanceDB
-
-=== "openai_assistant.py"
-
-    ```python
-    # Create knowledge Base with OpenAIEmbedder in LanceDB
-    knowledge_base = TextKnowledgeBase(
-        path="transcript.txt",
-        vector_db=LanceDb(
-            embedder=OpenAIEmbedder(api_key = openai.api_key),
-            table_name="transcript_documents",
-            uri="./t3mp/.lancedb",
-        ),
-        num_documents = 10
-    )
-    ```
-
-=== "ollama_assistant.py"
-
-    ```python
-    # Create knowledge Base with OllamaEmbedder in LanceDB
-    knowledge_base = TextKnowledgeBase(
-        path="transcript.txt",
-        vector_db=LanceDb(
-            embedder=OllamaEmbedder(model="nomic-embed-text",dimensions=768),
-            table_name="transcript_documents",
-            uri="./t2mp/.lancedb",
-        ),
-        num_documents = 10
-    )
-    ```
-Check out the list of **embedders** supported by **phidata** and their usage [here](https://docs.phidata.com/embedder/introduction).
-
-Here we have used `TextKnowledgeBase`, which loads text/docx files to the knowledge base.
-
-Let's see all the parameters that `TextKnowledgeBase` takes -
-
-| Name| Type | Purpose | Default |
-|:----|:-----|:--------|:--------|
-|`path`|`Union[str, Path]`| Path to text file(s). It can point to a single text file or a directory of text files.| provided by user |
-|`formats`|`List[str]`| File formats accepted by this knowledge base. |`[".txt"]`|
-|`vector_db`|`VectorDb`| Vector Database for the Knowledge Base. phidata provides a wrapper around many vector DBs, you can import it like this - `from phi.vectordb.lancedb import LanceDb` | provided by user |
-|`num_documents`|`int`| Number of results (documents/vectors) that vector search should return. |`5`|
-|`reader`|`TextReader`| phidata provides many types of reader objects which read data, clean it and create chunks of data, encapsulate each chunk inside an object of the `Document` class, and return **`List[Document]`**.  | `TextReader()` |
-|`optimize_on`|`int`| It is used to specify the number of documents on which to optimize the vector database. Supposed to create an index. |`1000`|
-
-??? Tip "Wonder! What is `Document` class?"
-    We know that, before storing the data in vectorDB, we need to split the data into smaller chunks upon which embeddings will be created and these embeddings along with the chunks will be stored in vectorDB. When the user queries over the vectorDB, some of these embeddings will be returned as the result based on the semantic similarity with the query.
-    
-    When the user queries over vectorDB, the queries are converted into embeddings, and a nearest neighbor search is performed over these query embeddings which returns the embeddings that correspond to most semantically similar chunks(parts of our data) present in vectorDB. 
-
-    Here, a “Document” is a class in phidata. Since there is an option to let phidata create and manage embeddings, it splits our data into smaller chunks(as expected). It does not directly create embeddings on it. Instead, it takes each chunk and encapsulates it inside the object of the `Document` class along with various other metadata related to the chunk. Then embeddings are created on these `Document` objects and stored in vectorDB.
-
-    ```python
-    class Document(BaseModel):
-        """Model for managing a document"""
-
-        content: str # <--- here data of chunk is stored 
-        id: Optional[str] = None
-        name: Optional[str] = None
-        meta_data: Dict[str, Any] = {}
-        embedder: Optional[Embedder] = None
-        embedding: Optional[List[float]] = None
-        usage: Optional[Dict[str, Any]] = None
-    ```
-
-However, using phidata you can load many other types of data in the knowledge base(other than text). Check out [phidata Knowledge Base](https://docs.phidata.com/knowledge/introduction) for more information.
-
-Let's dig deeper into the `vector_db` parameter and see what parameters `LanceDb` takes -
-
-| Name| Type | Purpose | Default |
-|:----|:-----|:--------|:--------|
-|`embedder`|`Embedder`| phidata provides many Embedders that abstract the interaction with embedding APIs and utilize it to generate embeddings. Check out other embedders [here](https://docs.phidata.com/embedder/introduction) | `OpenAIEmbedder` |
-|`distance`|`List[str]`| The choice of distance metric used to calculate the similarity between vectors, which directly impacts search results and performance in vector databases. |`Distance.cosine`|
-|`connection`|`lancedb.db.LanceTable`| LanceTable can be accessed through `.connection`. You can connect to an existing table of LanceDB, created outside of phidata, and utilize it. If not provided, it creates a new table using `table_name` parameter and adds it to `connection`. |`None`|
-|`uri`|`str`| It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. | `"/tmp/lancedb"` |
-|`table_name`|`str`|  If `connection` is not provided, it initializes and connects to a new **LanceDB table** with a specified(or default) name in the database present at `uri`. |`"phi"`|
-|`nprobes`|`int`| It refers to the number of partitions that the search algorithm examines to find the nearest neighbors of a given query vector. Higher values will yield better recall (more likely to find vectors if they exist) at the expense of latency. |`20`|
-
-
-!!! note
-    Since we just initialized the KnowledgeBase. The VectorDB table that corresponds to this Knowledge Base is not yet populated with our data. It will be populated in **Step 3**, once we perform the `load` operation. 
-    
-    You can check the state of the LanceDB table using - `knowledge_base.vector_db.connection.to_pandas()`
-
-Now that the Knowledge Base is initialized, , we can go to **step 2**.
-
-## **Step 2** -  Create an assistant with our choice of LLM and reference to the knowledge base.
-
-
-=== "openai_assistant.py"
-
-    ```python
-    # define an assistant with gpt-4o-mini llm and reference to the knowledge base created above
-    assistant = Assistant(
-        llm=OpenAIChat(model="gpt-4o-mini", max_tokens=1000, temperature=0.3,api_key = openai.api_key),
-        description="""You are an Expert in explaining youtube video transcripts. You are a bot that takes transcript of a video and answer the question based on it.
-        
-        This is transcript for the above timestamp: {relevant_document}
-        The user input is: {user_input}
-        generate highlights only when asked.
-        When asked to generate highlights from the video, understand the context for each timestamp and create key highlight points, answer in following way - 
-        [timestamp] - highlight 1
-        [timestamp] - highlight 2
-        ... so on
-        
-        Your task is to understand the user question, and provide an answer using the provided contexts. Your answers are correct, high-quality, and written by an domain expert. If the provided context does not contain the answer, simply state,'The provided context does not have the answer.'""",
-        knowledge_base=knowledge_base,
-        add_references_to_prompt=True,
-    )
-    ```
-
-=== "ollama_assistant.py"
-
-    ```python
-    # define an assistant with llama3.1 llm and reference to the knowledge base created above
-    assistant = Assistant(
-        llm=Ollama(model="llama3.1"),
-        description="""You are an Expert in explaining youtube video transcripts. You are a bot that takes transcript of a video and answer the question based on it.
-
-        This is transcript for the above timestamp: {relevant_document}
-        The user input is: {user_input}
-        generate highlights only when asked.
-        When asked to generate highlights from the video, understand the context for each timestamp and create key highlight points, answer in following way - 
-        [timestamp] - highlight 1
-        [timestamp] - highlight 2
-        ... so on
-
-        Your task is to understand the user question, and provide an answer using the provided contexts. Your answers are correct, high-quality, and written by an domain expert. If the provided context does not contain the answer, simply state,'The provided context does not have the answer.'""",
-        knowledge_base=knowledge_base,
-        add_references_to_prompt=True,
-    )
-    ```
-
-Assistants add **memory**, **knowledge**, and **tools** to LLMs. Here we will add only **knowledge** in this example. 
-
-Whenever we will give a query to LLM, the assistant will retrieve relevant information from our **Knowledge Base**(table in LanceDB) and pass it to LLM along with the user query in a structured way. 
-
-- The `add_references_to_prompt=True` always adds information from the knowledge base to the prompt, regardless of whether it is relevant to the question.
-
-To know more about an creating assistant in phidata, check out [phidata docs](https://docs.phidata.com/assistants/introduction) here.
-
-## **Step 3** - Load data to Knowledge Base.
-
-```python
-# load out data into the knowledge_base (populating the LanceTable)
-assistant.knowledge_base.load(recreate=False)
-```
-The above code loads the data to the Knowledge Base(LanceDB Table) and now it is ready to be used by the assistant. 
-
-| Name| Type | Purpose | Default |
-|:----|:-----|:--------|:--------|
-|`recreate`|`bool`| If True, it drops the existing table and recreates the table in the vectorDB. |`False`|
-|`upsert`|`bool`| If True and the vectorDB supports upsert, it will upsert documents to the vector db. | `False` |
-|`skip_existing`|`bool`| If True, skips documents that already exist in the vectorDB when inserting. |`True`|
-
-??? tip "What is upsert?"
-    Upsert is a database operation that combines "update" and "insert". It updates existing records if a document with the same identifier does exist, or inserts new records if no matching record exists. This is useful for maintaining the most current information without manually checking for existence.
-
-During the Load operation, phidata directly interacts with the LanceDB library and performs the loading of the table with our data in the following steps -     
-
-1. **Creates** and **initializes** the table if it does not exist.
-
-2. Then it **splits** our data into smaller **chunks**.
-
-    ??? question "How do they create chunks?"
-        **phidata** provides many types of **Knowledge Bases** based on the type of data. Most of them :material-information-outline:{ title="except LlamaIndexKnowledgeBase and LangChainKnowledgeBase"} has a property method called `document_lists` of type `Iterator[List[Document]]`. During the load operation, this property method is invoked. It traverses on the data provided by us (in this case, a text file(s)) using `reader`. Then it **reads**, **creates chunks**, and **encapsulates** each chunk inside a `Document` object and yields **lists of `Document` objects** that contain our data.
-
-3. Then **embeddings** are created on these chunks are **inserted** into the LanceDB Table 
-
-    ??? question "How do they insert your data as different rows in LanceDB Table?"
-        The chunks of your data are in the form - **lists of `Document` objects**. It was yielded in the step above.
-
-        for each `Document` in `List[Document]`, it does the following operations:
-        
-        - Creates embedding on `Document`.
-        - Cleans the **content attribute**(chunks of our data is here) of `Document`.
-        - Prepares data by creating `id` and loading `payload` with the metadata related to this chunk. (1)
-            { .annotate }
-
-            1.  Three columns will be added to the table - `"id"`, `"vector"`, and `"payload"` (payload contains various metadata including **`content`**)
-                
-        - Then add this data to LanceTable. 
-
-4. Now the internal state of `knowledge_base` is changed (embeddings are created and loaded in the table ) and it **ready to be used by assistant**.
-
-## **Step 4** - Start a cli chatbot with access to the Knowledge base  
-
-```python
-# start cli chatbot with knowledge base
-assistant.print_response("Ask me about something from the knowledge base")
-while True:
-    message = Prompt.ask(f"[bold] :sunglasses: User [/bold]")
-    if message in ("exit", "bye"):
-        break
-    assistant.print_response(message, markdown=True)
-```  
-
-
-For more information and amazing cookbooks of phidata, read the [phidata documentation](https://docs.phidata.com/introduction) and also visit [LanceDB x phidata docmentation](https://docs.phidata.com/vectordb/lancedb).