Bump version: 0.23.0-beta.3 → 0.23.0

Adds a script to change the lance dependency easily. To make this
change, I just had to run:

```bash
python ci/set_lance_version.py stable
```

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added a script to automate updating the Lance package version in
project dependencies.
- **Chores**
- Updated workflows to improve lockfile management and automate updates
during releases and publishing.
- Switched Lance dependencies from git-based references to fixed version
numbers for improved stability.
- Enhanced lockfile update script with an option to amend commits and
quieter output.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.10.0-beta.1"
+current_version = "0.20.0-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -24,34 +24,87 @@ commit = true
 message = "Bump version: {current_version} → {new_version}"
 commit_args = ""
 
+# Java maven files
+pre_commit_hooks = [
+  """
+    NEW_VERSION="${BVHOOK_NEW_MAJOR}.${BVHOOK_NEW_MINOR}.${BVHOOK_NEW_PATCH}"
+    if [ ! -z "$BVHOOK_NEW_PRE_L" ] && [ ! -z "$BVHOOK_NEW_PRE_N" ]; then
+        NEW_VERSION="${NEW_VERSION}-${BVHOOK_NEW_PRE_L}.${BVHOOK_NEW_PRE_N}"
+    fi
+    echo "Constructed new version: $NEW_VERSION"
+    cd java && mvn versions:set -DnewVersion=$NEW_VERSION && mvn versions:commit
+
+    # Check for any modified but unstaged pom.xml files
+    MODIFIED_POMS=$(git ls-files -m | grep pom.xml)
+    if [ ! -z "$MODIFIED_POMS" ]; then
+        echo "The following pom.xml files were modified but not staged. Adding them now:"
+        echo "$MODIFIED_POMS" | while read -r file; do
+            git add "$file"
+            echo "Added: $file"
+        done
+    fi
+    """,
+]
+
 [tool.bumpversion.parts.pre_l]
-values = ["beta", "final"]
 optional_value = "final"
+values = ["beta", "final"]
 
 [[tool.bumpversion.files]]
 filename = "node/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
 
 [[tool.bumpversion.files]]
 filename = "nodejs/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
 
 # nodejs binary packages
 [[tool.bumpversion.files]]
 glob = "nodejs/npm/*/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
+
+# vectodb node binary packages
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-darwin-arm64\": \"{new_version}\""
+search = "\"@lancedb/vectordb-darwin-arm64\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-darwin-x64\": \"{new_version}\""
+search = "\"@lancedb/vectordb-darwin-x64\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-linux-arm64-gnu\": \"{new_version}\""
+search = "\"@lancedb/vectordb-linux-arm64-gnu\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\""
+search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""
+search = "\"@lancedb/vectordb-win32-x64-msvc\": \"{current_version}\""
 
 # Cargo files
 # ------------
 [[tool.bumpversion.files]]
 filename = "rust/ffi/node/Cargo.toml"
-search = "\nversion = \"{current_version}\""
 replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
 
 [[tool.bumpversion.files]]
 filename = "rust/lancedb/Cargo.toml"
-search = "\nversion = \"{current_version}\""
 replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
+
+[[tool.bumpversion.files]]
+filename = "nodejs/Cargo.toml"
+replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""

.cargo/config.toml

@@ -31,6 +31,13 @@ rustflags = [
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 
+[target.x86_64-unknown-linux-musl]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
+
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"
+rustflags = ["-C", "target-feature=-crt-static"]
+
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
 
@@ -38,3 +45,7 @@ rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm
 # not found errors on systems that are missing it.
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-Ctarget-feature=+crt-static"]
+
+# Experimental target for Arm64 Windows
+[target.aarch64-pc-windows-msvc]
+rustflags = ["-Ctarget-feature=+crt-static"]

.github/workflows/build_linux_wheel/action.yml

@@ -36,8 +36,7 @@ runs:
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          yum install -y openssl-devel \
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip
     - name: Build Arm Manylinux Wheel
@@ -52,12 +51,7 @@ runs:
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          apt install -y unzip
+          yum install -y clang \
-          if [ $(uname -m) = "x86_64" ]; then
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
-            PROTOC_ARCH="x86_64"
-          else
-            PROTOC_ARCH="aarch_64"
-          fi
-          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -20,7 +20,7 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
+        # TODO: pass through interpreter
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
-        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -28,7 +28,7 @@ runs:
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: windows-wheels
         path: python\target\wheels

.github/workflows/docs.yml

@@ -18,17 +18,24 @@ concurrency:
   group: "pages"
   cancel-in-progress: true
 
+env:
+  # This reduces the disk space needed for the build
+  RUSTFLAGS: "-C debuginfo=0"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+
 jobs:
   # Single deploy job since we're just deploying
   build:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install dependecies needed for ubuntu
+      - name: Install dependencies needed for ubuntu
         run: |
           sudo apt install -y protobuf-compiler libssl-dev
           rustup update && rustup default
@@ -38,18 +45,18 @@ jobs:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: "docs/requirements.txt"
+      - uses: Swatinem/rust-cache@v2
       - name: Build Python
         working-directory: python
         run: |
-          python -m pip install -e .
+          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .
-          python -m pip install -r ../docs/requirements.txt
+          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
       - name: Set up node
         uses: actions/setup-node@v3
         with:
           node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
       - name: Install node dependencies
         working-directory: node
         run: |
@@ -72,9 +79,9 @@ jobs:
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.github/workflows/docs_test.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: ubuntu-24.04
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -49,7 +49,7 @@ jobs:
     - name: Build Python
       working-directory: docs/test
       run:
-        python -m pip install -r requirements.txt
+        python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r requirements.txt
     - name: Create test files
       run: |
         cd docs/test
@@ -60,7 +60,7 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/java-publish.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: "1.79.0"
+          toolchain: "1.81.0"
           cache-workspaces: "./java/core/lancedb-jni"
           # Disable full debug symbol generation to speed up CI build and keep memory down
           # "1" means line tables only, which is useful for panic tracebacks.
@@ -94,16 +94,21 @@ jobs:
           mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64
           cp ../liblancedb_jni_darwin_aarch64.zip/liblancedb_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblancedb_jni.dylib
           cp ../liblancedb_jni_linux_aarch64.zip/liblancedb_jni.so ./core/target/classes/nativelib/linux-aarch64/liblancedb_jni.so
+      - name: Dry run
+        if: github.event_name == 'pull_request'
+        run: |
+          mvn --batch-mode -DskipTests -Drust.release.build=true package
       - name: Set github
         run: |
           git config --global user.email "LanceDB Github Runner"
           git config --global user.name "dev+gha@lancedb.com"
       - name: Publish with Java 8
+        if: github.event_name == 'release'
         run: |
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
+          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}

.github/workflows/java.yml

@@ -35,6 +35,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: java/core/lancedb-jni
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
       - name: Run cargo fmt
         run: cargo fmt --check
         working-directory: ./java/core/lancedb-jni
@@ -68,6 +71,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: java/core/lancedb-jni
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
       - name: Run cargo fmt
         run: cargo fmt --check
         working-directory: ./java/core/lancedb-jni
@@ -110,4 +116,3 @@ jobs:
           -Djdk.reflect.useDirectMethodHandle=false \
           -Dio.netty.tryReflectionSetAccessible=true"
           JAVA_HOME=$JAVA_17 mvn clean test
-  

.github/workflows/license-header-check.yml

@@ -0,0 +1,31 @@
+name: Check license headers
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - rust/**
+      - python/**
+      - nodejs/**
+      - java/**
+      - .github/workflows/license-header-check.yml
+jobs:
+  check-licenses:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Install license-header-checker
+        working-directory: /tmp
+        run: |
+            curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
+            mv /tmp/bin/license-header-checker /usr/local/bin/
+      - name: Check license headers (rust)
+        run: license-header-checker -a -v ./rust/license_header.txt ./ rs && [[ -z `git status -s` ]]
+      - name: Check license headers (python)
+        run: license-header-checker -a -v ./python/license_header.txt python py && [[ -z `git status -s` ]]
+      - name: Check license headers (typescript)
+        run: license-header-checker -a -v ./nodejs/license_header.txt nodejs ts && [[ -z `git status -s` ]]
+      - name: Check license headers (java)
+        run: license-header-checker -a -v ./nodejs/license_header.txt java java && [[ -z `git status -s` ]]

.github/workflows/make-release-commit.yml

@@ -30,7 +30,7 @@ on:
         default: true
         type: boolean
       other:
-        description: 'Make a Node/Rust release'
+        description: 'Make a Node/Rust/Java release'
         required: true
         default: true
         type: boolean
@@ -43,7 +43,7 @@ on:
 jobs:
   make-release:
     # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
     steps:
@@ -57,15 +57,14 @@ jobs:
           # trigger any workflows watching for new tags. See:
           # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
           token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+      - name: Validate Lance dependency is at stable version
+        if: ${{ inputs.type == 'stable' }}
+        run: python ci/validate_stable_lance.py
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
       - name: Bump Python version
         if: ${{ inputs.python }}
         working-directory: python
@@ -85,6 +84,7 @@ jobs:
         run: |
           pip install bump-my-version PyGithub packaging
           bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
+          bash ci/update_lockfiles.sh --amend
       - name: Push new version tag
         if: ${{ !inputs.dry_run }}
         uses: ad-m/github-push-action@master
@@ -93,7 +93,3 @@ jobs:
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
           branch: ${{ github.ref }}
           tags: true
-      - uses: ./.github/workflows/update_package_lock
-        if: ${{ !inputs.dry_run && inputs.other }}
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nodejs.yml

@@ -47,12 +47,18 @@ jobs:
       run: |
         sudo apt update
         sudo apt install -y protobuf-compiler libssl-dev
+    - uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        components: rustfmt, clippy
     - name: Lint
       run: |
         cargo fmt --all -- --check
         cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint-ci
+    - name: Lint examples
+      working-directory: nodejs/examples
+      run: npm ci && npm run lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -91,6 +97,30 @@ jobs:
       env:
         S3_TEST: "1"
       run: npm run test
+    - name: Setup examples
+      working-directory: nodejs/examples
+      run: npm ci
+    - name: Test examples
+      working-directory: ./
+      env:
+        OPENAI_API_KEY: test
+        OPENAI_BASE_URL: http://0.0.0.0:8000
+      run: |
+        python ci/mock_openai.py &
+        cd nodejs/examples
+        npm test
+    - name: Check docs
+      run: |
+        # We run this as part of the job because the binary needs to be built
+        # first to export the types of the native code.
+        set -e
+        npm ci
+        npm run docs
+        if ! git diff --exit-code -- . ':(exclude)Cargo.lock'; then
+          echo "Docs need to be updated"
+          echo "Run 'npm run docs', fix any warnings, and commit the changes."
+          exit 1
+        fi
   macos:
     timeout-minutes: 30
     runs-on: "macos-14"

.github/workflows/npm-publish.yml

@@ -1,399 +1,32 @@
 name: NPM Publish
 
+env:
+  MACOSX_DEPLOYMENT_TARGET: '10.13'
+  CARGO_INCREMENTAL: '0'
+
+permissions:
+  contents: write
+  id-token: write
+
 on:
   push:
+    branches:
+      - main
     tags:
       - "v*"
+  pull_request:
+    # This should trigger a dry run (we skip the final publish step)
+    paths:
+      - .github/workflows/npm-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  node:
-    name: vectordb Typescript
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: "npm"
-          cache-dependency-path: node/package-lock.json
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
-        run: |
-          npm ci
-          npm run tsc
-          npm pack
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-package
-          path: |
-            node/vectordb-*.tgz
-
-  node-macos:
-    name: vectordb ${{ matrix.config.arch }}
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-14
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-darwin-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-darwin*.tgz
-
-  nodejs-macos:
-    name: lancedb ${{ matrix.config.arch }}
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-14
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build MacOS native nodejs modules
-        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-darwin-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-
-  node-linux:
-    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: warp-ubuntu-latest-arm64-4x
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      # To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
-      - name: Build Linux Artifacts
-        run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-linux-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-linux*.tgz
-
-  nodejs-linux:
-    name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
-      - name: Build Linux Artifacts
-        run: |
-          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-      # The generic files are the same in all distros so we just pick
-      # one to do the upload.
-      - name: Upload Generic Artifacts
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-dist
-          path: |
-            nodejs/dist/*
-            !nodejs/dist/*.node
-
-  node-windows:
-    name: vectordb ${{ matrix.target }}
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-windows
-          path: |
-            node/dist/lancedb-vectordb-win32*.tgz
-
-  nodejs-windows:
-    name: lancedb ${{ matrix.target }}
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows
-          path: |
-            nodejs/dist/*.node
-
-  release:
-    name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux, node-windows]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: node-*
-      - name: Display structure of downloaded files
-        run: ls -R
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: "https://registry.npmjs.org"
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        run: |
-          # Tag beta as "preview" instead of default "latest". See lancedb 
-          # npm publish step for more info.
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            PUBLISH_ARGS="--tag preview"
-          fi
-
-          mv */*.tgz .
-          for filename in *.tgz; do
-            npm publish $PUBLISH_ARGS $filename
-          done
-      - name: Notify Slack Action
-        uses: ravsamhq/notify-slack-action@2.3.0
-        if: ${{ always() }}
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-          notification_title: "{workflow} is failing"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-
-  release-nodejs:
-    name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          name: nodejs-dist
-          path: nodejs/dist
-      - uses: actions/download-artifact@v4
-        name: Download arch-specific binaries
-        with:
-          pattern: nodejs-*
-          path: nodejs/nodejs-artifacts
-          merge-multiple: true
-      - name: Display structure of downloaded files
-        run: find .
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: "https://registry.npmjs.org"
-      - name: Install napi-rs
-        run: npm install -g @napi-rs/cli
-      - name: Prepare artifacts
-        run: npx napi artifacts -d nodejs-artifacts
-      - name: Display structure of staged files
-        run: find npm
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        # By default, things are published to the latest tag. This is what is
-        # installed by default if the user does not specify a version. This is
-        # good for stable releases, but for pre-releases, we want to publish to
-        # the "preview" tag so they can install with `npm install lancedb@preview`.
-        # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420
-        run: |
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            npm publish --access public --tag preview
-          else
-            npm publish --access public
-          fi
-      - name: Notify Slack Action
-        uses: ravsamhq/notify-slack-action@2.3.0
-        if: ${{ always() }}
-        with:
-          status: ${{ job.status }}
-          notify_when: "failure"
-          notification_title: "{workflow} is failing"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-
-  update-package-lock:
-    needs: [release]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-  update-package-lock-nodejs:
-    needs: [release-nodejs]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -458,3 +91,464 @@ jobs:
           generate_release_notes: false
           name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
           body: ${{ steps.release_notes.outputs.changelog }}
+
+  build-lancedb:
+    strategy:
+      fail-fast: false
+      matrix:
+        settings:
+          - target: x86_64-apple-darwin
+            host: macos-latest
+            features: ","
+            pre_build: |-
+                brew install protobuf
+                rustup target add x86_64-apple-darwin
+          - target: aarch64-apple-darwin
+            host: macos-latest
+            features: fp16kernels
+            pre_build: brew install protobuf
+          - target: x86_64-pc-windows-msvc
+            host: windows-latest
+            features: ","
+            pre_build: |-
+              choco install --no-progress protoc ninja nasm
+              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
+              # There is an issue where choco doesn't add nasm to the path
+              export PATH="$PATH:/c/Program Files/NASM"
+              nasm -v
+          - target: aarch64-pc-windows-msvc
+            host: windows-latest
+            features: ","
+            pre_build: |-
+                choco install --no-progress protoc
+                rustup target add aarch64-pc-windows-msvc
+          - target: x86_64-unknown-linux-gnu
+            host: ubuntu-latest
+            features: fp16kernels
+            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
+            pre_build: |-
+              set -e &&
+              apt-get update &&
+              apt-get install -y protobuf-compiler pkg-config
+          - target: x86_64-unknown-linux-musl
+            # This one seems to need some extra memory
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
+            features: fp16kernels
+            pre_build: |-
+              set -e &&
+              apk add protobuf-dev curl &&
+              ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
+              ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
+              CC=gcc &&
+              CXX=g++
+          - target: aarch64-unknown-linux-gnu
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
+            features: "fp16kernels"
+            pre_build: |-
+              set -e &&
+              apt-get update &&
+              apt-get install -y protobuf-compiler pkg-config &&
+              # https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
+              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
+              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
+              rustup target add aarch64-unknown-linux-gnu
+          - target: aarch64-unknown-linux-musl
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
+            features: ","
+            pre_build: |-
+              set -e &&
+              apk add protobuf-dev &&
+              rustup target add aarch64-unknown-linux-musl &&
+              export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
+              export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
+    name: build - ${{ matrix.settings.target }}
+    runs-on: ${{ matrix.settings.host }}
+    defaults:
+      run:
+        working-directory: nodejs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        if: ${{ !matrix.settings.docker }}
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: nodejs/package-lock.json
+      - name: Install
+        uses: dtolnay/rust-toolchain@stable
+        if: ${{ !matrix.settings.docker }}
+        with:
+          toolchain: stable
+          targets: ${{ matrix.settings.target }}
+      - name: Cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            target/
+          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
+      - name: Setup toolchain
+        run: ${{ matrix.settings.setup }}
+        if: ${{ matrix.settings.setup }}
+        shell: bash
+      - name: Install dependencies
+        run: npm ci
+      - name: Build in docker
+        uses: addnab/docker-run-action@v3
+        if: ${{ matrix.settings.docker }}
+        with:
+          image: ${{ matrix.settings.docker }}
+          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
+                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
+                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
+                    -v ${{ github.workspace }}:/build -w /build/nodejs"
+          run: |
+            set -e
+            ${{ matrix.settings.pre_build }}
+            npx napi build --platform  --release --no-const-enum \
+              --features ${{ matrix.settings.features }} \
+              --target ${{ matrix.settings.target }} \
+              --dts ../lancedb/native.d.ts \
+              --js ../lancedb/native.js \
+              --strip \
+              dist/
+      - name: Build
+        run: |
+          ${{ matrix.settings.pre_build }}
+          npx napi build --platform  --release --no-const-enum \
+              --features ${{ matrix.settings.features }} \
+              --target ${{ matrix.settings.target }} \
+              --dts ../lancedb/native.d.ts \
+              --js ../lancedb/native.js \
+              --strip \
+              $EXTRA_ARGS \
+              dist/
+        if: ${{ !matrix.settings.docker }}
+        shell: bash
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: lancedb-${{ matrix.settings.target }}
+          path: nodejs/dist/*.node
+          if-no-files-found: error
+      # The generic files are the same in all distros so we just pick
+      # one to do the upload.
+      - name: Make generic artifacts
+        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
+        run: npm run tsc
+      - name: Upload Generic Artifacts
+        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-dist
+          path: |
+            nodejs/dist/*
+            !nodejs/dist/*.node
+  test-lancedb:
+    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
+    needs:
+      - build-lancedb
+    strategy:
+      fail-fast: false
+      matrix:
+        settings:
+          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
+          # - host: windows-latest
+          #   target: x86_64-pc-windows-msvc
+          - host: macos-latest
+            target: aarch64-apple-darwin
+          - target: x86_64-unknown-linux-gnu
+            host: ubuntu-latest
+          - target: aarch64-unknown-linux-gnu
+            host: buildjet-16vcpu-ubuntu-2204-arm
+        node:
+          - '20'
+    runs-on: ${{ matrix.settings.host }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: npm
+          cache-dependency-path: nodejs/package-lock.json
+      - name: Install dependencies
+        run: npm ci
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: lancedb-${{ matrix.settings.target }}
+          path: nodejs/dist/
+          # For testing purposes:
+          # run-id: 13982782871
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+          # For testing purposes:
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+          # run-id: 13982782871
+      - name: List packages
+        run: ls -R dist
+      - name: Move built files
+        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
+      - name: Test bindings
+        run: npm test
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    needs:
+      - test-lancedb
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: nodejs/package-lock.json
+          registry-url: "https://registry.npmjs.org"
+      - name: Install dependencies
+        run: npm ci
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+          # For testing purposes:
+          # run-id: 13982782871
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+      - uses: actions/download-artifact@v4
+        name: Download arch-specific binaries
+        with:
+          pattern: lancedb-*
+          path: nodejs/nodejs-artifacts
+          merge-multiple: true
+          # For testing purposes:
+          # run-id: 13982782871
+          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
+      - name: Display structure of downloaded files
+        run: find dist && find nodejs-artifacts
+      - name: Move artifacts
+        run: npx napi artifacts -d nodejs-artifacts
+      - name: List packages
+        run: find npm
+      - name: Publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        run: |
+          ARGS="--access public"
+          if [[ $DRY_RUN == "true" ]]; then
+            ARGS="$ARGS --dry-run"
+          fi
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            ARGS="$ARGS --tag preview"
+          fi
+          npm publish $ARGS
+
+
+  # ----------------------------------------------------------------------------
+  # vectordb release (legacy)
+  # ----------------------------------------------------------------------------
+  # TODO: delete this when we drop vectordb
+  node:
+    name: vectordb Typescript
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: "npm"
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-package
+          path: |
+            node/vectordb-*.tgz
+
+  node-macos:
+    name: vectordb ${{ matrix.config.arch }}
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-14
+    runs-on: ${{ matrix.config.runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build MacOS native node modules
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-darwin-${{ matrix.config.arch }}
+          path: |
+            node/dist/lancedb-vectordb-darwin*.tgz
+
+  node-linux-gnu:
+    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
+    runs-on: ${{ matrix.config.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: warp-ubuntu-latest-arm64-4x
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-linux-${{ matrix.config.arch }}-gnu
+          path: |
+            node/dist/lancedb-vectordb-linux*.tgz
+
+  node-windows:
+    name: vectordb ${{ matrix.target }}
+    runs-on: windows-2022
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-windows
+          path: |
+            node/dist/lancedb-vectordb-win32*.tgz
+
+  release:
+    name: vectordb NPM Publish
+    needs: [node, node-macos, node-linux-gnu, node-windows]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: node-*
+      - name: Display structure of downloaded files
+        run: ls -R
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        run: |
+          # Tag beta as "preview" instead of default "latest". See lancedb
+          # npm publish step for more info.
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            PUBLISH_ARGS="--tag preview"
+          fi
+
+          mv */*.tgz .
+          for filename in *.tgz; do
+            npm publish $PUBLISH_ARGS $filename
+          done
+      - name: Deprecate
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        # We need to deprecate the old package to avoid confusion.
+        # Each time we publish a new version, it gets undeprecated.
+        run: npm deprecate vectordb "Use @lancedb/lancedb instead."
+      - name: Update package-lock.json
+        run: bash ci/update_lockfiles.sh
+      - name: Push new commit
+        uses: ad-m/github-push-action@master
+      - name: Notify Slack Action
+        uses: ravsamhq/notify-slack-action@2.3.0
+        if: ${{ always() }}
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "{workflow} is failing"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

.github/workflows/pypi-publish.yml

@@ -4,6 +4,11 @@ on:
   push:
     tags:
       - 'python-v*'
+  pull_request:
+    # This should trigger a dry run (we skip the final publish step)
+    paths:
+      - .github/workflows/pypi-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
 
 jobs:
   linux:
@@ -15,15 +20,21 @@ jobs:
           - platform: x86_64
             manylinux: "2_17"
             extra_args: ""
+            runner: ubuntu-22.04
           - platform: x86_64
             manylinux: "2_28"
             extra_args: "--features fp16kernels"
+            runner: ubuntu-22.04
           - platform: aarch64
-            manylinux: "2_24"
+            manylinux: "2_17"
             extra_args: ""
-          # We don't build fp16 kernels for aarch64, because it uses 
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-          # cross compilation image, which doesn't have a new enough compiler.
+            runner: ubuntu-2404-8x-arm64
-    runs-on: "ubuntu-22.04"
+          - platform: aarch64
+            manylinux: "2_28"
+            extra_args: "--features fp16kernels"
+            runner: ubuntu-2404-8x-arm64
+    runs-on: ${{ matrix.config.runner }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -40,6 +51,7 @@ jobs:
           arm-build: ${{ matrix.config.platform == 'aarch64' }}
           manylinux: ${{ matrix.config.manylinux }}
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
@@ -69,6 +81,7 @@ jobs:
           python-minor-version: 8
           args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
@@ -83,17 +96,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - uses: ./.github/workflows/build_windows_wheel
         with:
           python-minor-version: 8
           args: "--release --strip"
           vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/python-v')
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/python.yml

@@ -13,6 +13,11 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  # Color output for pytest is off by default.
+  PYTEST_ADDOPTS: "--color=yes"
+  FORCE_COLOR: "1"
+
 jobs:
   lint:
     name: "Lint"
@@ -30,16 +35,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install ruff
         run: |
-          pip install ruff==0.5.4
+          pip install ruff==0.9.9
       - name: Format check
         run: ruff format --check .
       - name: Lint
         run: ruff check .
-  doctest:
+
-    name: "Doctest"
+  type-check:
+    name: "Type Check"
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
     defaults:
@@ -54,7 +60,36 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
+      - name: Install protobuf compiler
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+          pip install toml
+      - name: Install dependencies
+        run: |
+          python ../ci/parse_requirements.py pyproject.toml --extras dev,tests,embeddings > requirements.txt
+          pip install -r requirements.txt
+      - name: Run pyright
+        run: pyright
+
+  doctest:
+    name: "Doctest"
+    timeout-minutes: 30
+    runs-on: "ubuntu-24.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
           cache: "pip"
       - name: Install protobuf
         run: |
@@ -75,8 +110,8 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        python-minor-version: ["9", "11"]
+        python-minor-version: ["9", "12"]
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-24.04"
     defaults:
       run:
         shell: bash
@@ -101,6 +136,10 @@ jobs:
       - uses: ./.github/workflows/run_tests
         with:
           integration: true
+      - name: Test without pylance or pandas
+        run: |
+          pip uninstall -y pylance pandas
+          pytest -vv python/tests/test_table.py
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
@@ -127,7 +166,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -138,7 +177,7 @@ jobs:
         run: rm -rf target/wheels
   windows:
     name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 30
+    timeout-minutes: 60
     strategy:
       matrix:
         config:
@@ -157,7 +196,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -168,7 +207,7 @@ jobs:
         run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-24.04"
     defaults:
       run:
         shell: bash
@@ -189,6 +228,7 @@ jobs:
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
+          pip install pyarrow==16
           pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests

.github/workflows/run_tests/action.yml

@@ -24,8 +24,8 @@ runs:
     - name: pytest (with integration)
       shell: bash
       if: ${{ inputs.integration == 'true' }}
-      run: pytest -m "not slow" -x -v --durations=30 python/python/tests
+      run: pytest -m "not slow" -vv --durations=30 python/python/tests
     - name: pytest (no integration tests)
       shell: bash
       if: ${{ inputs.integration != 'true' }}
-      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests
+      run: pytest -m "not slow and not s3_test" -vv --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -22,75 +22,116 @@ env:
   # "1" means line tables only, which is useful for panic tracebacks.
   RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
+  CARGO_INCREMENTAL: 0
 
 jobs:
   lint:
     timeout-minutes: 30
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     defaults:
       run:
         shell: bash
-        working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
+      CC: clang-18
-      CXX: g++-12
+      CXX: clang++-18
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
-      with:
+        with:
           fetch-depth: 0
           lfs: true
-    - uses: Swatinem/rust-cache@v2
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
-      with:
+        with:
-        workspaces: rust
+          components: rustfmt, clippy
-    - name: Install dependencies
+      - uses: Swatinem/rust-cache@v2
-      run: |
+        with:
+          workspaces: rust
+      - name: Install dependencies
+        run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-    - name: Run format
+      - name: Run format
-      run: cargo fmt --all -- --check
+        run: cargo fmt --all -- --check
-    - name: Run clippy
+      - name: Run clippy
-      run: cargo clippy --all --all-features -- -D warnings
+        run: cargo clippy --workspace --tests --all-features -- -D warnings
+
+  build-no-lock:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      # Need up-to-date compilers for kernels
+      CC: clang
+      CXX: clang++
+    steps:
+      - uses: actions/checkout@v4
+      # Building without a lock file often requires the latest Rust version since downstream
+      # dependencies may have updated their minimum Rust version.
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: "stable"
+      # Remove cargo.lock to force a fresh build
+      - name: Remove Cargo.lock
+        run: rm -f Cargo.lock
+      - uses: rui314/setup-mold@v1
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build all
+        run: |
+          cargo build --benches --all-features --tests
+
   linux:
     timeout-minutes: 30
     # To build all features, we need more disk space than is available
-    # on the GitHub-provided runner. This is mostly due to the the
+    # on the free OSS github runner. This is mostly due to the the
     # sentence-transformers feature.
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-2404-4x-x64
     defaults:
       run:
         shell: bash
         working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
+      CC: clang-18
-      CXX: g++-12
+      CXX: clang++-18
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
-      with:
+        with:
           fetch-depth: 0
           lfs: true
-    - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@v2
-      with:
+        with:
           workspaces: rust
-    - name: Install dependencies
+      - name: Install dependencies
-      run: |
+        run: |
-          sudo apt update
+          # This shaves 2 minutes off this step in CI. This doesn't seem to be
+          # necessary in standard runners, but it is in the 4x runners.
+          sudo rm /var/lib/man-db/auto-update
           sudo apt install -y protobuf-compiler libssl-dev
-    - name: Start S3 integration test environment
+      - uses: rui314/setup-mold@v1
-      working-directory: .
+      - name: Make Swap
-      run: docker compose up --detach --wait
+        run: |
-    - name: Build
+          sudo fallocate -l 16G /swapfile
-      run: cargo build --all-features
+          sudo chmod 600 /swapfile
-    - name: Run tests
+          sudo mkswap /swapfile
-      run: cargo test --all-features
+          sudo swapon /swapfile
-    - name: Run examples
+      - name: Start S3 integration test environment
-      run: cargo run --example simple
+        working-directory: .
+        run: docker compose up --detach --wait
+      - name: Build
+        run: cargo build --all-features --tests --locked --examples
+      - name: Run tests
+        run: cargo test --all-features --locked
+      - name: Run examples
+        run: cargo run --example simple --locked
+
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: [ "macos-13", "macos-14" ]
+        mac-runner: ["macos-13", "macos-14"]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
@@ -99,8 +140,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-            fetch-depth: 0
+          fetch-depth: 0
-            lfs: true
+          lfs: true
       - name: CPU features
         run: sysctl -a | grep cpu
       - uses: Swatinem/rust-cache@v2
@@ -108,29 +149,78 @@ jobs:
           workspaces: rust
       - name: Install dependencies
         run: brew install protobuf
-      - name: Build
-        run: cargo build --all-features
       - name: Run tests
-        # Run with everything except the integration tests.
+        run: |
-        run: cargo test --features remote,fp16kernels
+          # Don't run the s3 integration tests since docker isn't available
+          # on this image.
+          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
+            | jq -r '.packages[] | .features | keys | .[]' \
+            | grep -v s3-test | sort | uniq | paste -s -d "," -`
+          cargo test --features $ALL_FEATURES --locked
+
   windows:
     runs-on: windows-2022
+    strategy:
+      matrix:
+        target:
+          - x86_64-pc-windows-msvc
+          - aarch64-pc-windows-msvc
+    defaults:
+      run:
+        working-directory: rust/lancedb
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
       - name: Install Protoc v21.12
-        working-directory: C:\
+        run: choco install --no-progress protoc
+      - name: Build
         run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
+          rustup target add ${{ matrix.target }}
-          Set-Location C:\protoc
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          cargo build --features remote --tests --locked --target ${{ matrix.target }}
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
       - name: Run tests
+        # Can only run tests when target matches host
+        if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build
+          cargo test --features remote --locked
-          cargo test
+
+  msrv:
+    # Check the minimum supported Rust version
+    name: MSRV Check - Rust v${{ matrix.msrv }}
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
+    env:
+      # Need up-to-date compilers for kernels
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install ${{ matrix.msrv }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ matrix.msrv }}
+      - name: Downgrade  dependencies
+        # These packages have newer requirements for MSRV
+        run: |
+          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
+          cargo update -p aws-sdk-dynamodb --precise 1.55.0
+          cargo update -p aws-config --precise 1.5.10
+          cargo update -p aws-sdk-kms --precise 1.51.0
+          cargo update -p aws-sdk-s3 --precise 1.65.0
+          cargo update -p aws-sdk-sso --precise 1.50.0
+          cargo update -p aws-sdk-ssooidc --precise 1.51.0
+          cargo update -p aws-sdk-sts --precise 1.51.0
+          cargo update -p home --precise 0.5.9
+      - name: cargo +${{ matrix.msrv }} check
+        run: cargo check --workspace --tests --benches --all-features

.github/workflows/update_package_lock/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock
-description: "Update node's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./node
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/update_package_lock_nodejs/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock_nodejs
-description: "Update nodejs's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./nodejs
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/upload_wheel/action.yml

@@ -17,11 +17,12 @@ runs:
     run: |
       python -m pip install --upgrade pip
       pip install twine
+      python3 -m pip install --upgrade pkginfo
   - name: Choose repo
     shell: bash
     id: choose_repo
     run: |
-      if [ ${{ github.ref }} == "*beta*" ]; then
+      if [[ ${{ github.ref }} == *beta* ]]; then
         echo "repo=fury" >> $GITHUB_OUTPUT
       else
         echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -32,7 +33,7 @@ runs:
       FURY_TOKEN: ${{ inputs.fury_token }}
       PYPI_TOKEN: ${{ inputs.pypi_token }}
     run: |
-      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
+      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
         WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
         echo "Uploading $WHEEL to Fury"
         curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/

.gitignore

@@ -9,7 +9,6 @@ venv
 .vscode
 .zed
 rust/target
-rust/Cargo.lock
 
 site
 
@@ -42,5 +41,3 @@ dist
 target
 
 **/sccache.log
-
-Cargo.lock

.pre-commit-config.yaml

@@ -1,21 +1,27 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
-    -   id: check-yaml
+      - id: check-yaml
-    -   id: end-of-file-fixer
+      - id: end-of-file-fixer
-    -   id: trailing-whitespace
+      - id: trailing-whitespace
--   repo: https://github.com/astral-sh/ruff-pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.2.2
+    rev: v0.9.9
     hooks:
-    - id: ruff
+      - id: ruff
-- repo: local
+  # - repo: https://github.com/RobertCraigie/pyright-python
-  hooks:
+  #   rev: v1.1.395
-    - id: local-biome-check
+  #   hooks:
-      name: biome check
+  #     - id: pyright
-      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+  #       args: ["--project", "python"]
-      language: system
+  #       additional_dependencies: [pyarrow-stubs]
-      types: [text]
+  - repo: local
-      files: "nodejs/.*"
+    hooks:
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*
+      - id: local-biome-check
+        name: biome check
+        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+        language: system
+        types: [text]
+        files: "nodejs/.*"
+        exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*

CONTRIBUTING.md

@@ -0,0 +1,78 @@
+# Contributing to LanceDB
+
+LanceDB is an open-source project and we welcome contributions from the community.
+This document outlines the process for contributing to LanceDB.
+
+## Reporting Issues
+
+If you encounter a bug or have a feature request, please open an issue on the
+[GitHub issue tracker](https://github.com/lancedb/lancedb).
+
+## Picking an issue
+
+We track issues on the GitHub issue tracker. If you are looking for something to
+work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
+
+If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
+
+## Configuring Git
+
+First, fork the repository on GitHub, then clone your fork:
+
+```bash
+git clone https://github.com/<username>/lancedb.git
+cd lancedb
+```
+
+Then add the main repository as a remote:
+
+```bash
+git remote add upstream https://github.com/lancedb/lancedb.git
+git fetch upstream
+```
+
+## Setting up your development environment
+
+We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
+
+* [Python](python/CONTRIBUTING.md)
+* [Typescript](nodejs/CONTRIBUTING.md)
+<!-- TODO: add Java contributing guide -->
+* [Documentation](docs/README.md)
+
+
+## Best practices for pull requests
+
+For the best chance of having your pull request accepted, please follow these guidelines:
+
+1. Unit test all bug fixes and new features. Your code will not be merged if it
+   doesn't have tests.
+1. If you change the public API, update the documentation in the `docs` directory.
+1. Aim to minimize the number of changes in each pull request. Keep to solving
+   one problem at a time, when possible.
+1. Before marking a pull request ready-for-review, do a self review of your code.
+   Is it clear why you are making the changes? Are the changes easy to understand?
+1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
+    * New feature: `feat: adding foo API`
+    * Bug fix: `fix: issue with foo API`
+    * Documentation change: `docs: adding foo API documentation`
+1. If your pull request is a work in progress, leave the pull request as a draft.
+   We will assume the pull request is ready for review when it is opened.
+1. When writing tests, test the error cases. Make sure they have understandable
+   error messages.
+
+## Project structure
+
+The core library is written in Rust. The Python, Typescript, and Java libraries
+are wrappers around the Rust library.
+
+* `src/lancedb`: Rust library source code
+* `python`: Python package source code
+* `nodejs`: Typescript package source code
+* `node`: **Deprecated** Typescript package source code
+* `java`: Java package source code
+* `docs`: Documentation source code
+
+## Release process
+
+For information on the release process, see: [release_process.md](release_process.md)

Cargo.toml

@@ -18,36 +18,55 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
+rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.17.0", "features" = ["dynamodb"] }
+lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.17.0" }
+lance-io = "=0.29.0"
-lance-linalg = { "version" = "=0.17.0" }
+lance-index = "=0.29.0"
-lance-table = { "version" = "=0.17.0" }
+lance-linalg = "=0.29.0"
-lance-testing = { "version" = "=0.17.0" }
+lance-table = "=0.29.0"
-lance-datafusion = { "version" = "=0.17.0" }
+lance-testing = "=0.29.0"
-lance-encoding = { "version" = "=0.17.0" }
+lance-datafusion = "=0.29.0"
+lance-encoding = "=0.29.0"
 # Note that this one does not include pyarrow
-arrow = { version = "52.2", optional = false }
+arrow = { version = "55.1", optional = false }
-arrow-array = "52.2"
+arrow-array = "55.1"
-arrow-data = "52.2"
+arrow-data = "55.1"
-arrow-ipc = "52.2"
+arrow-ipc = "55.1"
-arrow-ord = "52.2"
+arrow-ord = "55.1"
-arrow-schema = "52.2"
+arrow-schema = "55.1"
-arrow-arith = "52.2"
+arrow-arith = "55.1"
-arrow-cast = "52.2"
+arrow-cast = "55.1"
 async-trait = "0"
-chrono = "0.4.35"
+datafusion = { version = "47.0", default-features = false }
-datafusion-physical-plan = "40.0"
+datafusion-catalog = "47.0"
-half = { "version" = "=2.4.1", default-features = false, features = [
+datafusion-common = { version = "47.0", default-features = false }
+datafusion-execution = "47.0"
+datafusion-expr = "47.0"
+datafusion-physical-plan = "47.0"
+env_logger = "0.11"
+half = { "version" = "=2.5.0", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
-object_store = "0.10.2"
+moka = { version = "0.12", features = ["future"] }
+object_store = "0.11.0"
 pin-project = "1.0.7"
-snafu = "0.7.4"
+snafu = "0.8"
 url = "2"
 num-traits = "0.2"
+rand = "0.9"
 regex = "1.10"
 lazy_static = "1"
+semver = "1.0.25"
+# Temporary pins to work around downstream issues
+# https://github.com/apache/arrow-rs/commit/2fddf85afcd20110ce783ed5b4cdeb82293da30b
+chrono = "=0.4.41"
+# https://github.com/RustCrypto/formats/issues/1684
+base64ct = "=1.6.0"
+# Workaround for: https://github.com/eira-fransham/crunchy/issues/13
+crunchy = "=0.2.2"
+# Workaround for: https://github.com/Lokathor/bytemuck/issues/306
+bytemuck_derive = ">=1.8.1, <1.9.0"

README.md

@@ -1,85 +1,97 @@
+<a href="https://cloud.lancedb.com" target="_blank">
+  <img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
+</a>
 <div align="center">
-<p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
+[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
+[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
+[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
+[![Discord](https://img.shields.io/badge/-Discord-100000?style=for-the-badge&logo=discord&logoColor=white&labelColor=645cfb&color=645cfb)](https://discord.gg/zMM32dvNtd)
+[![Twitter](https://img.shields.io/badge/-Twitter-100000?style=for-the-badge&logo=x&logoColor=white&labelColor=645cfb&color=645cfb)](https://twitter.com/lancedb)
+[![LinkedIn](https://img.shields.io/badge/-LinkedIn-100000?style=for-the-badge&logo=linkedin&logoColor=white&labelColor=645cfb&color=645cfb)](https://www.linkedin.com/company/lancedb/)
 
-**Developer-friendly, database for multimodal AI**
 
-<a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
-<a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/)
-[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd)
-[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
-</p>
+# **The Multimodal AI Lakehouse**
 
-<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
+[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.github.io/lancedb/) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
+
+**The ultimate multimodal data platform for AI/ML applications.** 
+
+LanceDB is designed for fast, scalable, and production-ready vector search. It is built on top of the Lance columnar format. You can store, index, and search over petabytes of multimodal data and vectors with ease. 
+LanceDB is a central location where developers can build, train and analyze their AI workloads.
 
-</p>
 </div>
 
-<hr />
+<br>
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
+## **Demo: Multimodal Search by Keyword, Vector or with SQL**
+<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
 
-The key features of LanceDB include:
+## **Star LanceDB to get updates!**
 
-* Production-scale vector search with no servers to manage.
+<details>
+<summary>⭐ Click here ⭐  to see how fast we're growing!</summary>
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=lancedb/lancedb&theme=dark&type=Date">
+  <img width="100%" src="https://api.star-history.com/svg?repos=lancedb/lancedb&theme=dark&type=Date">
+</picture>
+</details>
 
-* Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
+## **Key Features**:
 
-* Support for vector similarity search, full-text search and SQL.
+- **Fast Vector Search**: Search billions of vectors in milliseconds with state-of-the-art indexing.
+- **Comprehensive Search**: Support for vector similarity search, full-text search and SQL.
+- **Multimodal Support**: Store, query and filter vectors, metadata and multimodal data (text, images, videos, point clouds, and more).
+- **Advanced Features**: Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure. GPU support in building vector index.
 
-* Native Python and Javascript/Typescript support.
+### **Products**:
+- **Open Source & Local**: 100% open source, runs locally or in your cloud. No vendor lock-in.
+- **Cloud and Enterprise**: Production-scale vector search with no servers to manage. Complete data sovereignty and security.
 
-* Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
+### **Ecosystem**:
+- **Columnar Storage**: Built on the Lance columnar format for efficient storage and analytics.
+- **Seamless Integration**: Python, Node.js, Rust, and REST APIs for easy integration. Native Python and Javascript/Typescript support.
+- **Rich Ecosystem**: Integrations with [**LangChain** 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [**LlamaIndex** 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-* GPU support in building vector index(*).
+## **How to Install**:
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+Follow the [Quickstart](https://lancedb.github.io/lancedb/basic/) doc to set up LanceDB locally. 
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+**API & SDK:** We also support Python, Typescript and Rust SDKs
 
-## Quick Start
+| Interface | Documentation |
+|-----------|---------------|
+| Python SDK | https://lancedb.github.io/lancedb/python/python/ |
+| Typescript SDK | https://lancedb.github.io/lancedb/js/globals/ |
+| Rust SDK | https://docs.rs/lancedb/latest/lancedb/index.html |
+| REST API | https://docs.lancedb.com/api-reference/introduction |
 
-**Javascript**
+## **Join Us and Contribute**
-```shell
-npm install @lancedb/lancedb
-```
 
-```javascript
+We welcome contributions from everyone! Whether you're a developer, researcher, or just someone who wants to help out. 
-import * as lancedb from "@lancedb/lancedb";
 
-const db = await lancedb.connect("data/sample-lancedb");
+If you have any suggestions or feature requests, please feel free to open an issue on GitHub or discuss it on our [**Discord**](https://discord.gg/G5DcmnZWKB) server.
-const table = await db.createTable("vectors", [
+
-	{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+[**Check out the GitHub Issues**](https://github.com/lancedb/lancedb/issues) if you would like to work on the features that are planned for the future. If you have any suggestions or feature requests, please feel free to open an issue on GitHub. 
-	{ id: 2, vector: [1.1, 1.2], item: "bar", price: 50 },
+
-], {mode: 'overwrite'});
+## **Contributors**
+
+<a href="https://github.com/lancedb/lancedb/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=lancedb/lancedb" />
+</a>
 
 
-const query = table.vectorSearch([0.1, 0.3]).limit(2);
+## **Stay in Touch With Us**
-const results = await query.toArray();
+<div align="center">
 
-// You can also search for rows by specific criteria without involving a vector search.
+</br>
-const rowsByCriteria = await table.query().where("price >= 10").toArray();
-```
 
-**Python**
+[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
-```shell
+[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
-pip install lancedb
+[![Discord](https://img.shields.io/badge/-Discord-100000?style=for-the-badge&logo=discord&logoColor=white&labelColor=645cfb&color=645cfb)](https://discord.gg/zMM32dvNtd)
-```
+[![Twitter](https://img.shields.io/badge/-Twitter-100000?style=for-the-badge&logo=x&logoColor=white&labelColor=645cfb&color=645cfb)](https://twitter.com/lancedb)
+[![LinkedIn](https://img.shields.io/badge/-LinkedIn-100000?style=for-the-badge&logo=linkedin&logoColor=white&labelColor=645cfb&color=645cfb)](https://www.linkedin.com/company/lancedb/)
 
-```python
+</div>
-import lancedb
-
-uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-table = db.create_table("my_table",
-                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_pandas()
-```
-
-## Blogs, Tutorials & Videos
-* 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -e
 ARCH=${1:-x86_64}
+TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
 # We pass down the current user so that when we later mount the local files
 # into the container, the files are accessible by the current user.
@@ -18,4 +19,4 @@ docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
     lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH
+    bash ci/manylinux_node/build_vectordb.sh $ARCH $TARGET_TRIPLE

ci/build_linux_artifacts_nodejs.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-ARCH=${1:-x86_64}
-
-# We pass down the current user so that when we later mount the local files
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_node
-docker build \
-    -t lancedb-node-manylinux-$ARCH \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
-
-# We turn on memory swap to avoid OOM killer
-docker run \
-    -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
-    lancedb-node-manylinux-$ARCH \
-    bash ci/manylinux_node/build_lancedb.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -1,34 +0,0 @@
-# Builds the macOS artifacts (nodejs binaries).
-# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-set -e
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/lancedb
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd nodejs
-    echo "Building nodejs library for $1"
-    export RUST_TARGET=$1
-    npm run build-release
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/build_windows_artifacts.ps1

@@ -3,6 +3,7 @@
 # Targets supported:
 # - x86_64-pc-windows-msvc
 # - i686-pc-windows-msvc
+# - aarch64-pc-windows-msvc
 
 function Prebuild-Rust {
     param (
@@ -31,7 +32,7 @@ function Build-NodeBinaries {
 
 $targets = $args[0]
 if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
+    $targets = "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"
 }
 
 Write-Host "Building artifacts for targets: $targets"

ci/build_windows_artifacts_nodejs.ps1

@@ -3,6 +3,7 @@
 # Targets supported:
 # - x86_64-pc-windows-msvc
 # - i686-pc-windows-msvc
+# - aarch64-pc-windows-msvc
 
 function Prebuild-Rust {
     param (
@@ -31,7 +32,7 @@ function Build-NodeBinaries {
 
 $targets = $args[0]
 if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
+    $targets = "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"
 }
 
 Write-Host "Building artifacts for targets: $targets"

ci/manylinux_node/Dockerfile

@@ -9,10 +9,6 @@ FROM quay.io/pypa/manylinux_2_28_${ARCH}
 ARG ARCH=x86_64
 ARG DOCKER_USER=default_user
 
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
 # Protobuf is also installed as root.
 COPY install_protobuf.sh install_protobuf.sh
 RUN ./install_protobuf.sh ${ARCH}

ci/manylinux_node/build_lancedb.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
-
-cd nodejs
-npm ci
-npm run build-release

ci/manylinux_node/build_vectordb.sh

@@ -2,18 +2,12 @@
 # Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
 set -e
 ARCH=${1:-x86_64}
+TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
-if [ "$ARCH" = "x86_64" ]; then
+#Alpine doesn't have .bashrc
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
+FILE=$HOME/.bashrc && test -f $FILE && source $FILE
-else 
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
 
 cd node
 npm ci
 npm run build-release
-npm run pack-build
+npm run pack-build -- -t $TARGET_TRIPLE

ci/manylinux_node/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1v \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

ci/mock_openai.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
+import argparse
+import json
+import http.server
+
+
+class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers["Content-Length"])
+        post_data = self.rfile.read(content_length)
+        post_data = json.loads(post_data.decode("utf-8"))
+        # See: https://platform.openai.com/docs/api-reference/embeddings/create
+
+        if isinstance(post_data["input"], str):
+            num_inputs = 1
+        else:
+            num_inputs = len(post_data["input"])
+
+        model = post_data.get("model", "text-embedding-ada-002")
+
+        data = []
+        for i in range(num_inputs):
+            data.append({
+                "object": "embedding",
+                "embedding": [0.1] * 1536,
+                "index": i,
+            })
+
+        response = {
+            "object": "list",
+            "data": data,
+            "model": model,
+            "usage": {
+                "prompt_tokens": 0,
+                "total_tokens": 0,
+            }
+        }
+
+        self.send_response(200)
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode("utf-8"))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
+    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
+    args = parser.parse_args()
+    port = args.port
+
+    print(f"server started on port {port}. Press Ctrl-C to stop.")
+    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
+
+    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
+        server.serve_forever()

ci/parse_requirements.py

@@ -0,0 +1,41 @@
+import argparse
+import toml
+
+
+def parse_dependencies(pyproject_path, extras=None):
+    with open(pyproject_path, "r") as file:
+        pyproject = toml.load(file)
+
+    dependencies = pyproject.get("project", {}).get("dependencies", [])
+    for dependency in dependencies:
+        print(dependency)
+
+    optional_dependencies = pyproject.get("project", {}).get(
+        "optional-dependencies", {}
+    )
+
+    if extras:
+        for extra in extras.split(","):
+            for dep in optional_dependencies.get(extra, []):
+                print(dep)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate requirements.txt from pyproject.toml"
+    )
+    parser.add_argument("path", type=str, help="Path to pyproject.toml")
+    parser.add_argument(
+        "--extras",
+        type=str,
+        help="Comma-separated list of extras to include",
+        default="",
+    )
+
+    args = parser.parse_args()
+
+    parse_dependencies(args.path, args.extras)
+
+
+if __name__ == "__main__":
+    main()

ci/set_lance_version.py

@@ -0,0 +1,174 @@
+import argparse
+import sys
+import json
+
+
+def run_command(command: str) -> str:
+    """
+    Run a shell command and return stdout as a string.
+    If exit code is not 0, raise an exception with the stderr output.
+    """
+    import subprocess
+
+    result = subprocess.run(command, shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise Exception(f"Command failed with error: {result.stderr.strip()}")
+    return result.stdout.strip()
+
+
+def get_latest_stable_version() -> str:
+    version_line = run_command("cargo info lance | grep '^version:'")
+    version = version_line.split(" ")[1].strip()
+    return version
+
+
+def get_latest_preview_version() -> str:
+    lance_tags = run_command(
+        "git ls-remote --tags https://github.com/lancedb/lance.git | grep 'refs/tags/v[0-9beta.-]\\+$'"
+    ).splitlines()
+    lance_tags = (
+        tag.split("refs/tags/")[1]
+        for tag in lance_tags
+        if "refs/tags/" in tag and "beta" in tag
+    )
+    from packaging.version import Version
+
+    latest = max(
+        (tag[1:] for tag in lance_tags if tag.startswith("v")), key=lambda t: Version(t)
+    )
+    return str(latest)
+
+
+def extract_features(line: str) -> list:
+    """
+    Extracts the features from a line in Cargo.toml.
+    Example: 'lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }'
+    Returns: ['dynamodb']
+    """
+    import re
+
+    match = re.search(r'"features"\s*=\s*\[(.*?)\]', line)
+    if match:
+        features_str = match.group(1)
+        return [f.strip('"') for f in features_str.split(",")]
+    return []
+
+
+def update_cargo_toml(line_updater):
+    """
+    Updates the Cargo.toml file by applying the line_updater function to each line.
+    The line_updater function should take a line as input and return the updated line.
+    """
+    with open("Cargo.toml", "r") as f:
+        lines = f.readlines()
+
+    new_lines = []
+    for line in lines:
+        if line.startswith("lance"):
+            # Update the line using the provided function
+            new_lines.append(line_updater(line))
+        else:
+            # Keep the line unchanged
+            new_lines.append(line)
+
+    with open("Cargo.toml", "w") as f:
+        f.writelines(new_lines)
+
+
+def set_stable_version(version: str):
+    """
+    Sets lines to
+    lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }
+    lance-io = "=0.29.0"
+    ...
+    """
+
+    def line_updater(line: str) -> str:
+        package_name = line.split("=", maxsplit=1)[0].strip()
+        features = extract_features(line)
+        if features:
+            return f'{package_name} = {{ "version" = "={version}", "features" = {json.dumps(features)} }}\n'
+        else:
+            return f'{package_name} = "={version}"\n'
+
+    update_cargo_toml(line_updater)
+
+
+def set_preview_version(version: str):
+    """
+    Sets lines to
+    lance = { "version" = "=0.29.0", "features" = ["dynamodb"], tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" }
+    lance-io = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" }
+    ...
+    """
+
+    def line_updater(line: str) -> str:
+        package_name = line.split("=", maxsplit=1)[0].strip()
+        features = extract_features(line)
+        base_version = version.split("-")[0]  # Get the base version without beta suffix
+        if features:
+            return f'{package_name} = {{ "version" = "={base_version}", "features" = {json.dumps(features)}, "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n'
+        else:
+            return f'{package_name} = {{ "version" = "={base_version}", "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n'
+
+    update_cargo_toml(line_updater)
+
+
+def set_local_version():
+    """
+    Sets lines to
+    lance = { path = "../lance/rust/lance", features = ["dynamodb"] }
+    lance-io = { path = "../lance/rust/lance-io" }
+    ...
+    """
+
+    def line_updater(line: str) -> str:
+        package_name = line.split("=", maxsplit=1)[0].strip()
+        features = extract_features(line)
+        if features:
+            return f'{package_name} = {{ "path" = "../lance/rust/{package_name}", "features" = {json.dumps(features)} }}\n'
+        else:
+            return f'{package_name} = {{ "path" = "../lance/rust/{package_name}" }}\n'
+
+    update_cargo_toml(line_updater)
+
+
+parser = argparse.ArgumentParser(description="Set the version of the Lance package.")
+parser.add_argument(
+    "version",
+    type=str,
+    help="The version to set for the Lance package. Use 'stable' for the latest stable version, 'preview' for latest preview version, or a specific version number (e.g., '0.1.0'). You can also specify 'local' to use a local path.",
+)
+args = parser.parse_args()
+
+if args.version == "stable":
+    latest_stable_version = get_latest_stable_version()
+    print(
+        f"Found latest stable version: \033[1mv{latest_stable_version}\033[0m",
+        file=sys.stderr,
+    )
+    set_stable_version(latest_stable_version)
+elif args.version == "preview":
+    latest_preview_version = get_latest_preview_version()
+    print(
+        f"Found latest preview version: \033[1mv{latest_preview_version}\033[0m",
+        file=sys.stderr,
+    )
+    set_preview_version(latest_preview_version)
+elif args.version == "local":
+    set_local_version()
+else:
+    # Parse the version number.
+    version = args.version
+    # Ignore initial v if present.
+    if version.startswith("v"):
+        version = version[1:]
+
+    if "beta" in version:
+        set_preview_version(version)
+    else:
+        set_stable_version(version)
+
+print("Updating lockfiles...", file=sys.stderr, end="")
+run_command("cargo metadata > /dev/null")
+print(" done.", file=sys.stderr)

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+
+# dbghelp.lib fwpuclnt.lib arm64rt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# ARM intrinsics
+# original dir: MSVC/
+
+# '__n128x4' redefined in arm_neon.h
+# "arm64_neon.h" included from intrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+# .lib
+
+# _Interlocked intrinsics
+# must always link with arm64rt.lib
+# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
+# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
+
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib

ci/sysroot-x86_64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
+
+# dbghelp.lib fwpuclnt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# x86 intrinsics
+# original dir: MSVC/
+
+# '_mm_movemask_epi8' defined in emmintrin.h
+# '__v4sf' defined in xmmintrin.h
+# '__v2si' defined in mmintrin.h
+# '__m128d' redefined in immintrin.h
+# '__m128i' redefined in intrin.h
+# '_mm_comlt_epu8' defined in ammintrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+# .lib
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib

ci/update_lockfiles.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+AMEND=false
+
+for arg in "$@"; do
+  if [[ "$arg" == "--amend" ]]; then
+    AMEND=true
+  fi
+done
+
+# This updates the lockfile without building
+cargo metadata --quiet > /dev/null
+
+pushd nodejs || exit 1
+npm install --package-lock-only --silent
+popd
+pushd node || exit 1
+npm install --package-lock-only --silent
+popd
+
+if git diff --quiet --exit-code; then
+  echo "No lockfile changes to commit; skipping amend."
+elif $AMEND; then
+  git add Cargo.lock nodejs/package-lock.json node/package-lock.json
+  git commit --amend --no-edit
+else
+  git add Cargo.lock nodejs/package-lock.json node/package-lock.json
+  git commit -m "Update lockfiles"
+fi

ci/validate_stable_lance.py

@@ -0,0 +1,34 @@
+import tomllib
+
+found_preview_lance = False
+
+with open("Cargo.toml", "rb") as f:
+    cargo_data = tomllib.load(f)
+
+    for name, dep in cargo_data["workspace"]["dependencies"].items():
+        if name == "lance" or name.startswith("lance-"):
+            if isinstance(dep, str):
+                version = dep
+            elif isinstance(dep, dict):
+                # Version doesn't have the beta tag in it, so we instead look
+                # at the git tag.
+                version = dep.get('tag', dep.get('version'))
+            else:
+                raise ValueError("Unexpected type for dependency: " + str(dep))
+
+            if "beta" in version:
+                found_preview_lance = True
+                print(f"Dependency '{name}' is a preview version: {version}")
+
+with open("python/pyproject.toml", "rb") as f:
+    py_proj_data = tomllib.load(f)
+
+    for dep in py_proj_data["project"]["dependencies"]:
+        if dep.startswith("pylance"):
+            if "b" in dep:
+                found_preview_lance = True
+                print(f"Dependency '{dep}' is a preview version")
+            break  # Only one pylance dependency
+
+if found_preview_lance:
+    raise ValueError("Found preview version of Lance in dependencies")

docs/README.md

@@ -2,43 +2,88 @@
 
 LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
 
-Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+Docs is built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
 unreleased features.
 
 ## Building the docs
 
 ### Setup
-1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
-2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+   Run `make develop` to install the Python package.
-3. Make sure you have node and npm setup
+2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
-4. Make sure protobuf and libssl are installed
 
-### Building node module and create markdown files
+### Preview the docs
 
-See [Javascript docs README](./src/javascript/README.md)
+```shell
-
-### Build docs
-From LanceDB repo root:
-
-Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-### Run local server
-
-You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
-
-```bash
 cd docs
 mkdocs serve
 ```
 
-### Run doctest for typescript example
+If you want to just generate the HTML files:
 
-```bash
+```shell
-cd lancedb/docs
+PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
-npm i
+```
-npm run build
+
-npm run all
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+## Adding examples
+
+To make sure examples are correct, we put examples in test files so they can be
+run as part of our test suites.
+
+You can see the tests are at:
+
+* Python: `python/python/tests/docs`
+* Typescript: `nodejs/examples/`
+
+### Checking python examples
+
+```shell
+cd python
+pytest -vv python/tests/docs
+```
+
+### Checking typescript examples
+
+The `@lancedb/lancedb` package must be built before running the tests:
+
+```shell
+pushd nodejs
+npm ci
+npm run build
+popd
+```
+
+Then you can run the examples by going to the `nodejs/examples` directory and
+running the tests like a normal npm package:
+
+```shell
+pushd nodejs/examples
+npm ci
+npm test
+popd
+```
+
+## API documentation
+
+### Python
+
+The Python API documentation is organized based on the file `docs/src/python/python.md`.
+We manually add entries there so we can control the organization of the reference page.
+**However, this means any new types must be manually added to the file.** No additional
+steps are needed to generate the API documentation.
+
+### Typescript
+
+The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
+
+When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
+The new files should be checked into the repository.
+
+```shell
+pushd nodejs
+npm run docs
+popd
 ```

docs/mkdocs.yml

@@ -4,6 +4,9 @@ repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
+watch:
+  - src
+  - ../python/python
 
 theme:
   name: "material"
@@ -34,6 +37,7 @@ theme:
     - navigation.footer
     - navigation.tracking
     - navigation.instant
+    - content.footnote.tooltips
   icon:
     repo: fontawesome/brands/github
     annotation: material/arrow-right-circle
@@ -54,10 +58,15 @@ plugins:
             show_signature_annotations: true
             show_root_heading: true
             members_order: source
+            docstring_section_style: list
+            signature_crossrefs: true
+            separate_signature: true
           import:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
+            - https://lancedb.github.io/lance/objects.inv
+            - https://docs.pydantic.dev/latest/objects.inv
   - mkdocs-jupyter
   - render_swagger:
       allow_arbitrary_locations: true
@@ -65,6 +74,11 @@ plugins:
 markdown_extensions:
   - admonition
   - footnotes
+  - pymdownx.critic
+  - pymdownx.caret
+  - pymdownx.keys
+  - pymdownx.mark
+  - pymdownx.tilde
   - pymdownx.details
   - pymdownx.highlight:
       anchor_linenums: true
@@ -84,6 +98,9 @@ markdown_extensions:
   - pymdownx.emoji:
       emoji_index: !!python/name:material.extensions.emoji.twemoji
       emoji_generator: !!python/name:material.extensions.emoji.to_svg
+  - markdown.extensions.toc:
+      baselevel: 1
+      permalink: ""
 
 nav:
   - Home:
@@ -92,20 +109,36 @@ nav:
       - 📚 Concepts:
           - Vector search: concepts/vector_search.md
           - Indexing:
-            - IVFPQ: concepts/index_ivfpq.md
+              - IVFPQ: concepts/index_ivfpq.md
-            - HNSW: concepts/index_hnsw.md
+              - HNSW: concepts/index_hnsw.md
           - Storage: concepts/storage.md
           - Data management: concepts/data_management.md
       - 🔨 Guides:
           - Working with tables: guides/tables.md
           - Building a vector index: ann_indexes.md
           - Vector Search: search.md
-          - Full-text search: fts.md
+          - Full-text search (native): fts.md
+          - Full-text search (tantivy-based): fts_tantivy.md
           - Building a scalar index: guides/scalar_index.md
           - Hybrid search:
               - Overview: hybrid_search/hybrid_search.md
               - Comparing Rerankers: hybrid_search/eval.md
               - Airbnb financial data example: notebooks/hybrid_search.ipynb
+          - Late interaction with MultiVector search:
+              - Overview: guides/multi-vector.md
+              - Example: notebooks/Multivector_on_LanceDB.ipynb
+          - RAG:
+              - Vanilla RAG: rag/vanilla_rag.md
+              - Multi-head RAG: rag/multi_head_rag.md
+              - Corrective RAG: rag/corrective_rag.md
+              - Agentic RAG: rag/agentic_rag.md
+              - Graph RAG: rag/graph_rag.md
+              - Self RAG: rag/self_rag.md
+              - Adaptive RAG: rag/adaptive_rag.md
+              - SFR RAG: rag/sfr_rag.md
+              - Advanced Techniques:
+                  - HyDE: rag/advanced_techniques/hyde.md
+                  - FLARE: rag/advanced_techniques/flare.md
           - Reranking:
               - Quickstart: reranking/index.md
               - Cohere Reranker: reranking/cohere.md
@@ -116,10 +149,13 @@ nav:
               - Jina Reranker: reranking/jina.md
               - OpenAI Reranker: reranking/openai.md
               - AnswerDotAi Rerankers: reranking/answerdotai.md
+              - Voyage AI Rerankers: reranking/voyageai.md
               - Building Custom Rerankers: reranking/custom_reranker.md
               - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
-          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+          - Versioning & Reproducibility:
+              - sync API: notebooks/reproducibility.ipynb
+              - async API: notebooks/reproducibility_async.ipynb
           - Configuring Storage: guides/storage.md
           - Migration Guide: migration.md
           - Tuning retrieval performance:
@@ -127,7 +163,8 @@ nav:
               - Reranking: guides/tuning_retrievers/2_reranking.md
               - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
       - 🧬 Managing embeddings:
-          - Overview: embeddings/index.md
+          - Understand Embeddings: embeddings/understanding_embeddings.md
+          - Get Started: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
           - Available models:
               - Overview: embeddings/default_embedding_functions.md
@@ -142,11 +179,13 @@ nav:
                   - Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
                   - AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
                   - IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
+                  - Voyage AI Embeddings: embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
               - Multimodal Embedding Functions:
                   - OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
                   - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
                   - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
           - User-defined embedding functions: embeddings/custom_embedding_function.md
+          - Variables and secrets: embeddings/variables_and_secrets.md
           - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
           - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
       - 🔌 Integrations:
@@ -154,6 +193,7 @@ nav:
           - Pandas and PyArrow: python/pandas_and_pyarrow.md
           - Polars: python/polars_arrow.md
           - DuckDB: python/duckdb.md
+          - Datafusion: python/datafusion.md
           - LangChain:
               - LangChain 🔗: integrations/langchain.md
               - LangChain demo: notebooks/langchain_demo.ipynb
@@ -165,6 +205,8 @@ nav:
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
           - dlt: integrations/dlt.md
+          - phidata: integrations/phidata.md
+          - Genkit: integrations/genkit.md
       - 🎯 Examples:
           - Overview: examples/index.md
           - 🐍 Python:
@@ -187,39 +229,51 @@ nav:
               - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
           - 🦀 Rust:
               - Overview: examples/examples_rust.md
-      - Studies:
+      - 📓 Studies:
           - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
       - 💭 FAQs: faq.md
+      - 🔍 Troubleshooting: troubleshooting.md
       - ⚙️ API reference:
           - 🐍 Python: python/python.md
           - 👾 JavaScript (vectordb): javascript/modules.md
           - 👾 JavaScript (lancedb): js/globals.md
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
-      - ☁️ LanceDB Cloud:
-          - Overview: cloud/index.md
-          - API reference:
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
 
   - Quick start: basic.md
   - Concepts:
       - Vector search: concepts/vector_search.md
       - Indexing:
-        - IVFPQ: concepts/index_ivfpq.md
+          - IVFPQ: concepts/index_ivfpq.md
-        - HNSW: concepts/index_hnsw.md
+          - HNSW: concepts/index_hnsw.md
       - Storage: concepts/storage.md
       - Data management: concepts/data_management.md
   - Guides:
       - Working with tables: guides/tables.md
+      - Working with SQL: guides/sql_querying.md
       - Building an ANN index: ann_indexes.md
       - Vector Search: search.md
-      - Full-text search: fts.md
+      - Full-text search (native): fts.md
+      - Full-text search (tantivy-based): fts_tantivy.md
       - Building a scalar index: guides/scalar_index.md
       - Hybrid search:
           - Overview: hybrid_search/hybrid_search.md
           - Comparing Rerankers: hybrid_search/eval.md
           - Airbnb financial data example: notebooks/hybrid_search.ipynb
+      - Late interaction with MultiVector search:
+          - Overview: guides/multi-vector.md
+          - Document search Example: notebooks/Multivector_on_LanceDB.ipynb
+      - RAG:
+          - Vanilla RAG: rag/vanilla_rag.md
+          - Multi-head RAG: rag/multi_head_rag.md
+          - Corrective RAG: rag/corrective_rag.md
+          - Agentic RAG: rag/agentic_rag.md
+          - Graph RAG: rag/graph_rag.md
+          - Self RAG: rag/self_rag.md
+          - Adaptive RAG: rag/adaptive_rag.md
+          - SFR RAG: rag/sfr_rag.md
+          - Advanced Techniques:
+              - HyDE: rag/advanced_techniques/hyde.md
+              - FLARE: rag/advanced_techniques/flare.md
       - Reranking:
           - Quickstart: reranking/index.md
           - Cohere Reranker: reranking/cohere.md
@@ -233,7 +287,9 @@ nav:
           - Building Custom Rerankers: reranking/custom_reranker.md
           - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
-      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+      - Versioning & Reproducibility:
+          - sync API: notebooks/reproducibility.ipynb
+          - async API: notebooks/reproducibility_async.ipynb
       - Configuring Storage: guides/storage.md
       - Migration Guide: migration.md
       - Tuning retrieval performance:
@@ -241,7 +297,8 @@ nav:
           - Reranking: guides/tuning_retrievers/2_reranking.md
           - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
   - Managing Embeddings:
-      - Overview: embeddings/index.md
+      - Understand Embeddings: embeddings/understanding_embeddings.md
+      - Get Started: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
       - Available models:
           - Overview: embeddings/default_embedding_functions.md
@@ -261,6 +318,7 @@ nav:
               - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
               - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
       - User-defined embedding functions: embeddings/custom_embedding_function.md
+      - Variables and secrets: embeddings/variables_and_secrets.md
       - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
       - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
   - Integrations:
@@ -268,6 +326,7 @@ nav:
       - Pandas and PyArrow: python/pandas_and_pyarrow.md
       - Polars: python/polars_arrow.md
       - DuckDB: python/duckdb.md
+      - Datafusion: python/datafusion.md
       - LangChain 🦜️🔗↗: integrations/langchain.md
       - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
       - LlamaIndex 🦙↗: integrations/llamaIndex.md
@@ -275,6 +334,8 @@ nav:
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
       - dlt: integrations/dlt.md
+      - phidata: integrations/phidata.md
+      - Genkit: integrations/genkit.md
   - Examples:
       - examples/index.md
       - 🐍 Python:
@@ -298,20 +359,14 @@ nav:
       - 🦀 Rust:
           - Overview: examples/examples_rust.md
   - Studies:
-    - studies/overview.md
+      - studies/overview.md
-    - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
+      - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
   - API reference:
       - Overview: api_reference.md
       - Python: python/python.md
       - Javascript (vectordb): javascript/modules.md
       - Javascript (lancedb): js/globals.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
-  - LanceDB Cloud:
-      - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
 
 extra_css:
   - styles/global.css
@@ -319,6 +374,7 @@ extra_css:
 
 extra_javascript:
   - "extra_js/init_ask_ai_widget.js"
+  - "extra_js/reo.js"
 
 extra:
   analytics:

docs/openapi.yml

@@ -38,6 +38,13 @@ components:
       required: true
       schema:
         type: string
+    index_name:
+      name: index_name
+      in: path
+      description: name of the index
+      required: true
+      schema:
+        type: string
   responses:
     invalid_request:
       description: Invalid request
@@ -164,7 +171,7 @@ paths:
                 distance_type:
                   type: string
                   description: |
-                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
+                    The distance metric to use for search. l2, Cosine, Dot and Hamming are supported. Default is l2.
                 bypass_vector_index:
                   type: boolean
                   description: |
@@ -443,7 +450,7 @@ paths:
                   type: string
                   nullable: false
                   description: |
-                    The metric type to use for the index. L2, Cosine, Dot are supported.
+                    The metric type to use for the index. l2, Cosine, Dot are supported.
                 index_type:
                   type: string
       responses:
@@ -485,3 +492,22 @@ paths:
           $ref: "#/components/responses/unauthorized"
         "404":
           $ref: "#/components/responses/not_found"
+  /v1/table/{name}/index/{index_name}/drop/:
+    post:
+      description: Drop an index from the table
+      tags:
+        - Tables
+      summary: Drop an index from the table
+      operationId: dropIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+        - $ref: "#/components/parameters/index_name"
+      responses:
+        "200":
+          description: Index successfully dropped
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"

docs/overrides/partials/main.html

@@ -0,0 +1,5 @@
+{% extends "base.html" %}
+
+{% block announce %}
+  📚 Starting June 1st, 2025, please use <a href="https://lancedb.github.io/documentation" target="_blank" rel="noopener noreferrer">lancedb.github.io/documentation</a> for the latest docs.
+{% endblock %}

docs/package-lock.json

@@ -19,7 +19,7 @@
     },
     "../node": {
       "name": "vectordb",
-      "version": "0.4.6",
+      "version": "0.12.0",
       "cpu": [
         "x64",
         "arm64"
@@ -31,9 +31,7 @@
         "win32"
       ],
       "dependencies": {
-        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -46,6 +44,7 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -62,15 +61,19 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "*",
+        "typescript": "^5.1.0",
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.6",
+        "@lancedb/vectordb-darwin-arm64": "0.12.0",
-        "@lancedb/vectordb-darwin-x64": "0.4.6",
+        "@lancedb/vectordb-darwin-x64": "0.12.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.12.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-x64-gnu": "0.12.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
+        "@lancedb/vectordb-win32-x64-msvc": "0.12.0"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
       }
     },
     "../node/node_modules/apache-arrow": {

docs/src/ann_indexes.md

@@ -18,25 +18,24 @@ See the [indexing](concepts/index_ivfpq.md) concepts guide for more information
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
+    === "Sync API"
 
-    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-    ```python
+        ```python
-    import lancedb
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-    import numpy as np
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-    uri = "data/sample-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index"
-    db = lancedb.connect(uri)
+        ```
+    === "Async API"
+        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-    # Create 10,000 sample vectors
+        ```python
-    data = [{"vector": row, "item": f"item {i}"}
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-ivfpq"
-    # Add the vectors to a table
+        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index_async"
-    tbl = db.create_table("my_vectors", data=data)
+        ```
-
-    # Create and train the index - you need to have enough data in the table for an effective training step
-    tbl.create_index(num_partitions=256, num_sub_vectors=96)
-    ```
 
 === "TypeScript"
 
@@ -45,9 +44,9 @@ Lance supports `IVF_PQ` index type by default.
         Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
 
         ```typescript
-        --8<--- "nodejs/examples/ann_indexes.ts:import"
+        --8<--- "nodejs/examples/ann_indexes.test.ts:import"
 
-        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:ingest"
         ```
 
     === "vectordb (deprecated)"
@@ -70,7 +69,7 @@ Lance supports `IVF_PQ` index type by default.
 
 The following IVF_PQ paramters can be specified:
 
-- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
+- **distance_type**: The distance metric to use. By default it uses euclidean distance "`l2`".
   We also support "cosine" and "dot" distance as well.
 - **num_partitions**: The number of partitions in the index. The default is the square root
   of the number of rows.
@@ -83,6 +82,7 @@ The following IVF_PQ paramters can be specified:
 - **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
   For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
   a single PQ code. The default is the dimension of the vector divided by 16.
+- **num_bits**: The number of bits used to encode each sub-vector. Only 4 and 8 are supported. The higher the number of bits, the higher the accuracy of the index, also the slower search. The default is 8.
 
 !!! note
 
@@ -126,6 +126,8 @@ You can specify the GPU device to train IVF partitions via
         accelerator="mps"
     )
     ```
+!!! note
+    GPU based indexing is not yet supported with our asynchronous client.
     
 Troubleshooting:
 
@@ -140,23 +142,27 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 - **limit** (default: 10): The amount of results that will be returned
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
+  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
-  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
+    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
-  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
   A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
+    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
-  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+!!! note
+    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
+
 
 === "Python"
+    === "Sync API"
 
-    ```python
+        ```python
-    tbl.search(np.random.random((1536))) \
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search"
-        .limit(2) \
+        ```
-        .nprobes(20) \
+    === "Async API"
-        .refine_factor(10) \
+
-        .to_pandas()
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async"
+        ```
 
     ```text
                                               vector       item       _distance
@@ -169,7 +175,7 @@ There are a couple of parameters that can be used to fine-tune the search:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search1"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search1"
         ```
 
     === "vectordb (deprecated)"
@@ -193,17 +199,23 @@ The search will return the data requested in addition to the distance of each it
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
+    === "Sync API"
 
-    ```python
+        ```python
-    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_filter"
-    ```
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_filter"
+        ```
 
 === "TypeScript"
 
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search2"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search2"
         ```
 
     === "vectordb (deprecated)"
@@ -218,10 +230,16 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
-    ```
 
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_select"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_select"
+        ```
 
     ```text
                                                 vector _distance
@@ -235,7 +253,7 @@ You can select the columns returned by the query using a select clause.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search3"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search3"
         ```
 
     === "vectordb (deprecated)"
@@ -273,9 +291,17 @@ Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` t
 
 `num_partitions` is used to decide how many partitions the first level `IVF` index uses.
 Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
-On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
+On `SIFT-1M` dataset, our benchmark shows that keeping each partition 4K-8K rows lead to a good latency / recall.
 
-`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
+`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. The number should be a factor of the vector dimension. Because
 PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
-less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
+less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
-more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+
+!!! note
+    if `num_sub_vectors` is set to be greater than the vector dimension, you will see errors like `attempt to divide by zero`
+
+### How to choose `m` and `ef_construction` for `IVF_HNSW_*` index?
+
+`m` determines the number of connections a new node establishes with its closest neighbors upon entering the graph. Typically, `m` falls within the range of 5 to 48. Lower `m` values are suitable for low-dimensional data or scenarios where recall is less critical. Conversely, higher `m` values are beneficial for high-dimensional data or when high recall is required. In essence, a larger `m` results in a denser graph with increased connectivity, but at the expense of higher memory consumption.
+
+`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase

docs/src/ann_indexes.ts

@@ -3,6 +3,7 @@ import * as vectordb from "vectordb";
 // --8<-- [end:import]
 
 (async () => {
+  console.log("ann_indexes.ts: start");
   // --8<-- [start:ingest]
   const db = await vectordb.connect("data/sample-lancedb");
 
@@ -49,5 +50,5 @@ import * as vectordb from "vectordb";
     .execute();
   // --8<-- [end:search3]
 
-  console.log("Ann indexes: done");
+  console.log("ann_indexes.ts: done");
 })();

docs/src/basic.md

@@ -133,21 +133,22 @@ recommend switching to stable releases.
 ## Connect to a database
 
 === "Python"
+    === "Sync API"
 
-    ```python
+        ```python
-    --8<-- "python/python/tests/docs/test_basic.py:imports"
+        --8<-- "python/python/tests/docs/test_basic.py:imports"
-    --8<-- "python/python/tests/docs/test_basic.py:connect"
 
-    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
+        --8<-- "python/python/tests/docs/test_basic.py:set_uri"
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:connect"
+        ```
+    === "Async API"
 
-    !!! note "Asynchronous Python API"
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:imports"
 
-        The asynchronous Python API is new and has some slight differences compared
+        --8<-- "python/python/tests/docs/test_basic.py:set_uri"
-        to the synchronous API.  Feel free to start using the asynchronous version.
+        --8<-- "python/python/tests/docs/test_basic.py:connect_async"
-        Once all features have migrated we will start to move the synchronous API to
+        ```
-        use the same syntax as the asynchronous API.  To help with this migration we
-        have created a [migration guide](migration.md) detailing the differences.
 
 === "Typescript[^1]"
 
@@ -157,7 +158,7 @@ recommend switching to stable releases.
         import * as lancedb from "@lancedb/lancedb";
         import * as arrow from "apache-arrow";
 
-        --8<-- "nodejs/examples/basic.ts:connect"
+        --8<-- "nodejs/examples/basic.test.ts:connect"
         ```
 
     === "vectordb (deprecated)"
@@ -191,28 +192,40 @@ table.
 
 === "Python"
 
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table"
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
-    ```
-
     If the table already exists, LanceDB will raise an error by default.
     If you want to overwrite the table, you can pass in `mode="overwrite"`
     to the `create_table` method.
 
-    You can also pass in a pandas DataFrame directly:
+    === "Sync API"
 
-    ```python
+        ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
+        --8<-- "python/python/tests/docs/test_basic.py:create_table"
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
+        ```
-    ```
+
+        You can also pass in a pandas DataFrame directly:
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
+        ```
+
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
+        ```
+
+        You can also pass in a pandas DataFrame directly:
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
+        ```
 
 === "Typescript[^1]"
 
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_table"
         ```
 
     === "vectordb (deprecated)"
@@ -255,10 +268,16 @@ similar to a `CREATE TABLE` statement in SQL.
 
 === "Python"
 
-      ```python
+    === "Sync API"
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
+
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
+        ```python
-      ```
+        --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
+        ```
 
     !!! note "You can define schema in Pydantic"
         LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
@@ -268,7 +287,7 @@ similar to a `CREATE TABLE` statement in SQL.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
         ```
 
     === "vectordb (deprecated)"
@@ -289,16 +308,22 @@ Once created, you can open a table as follows:
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:open_table"
+
-    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:open_table"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
+        ```
 
 === "Typescript[^1]"
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:open_table"
+        --8<-- "nodejs/examples/basic.test.ts:open_table"
         ```
 
     === "vectordb (deprecated)"
@@ -318,16 +343,22 @@ If you forget the name of your table, you can always get a listing of all table
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:table_names"
+
-    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:table_names"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
+        ```
 
 === "Typescript[^1]"
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:table_names"
+        --8<-- "nodejs/examples/basic.test.ts:table_names"
         ```
 
     === "vectordb (deprecated)"
@@ -348,16 +379,22 @@ After a table has been created, you can always add more data to it as follows:
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:add_data"
+
-    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:add_data"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
+        ```
 
 === "Typescript[^1]"
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:add_data"
+        --8<-- "nodejs/examples/basic.test.ts:add_data"
         ```
 
     === "vectordb (deprecated)"
@@ -378,10 +415,16 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
+
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:vector_search"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
+        ```
 
     This returns a pandas DataFrame with the results.
 
@@ -389,7 +432,7 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:vector_search"
+        --8<-- "nodejs/examples/basic.test.ts:vector_search"
         ```
 
     === "vectordb (deprecated)"
@@ -420,16 +463,22 @@ LanceDB allows you to create an ANN index on a table as follows:
 
 === "Python"
 
-    ```py
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:create_index"
+
-    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:create_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
+        ```
 
 === "Typescript[^1]"
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_index"
+        --8<-- "nodejs/examples/basic.test.ts:create_index"
         ```
 
     === "vectordb (deprecated)"
@@ -459,17 +508,23 @@ This can delete any number of rows that match the filter.
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
+
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
+        ```
 
 === "Typescript[^1]"
 
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:delete_rows"
+        --8<-- "nodejs/examples/basic.test.ts:delete_rows"
         ```
 
     === "vectordb (deprecated)"
@@ -491,7 +546,10 @@ simple or complex as needed. To see what expressions are supported, see the
 
 === "Python"
 
-      Read more: [lancedb.table.Table.delete][]
+    === "Sync API"
+        Read more: [lancedb.table.Table.delete][]
+    === "Async API"
+        Read more: [lancedb.table.AsyncTable.delete][]
 
 === "Typescript[^1]"
 
@@ -513,10 +571,16 @@ Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+
-    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+        ```
 
     This permanently removes the table and is not recoverable, unlike deleting rows.
     By default, if the table does not exist an exception is raised. To suppress this,
@@ -527,7 +591,7 @@ Use the `drop_table()` method on the database to remove a table.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.ts:drop_table"
+        --8<-- "nodejs/examples/basic.test.ts:drop_table"
         ```
 
     === "vectordb (deprecated)"
@@ -551,18 +615,25 @@ You can use the embedding API when working with embedding models. It automatical
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
+
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
+        ```python
-    ```
+        --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
+
+        --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
+        ```
+    === "Async API"
+
+        Coming soon to the async API.
+        https://github.com/lancedb/lancedb/issues/1938
 
 === "Typescript[^1]"
 
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
+        --8<-- "nodejs/examples/embedding.test.ts:openai_embeddings"
         ```
 
 === "Rust"

docs/src/basic_legacy.ts

@@ -107,7 +107,6 @@ const example = async () => {
   // --8<-- [start:search]
   const query = await tbl.search([100, 100]).limit(2).execute();
   // --8<-- [end:search]
-  console.log(query);
 
   // --8<-- [start:delete]
   await tbl.delete('item = "fizz"');
@@ -119,8 +118,9 @@ const example = async () => {
 };
 
 async function main() {
+  console.log("basic_legacy.ts: start");
   await example();
-  console.log("Basic example: done");
+  console.log("basic_legacy.ts: done");
 }
 
 main();

docs/src/cloud/cloud_faq.md

@@ -0,0 +1,34 @@
+This section provides answers to the most common questions asked about LanceDB Cloud. By following these guidelines, you can ensure a smooth, performant experience with LanceDB Cloud.
+
+### Should I reuse the database connection?
+Yes! It is recommended to establish a single database connection and maintain it throughout your interaction with the tables within. 
+
+LanceDB uses HTTP connections to communicate with the servers. By re-using the Connection object, you avoid the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
+
+### Should I re-use the `Table` object?
+`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the **latest version** of the data. 
+
+### What should I do if I need to search for rows by `id`?
+LanceDB Cloud currently does not support an ID or primary key column. You are recommended to add a 
+user-defined ID column. To significantly improve the query performance with SQL causes, a scalar BITMAP/BTREE index should be created on this column. 
+
+### What are the vector indexing types supported by LanceDB Cloud?
+We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff between query latency and query quality.
+
+### When I add new rows to a table, do I need to manually update the index?
+No! LanceDB Cloud triggers an asynchronous background job to index the new vectors.
+
+Even though indexing is asynchronous, your vectors will still be immediately searchable. LanceDB uses brute-force search to search over unindexed rows. This makes you new data is immediately available, but does increase latency temporarily. To disable the brute-force part of search, set the `fast_search` flag in your query to `true`.
+
+### Do I need to reindex the whole dataset if only a small portion of the data is deleted or updated?
+No! Similar to adding data to the table, LanceDB Cloud triggers an asynchronous background job to update the existing indices. Therefore, no action is needed from users and there is absolutely no 
+downtime expected.
+
+### How do I know whether an index has been created?
+While index creation in LanceDB Cloud is generally fast, querying immediately after a `create_index` call may result in errors. It's recommended to use `list_indices` to verify index creation before querying.
+
+### Why is my query latency higher than expected?
+Multiple factors can impact query latency. To reduce query latency, consider the following:
+- Send pre-warm queries: send a few queries to warm up the cache before an actual user query.
+- Check network latency: LanceDB Cloud is hosted in AWS `us-east-1` region. It is recommended to run queries from an EC2 instance that is in the same region.
+- Create scalar indices: If you are filtering on metadata, it is recommended to create scalar indices on those columns. This will speedup searches with metadata filtering. See [here](../guides/scalar_index.md) for more details on creating a scalar index.

docs/src/cloud/index.md

@@ -2,7 +2,7 @@
 
 LanceDB Cloud is a SaaS (software-as-a-service) solution that runs serverless in the cloud, clearly separating storage from compute. It's designed to be highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
 
-[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
+[Try out LanceDB Cloud (Public Beta)](https://cloud.lancedb.com){ .md-button .md-button--primary }
 
 ## Architecture
 

docs/src/concepts/index_hnsw.md

@@ -7,7 +7,7 @@ Approximate Nearest Neighbor (ANN) search is a method for finding data points ne
 There are three main types of ANN search algorithms:
 
 * **Tree-based search algorithms**: Use a tree structure to organize and store data points.
-* * **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
+* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
 * **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. 
 
 HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below.  
@@ -57,6 +57,13 @@ Then the greedy search routine operates as follows:
 
 ## Usage
 
+There are three key parameters to set when constructing an HNSW index:
+
+* `metric`: Use an `l2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `m`: The number of neighbors to select for each vector in the HNSW graph.
+* `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph.
+
+
 We can combine the above concepts to understand how to build and query an HNSW index in LanceDB.
 
 ### Construct index

docs/src/concepts/index_ivfpq.md

@@ -47,7 +47,7 @@ We can combine the above concepts to understand how to build and query an IVF-PQ
 
 There are three key parameters to set when constructing an IVF-PQ index:
 
-* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `metric`: Use an `l2` euclidean distance metric. We also support `dot` and `cosine` distance.
 * `num_partitions`: The number of partitions in the IVF portion of the index.
 * `num_sub_vectors`: The number of sub-vectors that will be created during Product Quantization (PQ).
 
@@ -56,10 +56,12 @@ In Python, the index can be created as follows:
 ```python
 # Create and train the index for a 1536-dimensional vector
 # Make sure you have enough data in the table for an effective training step
-tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
+tbl.create_index(metric="l2", num_partitions=256, num_sub_vectors=96)
 ```
+!!! note
+    `num_partitions`=256 and `num_sub_vectors`=96 does not work for every dataset. Those values needs to be adjusted for your particular dataset.
 
-The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See the [FAQs](#faq) below for best practices on choosing these parameters.
+The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See [here](../ann_indexes.md/#how-to-choose-num_partitions-and-num_sub_vectors-for-ivf_pq-index) for best practices on choosing these parameters.
 
 
 ### Query the index

docs/src/embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md

@@ -1,5 +1,5 @@
 # Huggingface embedding models
-We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
+We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. 
 
 Example usage - 
 ```python

docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md

@@ -6,6 +6,7 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
 | `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
+| `use_azure` | bool | `False` | Set true to use Azure OpenAPI SDK |
 
 
 ```python

docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md

@@ -0,0 +1,51 @@
+# VoyageAI Embeddings
+
+Voyage AI provides cutting-edge embedding and rerankers.
+
+
+Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API.
+
+Supported models are:
+
+- voyage-3
+- voyage-3-lite
+- voyage-finance-2
+- voyage-multilingual-2
+- voyage-law-2
+- voyage-code-2
+
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|--------|---------|
+| `name` | `str` | `None` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
+| `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. |
+| `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. |
+
+
+Usage Example:
+    
+```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    voyageai = EmbeddingFunctionRegistry
+        .get_instance()
+        .get("voyageai")
+        .create(name="voyage-3")
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
+
+    data = [ { "text": "hello world" },
+            { "text": "goodbye world" }]
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+```

docs/src/embeddings/custom_embedding_function.md

@@ -47,14 +47,22 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 === "TypeScript"
 
     ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:imports"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:imports"
 
-    --8<--- "nodejs/examples/custom_embedding_function.ts:embedding_impl"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:embedding_impl"
     ```
 
 
 This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and default settings.
 
+!!! danger "Use sensitive keys to prevent leaking secrets"
+    To prevent leaking secrets, such as API keys, you should add any sensitive
+    parameters of an embedding function to the output of the
+    [sensitive_keys()][lancedb.embeddings.base.EmbeddingFunction.sensitive_keys] /
+    [getSensitiveKeys()](../../js/namespaces/embedding/classes/EmbeddingFunction/#getsensitivekeys)
+    method. This prevents users from accidentally instantiating the embedding
+    function with hard-coded secrets.
+
 Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
 
 === "Python"
@@ -78,7 +86,7 @@ Now you can use this embedding function to create your table schema and that's i
 === "TypeScript"
 
     ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:call_custom_function"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:call_custom_function"
     ```
 
 !!! note

docs/src/embeddings/default_embedding_functions.md

@@ -53,6 +53,7 @@ These functions are registered by default to handle text embeddings.
 | [**Jina Embeddings**](available_embedding_models/text_embedding_functions/jina_embedding.md "jina") | 🔗 World-class embedding models to improve your search and RAG systems. You will need **jina api key**. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="Jina Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/jina_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 | [ **AWS Bedrock Functions**](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md "bedrock-text") | ☁️ AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/aws_bedrock.png" alt="AWS Bedrock Icon" width="120" height="35">](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 | [**IBM Watsonx.ai**](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md "watsonx") | 💡 Generate text embeddings using IBM's watsonx.ai platform. **Note**: watsonx.ai library is an optional dependency. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/watsonx.png" alt="Watsonx Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md) |
+| [**VoyageAI Embeddings**](available_embedding_models/text_embedding_functions/voyageai_embedding.md "voyageai") | 🌕 Voyage AI provides cutting-edge embedding and rerankers. This will help you get started with **VoyageAI** embedding models using LanceDB. Using voyageai API requires voyageai package. Install it via `pip`. | [<img src="https://www.voyageai.com/logo.svg" alt="VoyageAI Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/voyageai_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 
 
 
@@ -66,6 +67,7 @@ These functions are registered by default to handle text embeddings.
 [jina-key]: "jina"
 [aws-key]: "bedrock-text"
 [watsonx-key]: "watsonx"
+[voyageai-key]: "voyageai"
 
 
 ## Multi-modal Embedding Functions🖼️ 

docs/src/embeddings/embedding_functions.md

@@ -94,8 +94,8 @@ the embeddings at all:
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:embedding_function"
+        --8<-- "nodejs/examples/embedding.test.ts:embedding_function"
         ```
 
     === "vectordb (deprecated)"
@@ -150,7 +150,7 @@ need to worry about it when you query the table:
             .toArray()
         ```
 
-    === "vectordb (deprecated)
+    === "vectordb (deprecated)"
 
         ```ts
         const results = await table

docs/src/embeddings/index.md

@@ -51,8 +51,8 @@ LanceDB registers the OpenAI embeddings function in the registry as `openai`. Yo
 === "TypeScript"
 
     ```typescript
-    --8<--- "nodejs/examples/embedding.ts:imports"
+    --8<--- "nodejs/examples/embedding.test.ts:imports"
-    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
+    --8<--- "nodejs/examples/embedding.test.ts:openai_embeddings"
     ```
 
 === "Rust"
@@ -121,12 +121,10 @@ class Words(LanceModel):
     vector: Vector(func.ndims()) = func.VectorField()
 
 table = db.create_table("words", schema=Words)
-table.add(
+table.add([
-    [
+    {"text": "hello world"},
-        {"text": "hello world"},
+    {"text": "goodbye world"}
-        {"text": "goodbye world"}
+])
-    ]
-    )
 
 query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]

docs/src/embeddings/understanding_embeddings.md

@@ -0,0 +1,133 @@
+# Understand Embeddings
+
+The term **dimension** is a synonym for the number of elements in a feature vector. Each feature can be thought of as a different axis in a geometric space. 
+
+High-dimensional data means there are many features(or attributes) in the data.
+
+!!! example
+     1. An image is a data point and it might have thousands of dimensions because each pixel could be considered as a feature. 
+
+     2. Text data, when represented by each word or character, can also lead to high dimensions, especially when considering all possible words in a language.
+
+Embedding captures **meaning and relationships** within data by mapping high-dimensional data into a lower-dimensional space. It captures it by placing inputs that are more **similar in meaning** closer together in the **embedding space**. 
+
+## What are Vector Embeddings?
+
+Vector embeddings is a way to convert complex data, like text, images, or audio into numerical coordinates (called vectors) that can be plotted in an n-dimensional space(embedding space). 
+
+The closer these data points are related in the real world, the closer their corresponding numerical coordinates (vectors) will be to each other in the embedding space. This proximity in the embedding space reflects their semantic similarities, allowing machines to intuitively understand and process the data in a way that mirrors human perception of relationships and meaning.
+
+In a way, it captures the most important aspects of the data while ignoring the less important ones. As a result, tasks like searching for related content or identifying patterns become more efficient and accurate, as the embeddings make it possible to quantify how **closely related** different **data points** are and **reduce** the **computational complexity**.
+
+??? question "Are vectors and embeddings the same thing?"
+
+    When we say “vectors” we mean - **list of numbers** that **represents the data**. 
+    When we say “embeddings” we mean - **list of numbers** that **capture important details and relationships**.
+
+    Although the terms are often used interchangeably, “embeddings” highlight how the data is represented with meaning and structure, while “vector” simply refers to the numerical form of that representation.
+
+## Embedding vs Indexing
+
+We already saw that creating **embeddings** on data is a method of creating **vectors** for a **n-dimensional embedding space** that captures the meaning and relationships inherent in the data.
+
+Once we have these **vectors**, indexing comes into play. Indexing is a method of organizing these vector embeddings, that allows us to quickly and efficiently locate and retrieve them from the entire dataset of vector embeddings.
+
+## What types of data/objects can be embedded?
+
+The following are common types of data that can be embedded:
+
+1. **Text**: Text data includes sentences, paragraphs, documents, or any written content.
+2. **Images**:  Image data encompasses photographs, illustrations, or any visual content.
+3. **Audio**: Audio data includes sounds, music, speech, or any auditory content.
+4. **Video**:  Video data consists of moving images and sound, which can convey complex information.
+
+Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model.
+
+!!! tip "LanceDB vs Other traditional Vector DBs"
+    While many vector databases primarily focus on the storage and retrieval of vector embeddings, **LanceDB** uses **Lance file format** (operates on a disk-based architecture), which allows for the storage and management of not just embeddings but also **raw file data (bytes)**. This capability means that users can integrate various types of data, including images and text, alongside their vector embeddings in a unified system.
+
+    With the ability to store both vectors and associated file data, LanceDB enhances the querying process. Users can perform semantic searches that not only retrieve similar embeddings but also access related files and metadata, thus streamlining the workflow.
+
+## How does embedding works?
+
+As mentioned, after creating embedding, each data point is represented as a vector in a n-dimensional space (embedding space). The dimensionality of this space can vary depending on the complexity of the data and the specific embedding technique used.
+
+Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar. To quantify this closeness, we use distance as a metric which can be measured in the  following way - 
+
+1. **Euclidean Distance (l2)**: It calculates the straight-line distance between two points (vectors) in a multidimensional space.
+2. **Cosine Similarity**: It measures the cosine of the angle between two vectors, providing a normalized measure of similarity based on their direction.
+3. **Dot product**: It is calculated as the sum of the products of their corresponding components. To measure relatedness it considers both the magnitude and direction of the vectors.
+
+## How do you create and store vector embeddings for your data?
+
+1. **Creating embeddings**: Choose an embedding model, it can be a pre-trained model (open-source or commercial) or you can train a custom embedding model for your scenario. Then feed your preprocessed data into the chosen model to obtain embeddings.
+
+??? question "Popular choices for embedding models"
+    For text data, popular choices are OpenAI’s text-embedding models, Google Gemini text-embedding models, Cohere’s Embed models, and SentenceTransformers, etc.
+
+    For image data, popular choices are CLIP (Contrastive Language–Image Pretraining), Imagebind embeddings by meta (supports audio, video, and image), and Jina multi-modal embeddings, etc.
+
+2. **Storing vector embeddings**: This effectively requires **specialized databases** that can handle the complexity of vector data, as traditional databases often struggle with this task. Vector databases are designed specifically for storing and querying vector embeddings. They optimize for efficient nearest-neighbor searches and provide built-in indexing mechanisms.
+
+!!! tip "Why LanceDB"
+    LanceDB **automates** the entire process of creating and storing embeddings for your data. LanceDB allows you to define and use **embedding functions**, which can be **pre-trained models** or **custom models**. 
+    
+    This enables you to **generate** embeddings tailored to the nature of your data (e.g., text, images) and **store** both the **original data** and **embeddings** in a **structured schema** thus providing efficient querying capabilities for similarity searches.
+
+Let's quickly [get started](./index.md) and learn how to manage embeddings in LanceDB. 
+
+## Bonus: As a developer, what you can create using embeddings?
+
+As a developer, you can create a variety of innovative applications using vector embeddings. Check out the following - 
+
+<div class="grid cards" markdown>
+
+-   __Chatbots__
+
+    ---
+
+    Develop chatbots that utilize embeddings to retrieve relevant context and generate coherent, contextually aware responses to user queries.
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/chatbot.md)
+
+-   __Recommendation Systems__
+
+    ---
+
+    Develop systems that recommend content (such as articles, movies, or products) based on the similarity of keywords and descriptions, enhancing user experience.
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/recommendersystem.md)
+
+-   __Vector Search__
+
+    ---
+
+    Build powerful applications that harness the full potential of semantic search, enabling them to retrieve relevant data quickly and effectively. 
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/vector_search.md)
+
+-   __RAG Applications__
+
+    ---
+
+    Combine the strengths of large language models (LLMs) with retrieval-based approaches to create more useful applications.
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/rag.md)
+
+-   __Many more examples__
+
+    ---
+
+    Explore applied examples available as Colab notebooks or Python scripts to integrate into your applications.
+
+    [:octicons-arrow-right-24: More](../examples/examples_python.md)
+
+</div>
+
+
+
+
+
+
+
+

docs/src/embeddings/variables_and_secrets.md

@@ -0,0 +1,53 @@
+# Variable and Secrets
+
+Most embedding configuration options are saved in the table's metadata. However,
+this isn't always appropriate. For example, API keys should never be stored in the
+metadata. Additionally, other configuration options might be best set at runtime,
+such as the `device` configuration that controls whether to use GPU or CPU for
+inference. If you hardcoded this to GPU, you wouldn't be able to run the code on
+a server without one.
+
+To handle these cases, you can set variables on the embedding registry and
+reference them in the embedding configuration. These variables will be available
+during the runtime of your program, but not saved in the table's metadata. When
+the table is loaded from a different process, the variables must be set again.
+
+To set a variable, use the `set_var()` / `setVar()` method on the embedding registry.
+To reference a variable, use the syntax `$env:VARIABLE_NAME`. If there is a default
+value, you can use the syntax `$env:VARIABLE_NAME:DEFAULT_VALUE`.
+
+## Using variables to set secrets
+
+Sensitive configuration, such as API keys, must either be set as environment
+variables or using variables on the embedding registry. If you pass in a hardcoded
+value, LanceDB will raise an error. Instead, if you want to set an API key via
+configuration, use a variable:
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:register_secret"
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/embedding.test.ts:register_secret"
+    ```
+
+## Using variables to set the device parameter
+
+Many embedding functions that run locally have a `device` parameter that controls
+whether to use GPU or CPU for inference. Because not all computers have a GPU,
+it's helpful to be able to set the `device` parameter at runtime, rather than
+have it hard coded in the embedding configuration. To make it work even if the
+variable isn't set, you could provide a default value of `cpu` in the embedding
+configuration.
+
+Some embedding libraries even have a method to detect which devices are available,
+which could be used to dynamically set the device at runtime. For example, in Python
+you can check if a CUDA GPU is available using `torch.cuda.is_available()`.
+
+```python
+--8<-- "python/python/tests/docs/test_embeddings_optional.py:register_device"
+```

docs/src/examples/index.md

@@ -8,9 +8,5 @@ LanceDB provides language APIs, allowing you to embed a database in your languag
 * 👾 [JavaScript](examples_js.md) examples
 * 🦀 Rust examples (coming soon)
 
-## Applications powered by LanceDB
+!!! tip "Hosted LanceDB"
-
+    If you want S3 cost-efficiency and local performance via a simple serverless API, checkout **LanceDB Cloud**. For private deployments, high performance at extreme scale, or if you have strict security requirements, talk to us about **LanceDB Enterprise**. [Learn more](https://docs.lancedb.com/)
-| Project Name | Description |
-| --- | --- |
-| **Ultralytics Explorer 🚀**<br>[![Ultralytics](https://img.shields.io/badge/Ultralytics-Docs-green?labelColor=0f3bc4&style=flat-square&logo=https://cdn.prod.website-files.com/646dd1f1a3703e451ba81ecc/64994922cf2a6385a4bf4489_UltralyticsYOLO_mark_blue.svg&link=https://docs.ultralytics.com/datasets/explorer/)](https://docs.ultralytics.com/datasets/explorer/)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ultralytics/ultralytics/blob/main/docs/en/datasets/explorer/explorer.ipynb) |  - 🔍 **Explore CV Datasets**: Semantic search, SQL queries, vector similarity, natural language.<br>- 🖥️ **GUI & Python API**: Seamless dataset interaction.<br>- ⚡ **Efficient & Scalable**: Leverages LanceDB for large datasets.<br>- 📊 **Detailed Analysis**: Easily analyze data patterns.<br>- 🌐 **Browser GUI Demo**: Create embeddings, search images, run queries. |
-| **Website Chatbot🤖**<br>[![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/lancedb-vercel-chatbot)<br>[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&amp;env=OPENAI_API_KEY&amp;envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&amp;project-name=lancedb-vercel-chatbot&amp;repository-name=lancedb-vercel-chatbot&amp;demo-title=LanceDB%20Chatbot%20Demo&amp;demo-description=Demo%20website%20chatbot%20with%20LanceDB.&amp;demo-url=https%3A%2F%2Flancedb.vercel.app&amp;demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png) | - 🌐 **Chatbot from Sitemap/Docs**: Create a chatbot using site or document context.<br>- 🚀 **Embed LanceDB in Next.js**: Lightweight, on-prem storage.<br>- 🧠 **AI-Powered Context Retrieval**: Efficiently access relevant data.<br>- 🔧 **Serverless & Native JS**: Seamless integration with Next.js.<br>- ⚡ **One-Click Deploy on Vercel**: Quick and easy setup.. |

docs/src/examples/python_examples/chatbot.md

@@ -36,6 +36,6 @@
 [aware_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/chatbot_using_Llama2_&_lanceDB/main.ipynb
 [aware_ghost]: https://blog.lancedb.com/context-aware-chatbot-using-llama-2-lancedb-as-vector-database-4d771d95c755
 
-[csv_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Chat_with_csv_file
+[csv_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Chat_with_csv_file
-[csv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Chat_with_csv_file/main.ipynb
+[csv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Chat_with_csv_file/main.ipynb
 [csv_ghost]: https://blog.lancedb.com/p/d8c71df4-e55f-479a-819e-cde13354a6a3/

docs/src/examples/python_examples/multimodal.md

@@ -12,7 +12,7 @@ LanceDB supports multimodal search by indexing and querying vector representatio
 |:----------------|:-----------------|:-----------|
 | **Multimodal CLIP: DiffusionDB 🌐💥**    | Multi-Modal Search with **CLIP** and **LanceDB** Using **DiffusionDB** Data for Combined Text and Image Understanding ! 🔓 | [![GitHub](../../assets/github.svg)][Clip_diffusionDB_github] <br>[![Open In Collab](../../assets/colab.svg)][Clip_diffusionDB_colab] <br>[![Python](../../assets/python.svg)][Clip_diffusionDB_python] <br>[![Ghost](../../assets/ghost.svg)][Clip_diffusionDB_ghost] |
 | **Multimodal CLIP: Youtube Videos 📹👀** | Search **Youtube videos** using Multimodal CLIP, finding relevant content with ease and accuracy! 🎯                                                                                                                                                                | [![Github](../../assets/github.svg)][Clip_youtube_github]                                                                                                    <br>[![Open In Collab](../../assets/colab.svg)][Clip_youtube_colab]                                                                                           <br> [![Python](../../assets/python.svg)][Clip_youtube_python]         <br>[![Ghost](../../assets/ghost.svg)][Clip_youtube_python] |
-| **Multimodal Image + Text Search 📸🔍**               | Find **relevant documents** and **images** with a single query using **LanceDB's** multimodal search capabilities, to seamlessly integrate text and visuals ! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
+| **Multimodal Image + Text Search 📸🔍**               | Find **relevant documents** and **images** with a single query using **LanceDB's** multimodal search capabilities, to seamlessly integrate text and visuals ! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
 | **Cambrian-1: Vision-Centric Image Exploration 🔍👀** | Learn how **Cambrian-1** works, using an example of **Vision-Centric** exploration on images found through vector search ! Work on **Flickr-8k** dataset 🔎                        | [![Kaggle](https://img.shields.io/badge/Kaggle-035a7d?style=for-the-badge&logo=kaggle&logoColor=white)](https://www.kaggle.com/code/prasantdixit/cambrian-1-vision-centric-exploration-of-images/)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/cambrian-1-vision-centric-exploration/)                                                                                                                                                                                                                   |
 
 

docs/src/examples/python_examples/rag.md

@@ -70,12 +70,12 @@ Build RAG (Retrieval-Augmented Generation) with  LanceDB, a powerful solution fo
 [flare_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb
 [flare_ghost]: https://blog.lancedb.com/better-rag-with-active-retrieval-augmented-generation-flare-3b66646e2a9f/
 
-[query_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker
+[query_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/QueryExpansion%26Reranker
-[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker/main.ipynb
+[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/QueryExpansion&Reranker/main.ipynb
 
 
-[fusion_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion
+[fusion_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/RAG_Fusion
-[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion/main.ipynb
+[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/RAG_Fusion/main.ipynb
 
 [agentic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG
 [agentic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb

docs/src/examples/python_examples/recommendersystem.md

@@ -19,8 +19,8 @@ Deliver personalized experiences with Recommender Systems. 🎁
 [movie_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.py
 
 
-[genre_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommendation-with-genres
+[genre_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/movie-recommendation-with-genres
-[genre_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommendation-with-genres/movie_recommendation_with_doc2vec_and_lancedb.ipynb
+[genre_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/movie-recommendation-with-genres/movie_recommendation_with_doc2vec_and_lancedb.ipynb
 [genre_ghost]: https://blog.lancedb.com/movie-recommendation-system-using-lancedb-and-doc2vec/
 
 [product_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/product-recommender
@@ -33,5 +33,5 @@ Deliver personalized experiences with Recommender Systems. 🎁
 [arxiv_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender/main.py
  
 
-[food_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Food_recommendation
+[food_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Food_recommendation
-[food_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Food_recommendation/main.ipynb
+[food_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Food_recommendation/main.ipynb

docs/src/examples/python_examples/vector_search.md

@@ -37,16 +37,16 @@ LanceDB implements vector search algorithms for efficient document retrieval and
 [NER_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/NER-powered-Semantic-Search/NER_powered_Semantic_Search_with_LanceDB.ipynb
 [NER_ghost]: https://blog.lancedb.com/ner-powered-semantic-search-using-lancedb-51051dc3e493
 
-[audio_search_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/audio_search
+[audio_search_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/audio_search
-[audio_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb
+[audio_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.ipynb
-[audio_search_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.py
+[audio_search_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.py
 
-[mls_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multi-lingual-wiki-qa
+[mls_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multi-lingual-wiki-qa
-[mls_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multi-lingual-wiki-qa/main.ipynb
+[mls_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.ipynb
-[mls_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multi-lingual-wiki-qa/main.py
+[mls_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.py
 
-[fr_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/facial_recognition
+[fr_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/facial_recognition
-[fr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/facial_recognition/main.ipynb
+[fr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/facial_recognition/main.ipynb
 
 [sentiment_analysis_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews
 [sentiment_analysis_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews/Sentiment_Analysis_using_LanceDB.ipynb
@@ -70,8 +70,8 @@ LanceDB implements vector search algorithms for efficient document retrieval and
 [openvino_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Accelerate-Vector-Search-Applications-Using-OpenVINO/clip_text_image_search.ipynb
 [openvino_ghost]: https://blog.lancedb.com/accelerate-vector-search-applications-using-openvino-lancedb/
 
-[zsic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/zero-shot-image-classification
+[zsic_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/zero-shot-image-classification
-[zsic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/zero-shot-image-classification/main.ipynb
+[zsic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/zero-shot-image-classification/main.ipynb
 [zsic_ghost]: https://blog.lancedb.com/zero-shot-image-classification-with-vector-search/
 
 

docs/src/extra_js/reo.js

@@ -0,0 +1 @@
+!function(){var e,t,n;e="9627b71b382d201",t=function(){Reo.init({clientID:"9627b71b382d201"})},(n=document.createElement("script")).src="https://static.reo.dev/"+e+"/reo.js",n.defer=!0,n.onload=t,document.head.appendChild(n)}();

docs/src/fts.md

@@ -1,49 +1,29 @@
-# Full-text search
+# Full-text search (Native FTS)
 
-LanceDB provides support for full-text search via Lance (before via [Tantivy](https://github.com/quickwit-oss/tantivy) (Python only)), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+LanceDB provides support for full-text search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
-
-Currently, the Lance full text search is missing some features that are in the Tantivy full text search. This includes phrase queries, re-ranking, and customizing the tokenizer. Thus, in Python, Tantivy is still the default way to do full text search and many of the instructions below apply just to Tantivy-based indices.
-
-
-## Installation (Only for Tantivy-based FTS)
 
 !!! note
-    No need to install the tantivy dependency if using native FTS
+    The Python SDK uses tantivy-based FTS by default, need to pass `use_tantivy=False` to use native FTS.
-
-To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
-
-```sh
-# Say you want to use tantivy==0.20.1
-pip install tantivy==0.20.1
-```
 
 ## Example
 
 Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
 
 === "Python"
+    === "Sync API"
 
-    ```python
+        ```python
-    import lancedb
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+        --8<-- "python/python/tests/docs/test_search.py:basic_fts"
+        ```
+    === "Async API"
 
-    uri = "data/sample-lancedb"
+        ```python
-    db = lancedb.connect(uri)
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    table = db.create_table(
+        --8<-- "python/python/tests/docs/test_search.py:basic_fts_async"
-        "my_table",
+        ```
-        data=[
-            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
-            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
-        ],
-    )
-
-    # passing `use_tantivy=False` to use lance FTS index
-    # `use_tantivy=True` by default
-    table.create_fts_index("text")
-    table.search("puppy").limit(10).select(["text"]).to_list()
-    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
-    # ...
-    ```
 
 === "TypeScript"
 
@@ -62,7 +42,7 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex
     });
 
     await tbl
-        .search("puppy", queryType="fts")
+        .search("puppy", "fts")
         .select(["text"])
         .limit(10)
         .toArray();
@@ -93,58 +73,104 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex
     ```
 
 It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
-For now, this is supported in tantivy way only.
 
-Passing `fts_columns="text"` if you want to specify the columns to search, but it's not available for Tantivy-based full text search.
+Passing `fts_columns="text"` if you want to specify the columns to search.
 
 !!! note
     LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
 
 ## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+By default the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are with length greater than 40, and lowercase all words.
 
-For now, only the Tantivy-based FTS index supports to specify the tokenizer, so it's only available in Python with `use_tantivy=True`.
+Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.
 
-=== "use_tantivy=True"
+For example, to enable stemming for English:
+=== "Sync API"
 
     ```python
-    table.create_fts_index("text", use_tantivy=True, tokenizer_name="en_stem")
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem"
     ```
+=== "Async API"
 
-=== "use_tantivy=False"
+    ```python
-
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem_async"
-    [**Not supported yet**](https://github.com/lancedb/lance/issues/1195)
+    ```
 
 the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
 
-## Index multiple columns
+The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.
 
-If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
+For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
-
+=== "Sync API"
-=== "use_tantivy=True"
 
     ```python
-    table.create_fts_index(["text1", "text2"])
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding"
     ```
+=== "Async API"
 
-=== "use_tantivy=False"
+    ```python
-
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding_async"
-    [**Not supported yet**](https://github.com/lancedb/lance/issues/1195)
+    ```
-
-Note that the search API call does not change - you can search over all indexed columns at once.
 
 ## Filtering
 
-Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
+LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported.
-applied on top of the full text search results. This can be invoked via the familiar
-`where` syntax:
 
+This can be invoked via the familiar `where` syntax.
+ 
+With pre-filtering:
 === "Python"
 
-    ```python
+    === "Sync API"
-    table.search("puppy").limit(10).where("meta='foo'").to_list()
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering_async"
+        ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl
+    .search("puppy")
+    .select(["id", "doc"])
+    .limit(10)
+    .where("meta='foo'")
+    .prefilter(true)
+    .toArray();
     ```
 
+=== "Rust"
+
+    ```rust
+    table
+        .query()
+        .full_text_search(FullTextSearchQuery::new("puppy".to_owned()))
+        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
+        .limit(10)
+        .only_if("meta='foo'")
+        .execute()
+        .await?;
+    ```
+
+With post-filtering:
+=== "Python"
+
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering_async"
+        ```
+
 === "TypeScript"
 
     ```typescript
@@ -153,6 +179,7 @@ applied on top of the full text search results. This can be invoked via the fami
     .select(["id", "doc"])
     .limit(10)
     .where("meta='foo'")
+    .prefilter(false)
     .toArray();
     ```
 
@@ -163,104 +190,69 @@ applied on top of the full text search results. This can be invoked via the fami
         .query()
         .full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
         .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
+        .postfilter()
         .limit(10)
         .only_if("meta='foo'")
         .execute()
         .await?;
     ```
 
-## Sorting
-
-!!! warning "Warn"
-    Sorting is available for only Tantivy-based FTS
-
-You can pre-sort the documents by specifying `ordering_field_names` when
-creating the full-text search index. Once pre-sorted, you can then specify
-`ordering_field_name` while searching to return results sorted by the given
-field. For example,
-
-```python
-table.create_fts_index(["text_field"], use_tantivy=True, ordering_field_names=["sort_by_field"])
-
-(table.search("terms", ordering_field_name="sort_by_field")
- .limit(20)
- .to_list())
-```
-
-!!! note
-    If you wish to specify an ordering field at query time, you must also
-    have specified it during indexing time. Otherwise at query time, an
-    error will be raised that looks like `ValueError: The field does not exist: xxx`
-
-!!! note
-    The fields to sort on must be of typed unsigned integer, or else you will see
-    an error during indexing that looks like
-    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
-
-!!! note
-    You can specify multiple fields for ordering at indexing time.
-    But at query time only one ordering field is supported.
-
-
 ## Phrase queries vs. terms queries
 
 !!! warning "Warn"
-    Lance-based FTS doesn't support queries combining by boolean operators `OR`, `AND`.
+    Lance-based FTS doesn't support queries using boolean operators `OR`, `AND`.
 
 For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+or a **terms** search query like `old man sea`. For more details on the terms
 query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
 
-!!! tip "Note"
+To search for a phrase, the index must be created with `with_position=True`:
-    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
+=== "Sync API"
 
-    ```py
+    ```python
-    # This raises a syntax error
+    --8<-- "python/python/tests/docs/test_search.py:fts_with_position"
-    table.search("they could have been dogs OR cats")
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_with_position_async"
+    ```
+This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
+
+
+## Incremental indexing
+
+LanceDB supports incremental indexing, which means you can add new records to the table without reindexing the entire table.
+
+This can make the query more efficient, especially when the table is large and the new records are relatively small.
+
+=== "Python"
+
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index_async"
+        ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl.add([{ vector: [3.1, 4.1], text: "Frodo was a happy puppy" }]);
+    await tbl.optimize();
     ```
 
-    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+=== "Rust"
-    the query is treated as a phrase query.
 
-    ```py
+    ```rust
-    # This works!
+    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    table.search("they could have been dogs or cats")
+    tbl.add(more_data).execute().await?;
+    tbl.optimize(OptimizeAction::All).execute().await?;
     ```
+!!! note
 
-It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+    New data added after creating the FTS index will appear in search results while incremental index is still progress, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 
-query you want to perform. To make this simpler, when you want to perform a phrase query, you can
-enforce it in one of two ways:
-
-1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
-a phrase query.
-1. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
-itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
-is treated as a phrase query.
-
-In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
-double quotes replaced by single quotes.
-
-
-## Configurations (Only for Tantivy-based FTS)
-
-By default, LanceDB configures a 1GB heap size limit for creating the index. You can
-reduce this if running on a smaller node, or increase this for faster performance while
-indexing a larger corpus.
-
-```python
-# configure a 512MB heap size
-heap = 1024 * 1024 * 512
-table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
-```
-
-## Current limitations
-
-For that Tantivy-based FTS:
-
-1. Currently we do not yet support incremental writes.
-   If you add data after FTS index creation, it won't be reflected
-   in search results until you do a full reindex.
-
-2. We currently only support local filesystem paths for the FTS index.
-   This is a tantivy limitation. We've implemented an object store plugin
-   but there's no way in tantivy-py to specify to use it.

docs/src/fts_tantivy.md

@@ -0,0 +1,160 @@
+# Full-text search (Tantivy-based FTS)
+
+LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+
+The tantivy-based FTS is only available in Python synchronous APIs and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
+
+## Installation
+
+To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
+
+```sh
+# Say you want to use tantivy==0.20.1
+pip install tantivy==0.20.1
+```
+
+## Example
+
+Consider that we have a LanceDB table named `my_table`, whose string column `content` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+table = db.create_table(
+    "my_table",
+    data=[
+        {"id": 1, "vector": [3.1, 4.1], "title": "happy puppy", "content": "Frodo was a happy puppy", "meta": "foo"},
+        {"id": 2, "vector": [5.9, 26.5], "title": "playing kittens", "content": "There are several kittens playing around the puppy", "meta": "bar"},
+    ],
+)
+
+# passing `use_tantivy=False` to use lance FTS index
+# `use_tantivy=True` by default
+table.create_fts_index("content", use_tantivy=True)
+table.search("puppy").limit(10).select(["content"]).to_list()
+# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
+# ...
+```
+
+It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
+
+!!! note
+    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
+
+## Tokenization
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+
+```python
+table.create_fts_index("content", use_tantivy=True, tokenizer_name="en_stem", replace=True)
+```
+
+the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+
+## Index multiple columns
+
+If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
+
+```python
+table.create_fts_index(["title", "content"], use_tantivy=True, replace=True)
+```
+
+Note that the search API call does not change - you can search over all indexed columns at once.
+
+## Filtering
+
+Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
+applied on top of the full text search results (see [native FTS](fts.md) if you need pre-filtering). This can be invoked via the familiar
+`where` syntax:
+
+```python
+table.search("puppy").limit(10).where("meta='foo'").to_list()
+```
+
+## Sorting
+
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example,
+
+```python
+table.create_fts_index(["content"], use_tantivy=True, ordering_field_names=["id"], replace=True)
+
+(table.search("puppy", ordering_field_name="id")
+ .limit(20)
+ .to_list())
+```
+
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
+
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see
+    an error during indexing that looks like
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
+
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
+
+
+## Phrase queries vs. terms queries
+
+For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
+or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
+
+!!! tip "Note"
+    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
+
+    ```py
+    # This raises a syntax error
+    table.search("they could have been dogs OR cats")
+    ```
+
+    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+    the query is treated as a phrase query.
+
+    ```py
+    # This works!
+    table.search("they could have been dogs or cats")
+    ```
+
+It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+query you want to perform. To make this simpler, when you want to perform a phrase query, you can
+enforce it in one of two ways:
+
+1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
+a phrase query.
+1. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
+itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
+is treated as a phrase query.
+
+In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
+double quotes replaced by single quotes.
+
+
+## Configurations
+
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can
+reduce this if running on a smaller node, or increase this for faster performance while
+indexing a larger corpus.
+
+```python
+# configure a 512MB heap size
+heap = 1024 * 1024 * 512
+table.create_fts_index(["title", "content"], use_tantivy=True, writer_heap_size=heap, replace=True)
+```
+
+## Current limitations
+
+1. New data added after creating the FTS index will appear in search results, but with increased latency due to a flat search on the unindexed portion. Re-indexing with `create_fts_index` will reduce latency. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 
+
+2. We currently only support local filesystem paths for the FTS index.
+   This is a tantivy limitation. We've implemented an object store plugin
+   but there's no way in tantivy-py to specify to use it.

docs/src/guides/multi-vector.md

@@ -0,0 +1,85 @@
+# Late interaction & MultiVector embedding type
+Late interaction is a technique used in retrieval that calculates the relevance of a query to a document by comparing their multi-vector representations. The key difference between late interaction and other popular methods:
+
+![late interaction vs other methods](https://raw.githubusercontent.com/lancedb/assets/b035a0ceb2c237734e0d393054c146d289792339/docs/assets/integration/colbert-blog-interaction.svg)
+
+
+[ Illustration from https://jina.ai/news/what-is-colbert-and-late-interaction-and-why-they-matter-in-search/]
+
+<b>No interaction:</b> Refers to independently embedding the query and document, that are compared to calcualte similarity without any interaction between them. This is typically used in vector search operations.
+
+<b>Partial interaction</b> Refers to a specific approach where the similarity computation happens primarily between query vectors and document vectors, without extensive interaction between individual components of each. An example of this is dual-encoder models like BERT.
+
+<b>Early full interaction</b> Refers to techniques like cross-encoders that process query and docs in pairs with full interaction across various stages of encoding. This is a powerful, but relatively slower technique. Because it requires processing query and docs in pairs, doc embeddings can't be pre-computed for fast retrieval. This is why cross encoders are typically used as reranking models combined with vector search. Learn more about [LanceDB Reranking support](https://lancedb.github.io/lancedb/reranking/).
+
+<b>Late interaction</b> Late interaction is a technique that calculates the doc and query similarity independently and then the interaction or evaluation happens during the retrieval process. This is typically used in retrieval models like ColBERT. Unlike early interaction, It allows speeding up the retrieval process without compromising the depth of semantic analysis.
+
+## Internals of ColBERT 
+Let's take a look at the steps involved in performing late interaction based retrieval using ColBERT:
+
+• ColBERT employs BERT-based encoders for both queries `(fQ)` and documents `(fD)`
+• A single BERT model is shared between query and document encoders and special tokens distinguish input types: `[Q]` for queries and `[D]` for documents
+
+**Query Encoder (fQ):**
+• Query q is tokenized into WordPiece tokens: `q1, q2, ..., ql`. `[Q]` token is prepended right after BERT's `[CLS]` token
+• If query length < Nq, it's padded with [MASK] tokens up to Nq.
+• The padded sequence goes through BERT's transformer architecture
+• Final embeddings are L2-normalized.
+
+**Document Encoder (fD):**
+• Document d is tokenized into tokens `d1, d2, ..., dm`. `[D]` token is prepended after `[CLS]` token
+• Unlike queries, documents are NOT padded with `[MASK]` tokens
+• Document tokens are processed through BERT and the same linear layer
+
+**Late Interaction:**
+• Late interaction estimates relevance score `S(q,d)` using embedding `Eq` and `Ed`. Late interaction happens after independent encoding
+• For each query embedding, maximum similarity is computed against all document embeddings
+• The similarity measure can be cosine similarity or squared L2 distance
+
+**MaxSim Calculation:**
+```
+S(q,d) := Σ max(Eqi⋅EdjT)
+          i∈|Eq| j∈|Ed|
+```
+• This finds the best matching document embedding for each query embedding
+• Captures relevance based on strongest local matches between contextual embeddings
+
+## LanceDB MultiVector type
+LanceDB supports multivector type, this is useful when you have multiple vectors for a single item (e.g. with ColBert and ColPali).
+
+You can index on a column with multivector type and search on it, the query can be single vector or multiple vectors. For now, only cosine metric is supported for multivector search. The vector value type can be float16, float32 or float64. LanceDB integrateds [ConteXtualized Token Retriever(XTR)](https://arxiv.org/abs/2304.01982), which introduces a simple, yet novel, objective function that encourages the model to retrieve the most important document tokens first. 
+
+```python
+import lancedb
+import numpy as np
+import pyarrow as pa
+
+db = lancedb.connect("data/multivector_demo")
+schema = pa.schema(
+    [
+        pa.field("id", pa.int64()),
+        # float16, float32, and float64 are supported
+        pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
+    ]
+)
+data = [
+    {
+        "id": i,
+        "vector": np.random.random(size=(2, 256)).tolist(),
+    }
+    for i in range(1024)
+]
+tbl = db.create_table("my_table", data=data, schema=schema)
+
+# only cosine similarity is supported for multi-vectors
+tbl.create_index(metric="cosine")
+
+# query with single vector
+query = np.random.random(256).astype(np.float16)
+tbl.search(query).to_arrow()
+
+# query with multiple vectors
+query = np.random.random(size=(2, 256))
+tbl.search(query).to_arrow()
+```
+Find more about vector search in LanceDB [here](https://lancedb.github.io/lancedb/search/#multivector-type).

docs/src/guides/scalar_index.md

@@ -1,38 +1,51 @@
-# Building Scalar Index
+# Building a Scalar Index
 
-Similar to many SQL databases, LanceDB supports several types of Scalar indices to accelerate search
+Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
+
+Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
 over scalar columns.
 
-- `BTREE`: The most common type is BTREE. This index is inspired by the btree data structure
+- `BTREE`: The most common type is BTREE. The index stores a copy of the
-  although only the first few layers of the btree are cached in memory.
+  column in sorted order. This sorted copy allows a binary search to be used to
-  It will perform well on columns with a large number of unique values and few rows per value.
+  satisfy queries.
-- `BITMAP`: this index stores a bitmap for each unique value in the column.
+- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
-  This index is useful for columns with a finite number of unique values and many rows per value.
+  uses a series of bits to indicate whether a value is present in a row of a table
-  For example, columns that represent "categories", "labels", or "tags"
+- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
-- `LABEL_LIST`: a special index that is used to index list columns whose values have a finite set of possibilities.
+  support queries with `array_contains_all` and `array_contains_any`
+  using an underlying bitmap index.
   For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.
 
+!!! tips "How to choose the right scalar index type"
+
+    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
+    
+    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
+    
+    `LABEL_LIST`: This index should be used for columns containing list-type data.
+
 | Data Type                                                       | Filter                                    | Index Type   |
 | --------------------------------------------------------------- | ----------------------------------------- | ------------ |
 | Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
 | Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
 | List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |
 
+### Create a scalar index
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
-    books = [
-      {"book_id": 1, "publisher": "plenty of books", "tags": ["fantasy", "adventure"]},
-      {"book_id": 2, "publisher": "book town", "tags": ["non-fiction"]},
-      {"book_id": 3, "publisher": "oreilly", "tags": ["textbook"]}
-    ]
 
-    db = lancedb.connect("./db")
+        ```python
-    table = db.create_table("books", books)
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-    table.create_scalar_index("book_id")  # BTree by default
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-    table.create_scalar_index("publisher", index_type="BITMAP")
+        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index"
-    ```
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
+        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index_async"
+        ```
 
 === "Typescript"
 
@@ -46,16 +59,22 @@ over scalar columns.
         await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
         ```
 
-For example, the following scan will be faster if the column `my_col` has a scalar index:
+The following scan will be faster if the column `book_id` has a scalar index:
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
 
-    table = db.open_table("books")
+        ```python
-    my_df = table.search().where("book_id = 2").to_pandas()
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-    ```
+        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index_async"
+        ```
 
 === "Typescript"
 
@@ -76,22 +95,18 @@ Scalar indices can also speed up scans containing a vector search or full text s
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
 
-    data = [
+        ```python
-      {"book_id": 1, "vector": [1, 2]},
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-      {"book_id": 2, "vector": [3, 4]},
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index"
-      {"book_id": 3, "vector": [5, 6]}
+        ```
-    ]
+    === "Async API"
-    table = db.create_table("book_with_embeddings", data)
 
-    (
+        ```python
-        table.search([1, 2])
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        .where("book_id != 3", prefilter=True)
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index_async"
-        .to_pandas()
+        ```
-    )
-    ```
 
 === "Typescript"
 
@@ -106,3 +121,36 @@ Scalar indices can also speed up scans containing a vector search or full text s
           .limit(10)
           .toArray();
         ```
+### Update a scalar index
+Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
+=== "Python"
+
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index_async"
+        ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
+    await tbl.optimize();
+    ```
+
+=== "Rust"
+
+    ```rust
+    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    tbl.add(more_data).execute().await?;
+    tbl.optimize(OptimizeAction::All).execute().await?;
+    ```
+
+!!! note
+
+    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.

docs/src/guides/sql_querying.md

@@ -0,0 +1,66 @@
+You can use DuckDB and Apache Datafusion to query your LanceDB tables using SQL.
+This guide will show how to query Lance tables them using both.
+
+We will re-use the dataset [created previously](./pandas_and_pyarrow.md):
+
+```python
+import lancedb
+
+db = lancedb.connect("data/sample-lancedb")
+data = [
+    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
+]
+table = db.create_table("pd_table", data=data)
+```
+
+## Querying a LanceDB Table with DuckDb
+
+The `to_lance` method converts the LanceDB table to a `LanceDataset`, which is accessible to DuckDB through the Arrow compatibility layer.
+To query the resulting Lance dataset in DuckDB, all you need to do is reference the dataset by the same name in your SQL query.
+
+```python
+import duckdb
+
+arrow_table = table.to_lance()
+
+duckdb.query("SELECT * FROM arrow_table")
+```
+
+```
+┌─────────────┬─────────┬────────┐
+│   vector    │  item   │ price  │
+│   float[]   │ varchar │ double │
+├─────────────┼─────────┼────────┤
+│ [3.1, 4.1]  │ foo     │   10.0 │
+│ [5.9, 26.5] │ bar     │   20.0 │
+└─────────────┴─────────┴────────┘
+```
+
+## Querying a LanceDB Table with Apache Datafusion
+
+Have the required imports before doing any querying.
+
+=== "Python"
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-session-context"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-ffi-dataset"
+    ```
+
+Register the table created with the Datafusion session context.
+
+=== "Python"
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:lance_sql_basic"
+    ```
+
+```
+┌─────────────┬─────────┬────────┐
+│   vector    │  item   │ price  │
+│   float[]   │ varchar │ double │
+├─────────────┼─────────┼────────┤
+│ [3.1, 4.1]  │ foo     │   10.0 │
+│ [5.9, 26.5] │ bar     │   20.0 │
+└─────────────┴─────────┴────────┘
+```

docs/src/guides/storage.md

@@ -12,25 +12,52 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 === "Python"
 
     AWS S3:
+    === "Sync API"
 
-    ```python
+        ```python
-    import lancedb
+        import lancedb
-    db = lancedb.connect("s3://bucket/path")
+        db = lancedb.connect("s3://bucket/path")
-    ```
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("s3://bucket/path")
+        ```
 
     Google Cloud Storage:
 
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = lancedb.connect("gs://bucket/path")
+        ```python
-    ```
+        import lancedb
+        db = lancedb.connect("gs://bucket/path")
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("gs://bucket/path")
+        ```
 
     Azure Blob Storage:
 
-    ```python
+    <!-- skip-test -->
-    import lancedb
+    === "Sync API"
-    db = lancedb.connect("az://bucket/path")
+
-    ```
+        ```python
+        import lancedb
+        db = lancedb.connect("az://bucket/path")
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("az://bucket/path")
+        ```
+    Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
+
 
 === "TypeScript"
 
@@ -87,22 +114,28 @@ In most cases, when running in the respective cloud and permissions are set up c
 export TIMEOUT=60s
 ```
 
-!!! note "`storage_options` availability"
-
-    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
-    It is not yet supported in the Python synchronous API.
-
 If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "s3://bucket/path",
+        import lancedb
-        storage_options={"timeout": "60s"}
+        db = lancedb.connect(
-    )
+            "s3://bucket/path",
-    ```
+            storage_options={"timeout": "60s"}
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://bucket/path",
+            storage_options={"timeout": "60s"}
+        )
+        ```
 
 === "TypeScript"
 
@@ -130,15 +163,29 @@ Getting even more specific, you can set the `timeout` for only a particular tabl
 === "Python"
 
     <!-- skip-test -->
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async("s3://bucket/path")
+        ```python
-    table = await db.create_table(
+        import lancedb
-        "table",
+        db = lancedb.connect("s3://bucket/path")
-        [{"a": 1, "b": 2}],
+        table = db.create_table(
-        storage_options={"timeout": "60s"}
+            "table",
-    )
+            [{"a": 1, "b": 2}],
-    ```
+            storage_options={"timeout": "60s"}
+        )
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("s3://bucket/path")
+        async_table = await async_db.create_table(
+            "table",
+            [{"a": 1, "b": 2}],
+            storage_options={"timeout": "60s"}
+        )
+        ```
 
 === "TypeScript"
 
@@ -196,17 +243,32 @@ These can be set as environment variables or passed in the `storage_options` par
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "s3://bucket/path",
+        import lancedb
-        storage_options={
+        db = lancedb.connect(
-            "aws_access_key_id": "my-access-key",
+            "s3://bucket/path",
-            "aws_secret_access_key": "my-secret-key",
+            storage_options={
-            "aws_session_token": "my-session-token",
+                "aws_access_key_id": "my-access-key",
-        }
+                "aws_secret_access_key": "my-secret-key",
-    )
+                "aws_session_token": "my-session-token",
-    ```
+            }
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://bucket/path",
+            storage_options={
+                "aws_access_key_id": "my-access-key",
+                "aws_secret_access_key": "my-secret-key",
+                "aws_session_token": "my-session-token",
+            }
+        )
+        ```
 
 === "TypeScript"
 
@@ -280,7 +342,7 @@ For **read and write access**, LanceDB will need a policy such as:
             "Action": [
               "s3:PutObject",
               "s3:GetObject",
-              "s3:DeleteObject",
+              "s3:DeleteObject"
             ],
             "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
         },
@@ -312,7 +374,7 @@ For **read-only access**, LanceDB will need a policy such as:
         {
             "Effect": "Allow",
             "Action": [
-              "s3:GetObject",
+              "s3:GetObject"
             ],
             "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
         },
@@ -350,12 +412,22 @@ name of the table to use.
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+        import lancedb
-    )
+        db = lancedb.connect(
-    ```
+            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+        )    
+        ```
 
 === "JavaScript"
 
@@ -443,16 +515,30 @@ LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you m
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "s3://bucket/path",
+        import lancedb
-        storage_options={
+        db = lancedb.connect(
-            "region": "us-east-1",
+            "s3://bucket/path",
-            "endpoint": "http://minio:9000",
+            storage_options={
-        }
+                "region": "us-east-1",
-    )
+                "endpoint": "http://minio:9000",
-    ```
+            }
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://bucket/path",
+            storage_options={
+                "region": "us-east-1",
+                "endpoint": "http://minio:9000",
+            }
+        )    
+        ```
 
 === "TypeScript"
 
@@ -498,22 +584,36 @@ This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` envir
 
 #### S3 Express
 
-LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
+LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional infrastructure configuration for the compute service, such as EC2 or Lambda. Please refer to [Networking requirements for S3 Express One Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html).
 
 To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
 
 === "Python"
 
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "s3://my-bucket--use1-az4--x-s3/path",
+        import lancedb
-        storage_options={
+        db = lancedb.connect(
-            "region": "us-east-1",
+            "s3://my-bucket--use1-az4--x-s3/path",
-            "s3_express": "true",
+            storage_options={
-        }
+                "region": "us-east-1",
-    )
+                "s3_express": "true",
-    ```
+            }
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            storage_options={
+                "region": "us-east-1",
+                "s3_express": "true",
+            }
+        )    
+        ```
 
 === "TypeScript"
 
@@ -554,15 +654,29 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
 === "Python"
 
     <!-- skip-test -->
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "gs://my-bucket/my-database",
+        import lancedb
-        storage_options={
+        db = lancedb.connect(
-            "service_account": "path/to/service-account.json",
+            "gs://my-bucket/my-database",
-        }
+            storage_options={
-    )
+                "service_account": "path/to/service-account.json",
-    ```
+            }
+        )
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "gs://my-bucket/my-database",
+            storage_options={
+                "service_account": "path/to/service-account.json",
+            }
+        )    
+        ```
 
 === "TypeScript"
 
@@ -614,16 +728,31 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
 === "Python"
 
     <!-- skip-test -->
-    ```python
+    === "Sync API"
-    import lancedb
+
-    db = await lancedb.connect_async(
+        ```python
-        "az://my-container/my-database",
+        import lancedb
-        storage_options={
+        db = lancedb.connect(
-            account_name: "some-account",
+            "az://my-container/my-database",
-            account_key: "some-key",
+            storage_options={
-        }
+                account_name: "some-account",
-    )
+                account_key: "some-key",
-    ```
+            }
+        )
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "az://my-container/my-database",
+            storage_options={
+                account_name: "some-account",
+                account_key: "some-key",
+            }
+        )   
+        ```
 
 === "TypeScript"
 

docs/src/guides/tables/merge_insert.md

@@ -0,0 +1,135 @@
+The merge insert command is a flexible API that can be used to perform:
+
+1. Upsert
+2. Insert-if-not-exists
+3. Replace range
+
+It works by joining the input data with the target table on a key you provide.
+Often this key is a unique row id key. You can then specify what to do when
+there is a match and when there is not a match. For example, for upsert you want
+to update if the row has a match and insert if the row doesn't have a match.
+Whereas for insert-if-not-exists you only want to insert if the row doesn't have
+a match.
+
+You can also read more in the API reference:
+
+* Python
+    * Sync: [lancedb.table.Table.merge_insert][]
+    * Async: [lancedb.table.AsyncTable.merge_insert][]
+* Typescript: [lancedb.Table.mergeInsert](../../js/classes/Table.md/#mergeinsert)
+
+!!! tip "Use scalar indices to speed up merge insert"
+
+    The merge insert command needs to perform a join between the input data and the
+    target table on the `on` key you provide. This requires scanning that entire
+    column, which can be expensive for large tables. To speed up this operation,
+    you can create a scalar index on the `on` column, which will allow LanceDB to
+    find matches without having to scan the whole tables.
+
+    Read more about scalar indices in [Building a Scalar Index](../scalar_index.md)
+    guide.
+
+!!! info "Embedding Functions"
+
+    Like the create table and add APIs, the merge insert API will automatically
+    compute embeddings if the table has a embedding definition in its schema.
+    If the input data doesn't contain the source column, or the vector column
+    is already filled, then the embeddings won't be computed. See the
+    [Embedding Functions](../../embeddings/embedding_functions.md) guide for more
+    information.
+
+## Upsert
+
+Upsert updates rows if they exist and inserts them if they don't. To do this
+with merge insert, enable both `when_matched_update_all()` and
+`when_not_matched_insert_all()`.
+
+=== "Python"
+
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic"
+        ```
+
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic_async"
+        ```
+
+=== "Typescript"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/merge_insert.test.ts:upsert_basic"
+        ```
+
+!!! note "Providing subsets of columns"
+
+    If a column is nullable, it can be omitted from input data and it will be
+    considered `null`. Columns can also be provided in any order.
+
+## Insert-if-not-exists
+
+To avoid inserting duplicate rows, you can use the insert-if-not-exists command.
+This will only insert rows that do not have a match in the target table. To do
+this with merge insert, enable just `when_not_matched_insert_all()`.
+
+
+=== "Python"
+
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_merge_insert.py:insert_if_not_exists"
+        ```
+
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_merge_insert.py:insert_if_not_exists_async"
+        ```
+
+=== "Typescript"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/merge_insert.test.ts:insert_if_not_exists"
+        ```
+
+
+## Replace range
+
+You can also replace a range of rows in the target table with the input data.
+For example, if you have a table of document chunks, where each chunk has
+both a `doc_id` and a `chunk_id`, you can replace all chunks for a given
+`doc_id` with updated chunks. This can be tricky otherwise because if you
+try to use upsert when the new data has fewer chunks you will end up with
+extra chunks. To avoid this, add another clause to delete any chunks for
+the document that are not in the new data, with
+`when_not_matched_by_source_delete`.
+
+=== "Python"
+
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_merge_insert.py:replace_range"
+        ```
+
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_merge_insert.py:replace_range_async"
+        ```
+
+=== "Typescript"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/merge_insert.test.ts:replace_range"
+        ```

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,8 +1,8 @@
 ## Improving retriever performance
 
-Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
-VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retrievers are a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
 
 There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
 
@@ -19,7 +19,7 @@ Using different embedding models is something that's very specific to the use ca
 
 
 ## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv).
 
 ### Using different query types
 Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
@@ -45,14 +45,14 @@ table.add(df[["context"]].to_dict(orient="records"))
 queries = df["query"].tolist()
 ```
 
-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset:
 
 * <b> Vector Search: </b>
 
     ```python
     table.search(quries[0], query_type="vector").limit(5).to_pandas()
     ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement:
 
     ```python
     table.search(quries[0]).limit(5).to_pandas()
@@ -77,7 +77,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
 * <b> Hybrid Search: </b>
 
-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset:
     ```python
     table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
     ```
@@ -87,7 +87,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
     !!! note "Note"
         By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/).
 
     
 

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -1,6 +1,6 @@
 Continuing from the previous section, we can now rerank the results using more complex rerankers.
 
-Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 ## Reranking search results
 You can rerank any search results using a reranker. The syntax for reranking is as follows:
@@ -62,9 +62,6 @@ Let us take a look at the same datasets from the previous sections, using the sa
 | Reranked fts  | 0.672    |
 | Hybrid | 0.759 |
 
-### SQuAD Dataset
-
-
 ### Uber10K sec filing Dataset
 
 | Query Type | Hit-rate@5 |

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -1,5 +1,5 @@
 ## Finetuning the Embedding Model
-Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
 
@@ -16,7 +16,7 @@ validation_df.to_csv("data_val.csv", index=False)
 You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
 
 
-Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
+We parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node:
 ```python
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.readers.file import PagedCSVReader
@@ -43,7 +43,7 @@ val_dataset = generate_qa_embedding_pairs(
 )
 ```
 
-Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model:
 
 ```python
 from llama_index.finetuning import SentenceTransformersFinetuneEngine
@@ -57,7 +57,7 @@ finetune_engine = SentenceTransformersFinetuneEngine(
 finetune_engine.finetune()
 embed_model = finetune_engine.get_finetuned_model()
 ```
-This saves the fine tuned embedding model in `tuned_model` folder. This al
+This saves the fine tuned embedding model in `tuned_model` folder.
 
 # Evaluation results
 In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 

docs/src/hybrid_search/eval.md

@@ -3,22 +3,22 @@
 Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
 
 ## The challenge of (re)ranking search results
-Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step: reranking.
 There are two approaches for reranking search results from multiple sources.
 
-* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example: Weighted linear combination of semantic search & keyword-based search results.
 
-* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models
 
-Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset or application specific so it's hard to generalize.
 
 ### Example evaluation of hybrid search with Reranking
 
-Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+Here's some evaluation numbers from an experiment comparing these rerankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
 
 <b> With OpenAI ada2 embedding </b>
 
-Vector Search baseline - `0.64`
+Vector Search baseline: `0.64`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |
@@ -33,7 +33,7 @@ Vector Search baseline - `0.64`
 
 <b> With OpenAI embedding-v3-small </b>
 
-Vector Search baseline - `0.59`
+Vector Search baseline: `0.59`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |

docs/src/hybrid_search/hybrid_search.md

@@ -5,57 +5,46 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
 
-```python
+=== "Sync API"
-import os
 
-import lancedb
+    ```python
-import openai
+    --8<-- "python/python/tests/docs/test_search.py:import-os"
-from lancedb.embeddings import get_registry
+    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-from lancedb.pydantic import LanceModel, Vector
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
+    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search"
+    ```
+=== "Async API"
 
-db = lancedb.connect("~/.lancedb")
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:import-os"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
+    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search_async"
+    ```
 
-# Ingest embedding function in LanceDB table
-# Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
-# OR set the key here as a variable
-    openai.api_key = "sk-..."
-embeddings = get_registry().get("openai").create()
-
-class Documents(LanceModel):
-    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
-    text: str = embeddings.SourceField()
-
-table = db.create_table("documents", schema=Documents)
-
-data = [
-    { "text": "rebel spaceships striking from a hidden base"},
-    { "text": "have won their first victory against the evil Galactic Empire"},
-    { "text": "during the battle rebel spies managed to steal secret plans"},
-    { "text": "to the Empire's ultimate weapon the Death Star"}
-]
-
-# ingest docs with auto-vectorization
-table.add(data)
-
-# Create a fts index before the hybrid search
-table.create_fts_index("text")
-# hybrid search with default re-ranker
-results = table.search("flower moon", query_type="hybrid").to_pandas()
-```
 !!! Note
     You can also pass the vector and text query manually. This is useful if you're not using the embedding API or if you're using a separate embedder service.
 ### Explicitly passing the vector and text query
-```python
+=== "Sync API"
-vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
-text_query = "flower moon"
-results = table.search(query_type="hybrid")
-                .vector(vector_query)
-                .text(text_query)
-                .limit(5)
-                .to_pandas()
 
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text_async"
+    ```
 
 By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
 
@@ -68,7 +57,7 @@ By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion scor
 
 
 ## Available Rerankers
-LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. 
+LanceDB provides a number of rerankers out of the box. You can use any of these rerankers by passing them to the `rerank()` method. 
 Go to [Rerankers](../reranking/index.md) to learn more about using the available rerankers and implementing custom rerankers.
 
 

docs/src/index.md

@@ -4,6 +4,9 @@ LanceDB is an open-source vector database for AI that's designed to store, manag
 
 Both the database and the underlying data format are designed from the ground up to be **easy-to-use**, **scalable** and **cost-effective**.
 
+!!! tip "Hosted LanceDB"
+    If you want S3 cost-efficiency and local performance via a simple serverless API, checkout **LanceDB Cloud**. For private deployments, high performance at extreme scale, or if you have strict security requirements, talk to us about **LanceDB Enterprise**. [Learn more](https://docs.lancedb.com/)
+
 ![](assets/lancedb_and_lance.png)
 
 ## Truly multi-modal
@@ -20,7 +23,7 @@ LanceDB **OSS** is an **open-source**, batteries-included embedded vector databa
 
 LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverless in the cloud, making the storage clearly separated from compute. It's designed to be cost-effective and highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
 
-[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
+[Try out LanceDB Cloud (Public Beta) Now](https://cloud.lancedb.com){ .md-button .md-button--primary }
 
 ## Why use LanceDB?
 
@@ -49,7 +52,8 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Working with tables](guides/tables.md): Learn how to work with tables and their associated functions
 * [Indexing](ann_indexes.md): Understand how to create indexes
 * [Vector search](search.md): Learn how to perform vector similarity search
-* [Full-text search](fts.md): Learn how to perform full-text search
+* [Full-text search (native)](fts.md): Learn how to perform full-text search
+* [Full-text search (tantivy-based)](fts_tantivy.md): Learn how to perform full-text search using Tantivy
 * [Managing embeddings](embeddings/index.md): Managing embeddings and the embedding functions API in LanceDB
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references

docs/src/integrations/genkit.md

@@ -0,0 +1,183 @@
+### genkitx-lancedb
+This is a lancedb plugin for genkit framework. It allows you to use LanceDB for ingesting and rereiving data using genkit framework.
+
+![integration-banner-genkit](https://github.com/user-attachments/assets/a6cc28af-98e9-4425-b87c-7ab139bd7893)
+
+### Installation
+```bash
+pnpm install genkitx-lancedb
+```
+
+### Usage
+
+Adding LanceDB plugin to your genkit instance.
+
+```ts
+import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from 'genkitx-lancedb';
+import { textEmbedding004, vertexAI } from '@genkit-ai/vertexai';
+import { gemini } from '@genkit-ai/vertexai';
+import { z, genkit } from 'genkit';
+import { Document } from 'genkit/retriever';
+import { chunk } from 'llm-chunk';
+import { readFile } from 'fs/promises';
+import path from 'path';
+import pdf from 'pdf-parse/lib/pdf-parse';
+
+const ai = genkit({
+  plugins: [
+    // vertexAI provides the textEmbedding004 embedder
+    vertexAI(),
+
+    // the local vector store requires an embedder to translate from text to vector
+    lancedb([
+      {
+        dbUri: '.db', // optional lancedb uri, default to .db
+        tableName: 'table', // optional table name, default to table
+        embedder: textEmbedding004,
+      },
+    ]),
+  ],
+});
+```
+
+You can run this app with the following command:
+```bash
+genkit start -- tsx --watch src/index.ts
+```
+
+This'll add LanceDB as a retriever and indexer to the genkit instance. You can see it in the GUI view
+<img width="1710" alt="Screenshot 2025-05-11 at 7 21 05 PM" src="https://github.com/user-attachments/assets/e752f7f4-785b-4797-a11e-72ab06a531b7" />
+
+**Testing retrieval on a sample table**
+Let's see the raw retrieval results
+
+<img width="1710" alt="Screenshot 2025-05-11 at 7 21 05 PM" src="https://github.com/user-attachments/assets/b8d356ed-8421-4790-8fc0-d6af563b9657" />
+On running this query, you'll 5 results fetched from the lancedb table, where each result looks something like this:
+<img width="1417" alt="Screenshot 2025-05-11 at 7 21 18 PM" src="https://github.com/user-attachments/assets/77429525-36e2-4da6-a694-e58c1cf9eb83" />
+
+
+
+## Creating a custom RAG flow
+
+Now that we've seen how you can use LanceDB for in a genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retreiver with its outputs postprocessed an fed into an LLM for final response
+
+### Creating custom indexer flows
+You can also create custom indexer flows, utilizing more options and features provided by LanceDB.
+
+```ts
+export const menuPdfIndexer = lancedbIndexerRef({
+   // Using all defaults, for dbUri, tableName, and embedder, etc
+});
+
+const chunkingConfig = {
+  minLength: 1000,
+  maxLength: 2000,
+  splitter: 'sentence',
+  overlap: 100,
+  delimiters: '',
+} as any;
+
+
+async function extractTextFromPdf(filePath: string) {
+  const pdfFile = path.resolve(filePath);
+  const dataBuffer = await readFile(pdfFile);
+  const data = await pdf(dataBuffer);
+  return data.text;
+}
+
+export const indexMenu = ai.defineFlow(
+  {
+    name: 'indexMenu',
+    inputSchema: z.string().describe('PDF file path'),
+    outputSchema: z.void(),
+  },
+  async (filePath: string) => {
+    filePath = path.resolve(filePath);
+
+    // Read the pdf.
+    const pdfTxt = await ai.run('extract-text', () =>
+      extractTextFromPdf(filePath)
+    );
+
+    // Divide the pdf text into segments.
+    const chunks = await ai.run('chunk-it', async () =>
+      chunk(pdfTxt, chunkingConfig)
+    );
+
+    // Convert chunks of text into documents to store in the index.
+    const documents = chunks.map((text) => {
+      return Document.fromText(text, { filePath });
+    });
+
+    // Add documents to the index.
+    await ai.index({
+      indexer: menuPdfIndexer,
+      documents,
+      options: {
+        writeMode: WriteMode.Overwrite,
+      } as any
+    });
+  }
+);
+```
+
+<img width="1316" alt="Screenshot 2025-05-11 at 8 35 56 PM" src="https://github.com/user-attachments/assets/e2a20ce4-d1d0-4fa2-9a84-f2cc26e3a29f" />
+
+In your console, you can see the logs
+
+<img width="511" alt="Screenshot 2025-05-11 at 7 19 14 PM" src="https://github.com/user-attachments/assets/243f26c5-ed38-40b6-b661-002f40f0423a" />
+
+### Creating custom retriever flows
+You can also create custom retriever flows, utilizing more options and features provided by LanceDB.
+```ts
+export const menuRetriever = lancedbRetrieverRef({
+  tableName: "table", // Use the same table name as the indexer.
+  displayName: "Menu", // Use a custom display name.
+
+export const menuQAFlow = ai.defineFlow(
+  { name: "Menu", inputSchema: z.string(), outputSchema: z.string() },
+  async (input: string) => {
+    // retrieve relevant documents
+    const docs = await ai.retrieve({
+      retriever: menuRetriever,
+      query: input,
+      options: { 
+        k: 3,
+      },
+    });
+
+    const extractedContent = docs.map(doc => {
+      if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) {
+        if (doc.content[0].media && doc.content[0].media.url) {
+          return doc.content[0].media.url;
+        }
+      }
+      return "No content found";
+    });
+
+    console.log("Extracted content:", extractedContent);
+
+    const { text } = await ai.generate({
+      model: gemini('gemini-2.0-flash'),
+      prompt: `
+You are acting as a helpful AI assistant that can answer 
+questions about the food available on the menu at Genkit Grub Pub.
+
+Use only the context provided to answer the question.
+If you don't know, do not make up an answer.
+Do not add or change items on the menu.
+
+Context:
+${extractedContent.join('\n\n')}
+
+Question: ${input}`,
+      docs,
+    });
+    
+    return text;
+  }
+);
+```
+Now using our retrieval flow, we can ask question about the ingsted PDF
+<img width="1306" alt="Screenshot 2025-05-11 at 7 18 45 PM" src="https://github.com/user-attachments/assets/86c66b13-7c12-4d5f-9d81-ae36bfb1c346" />
+

docs/src/integrations/langchain.md

@@ -1,5 +1,10 @@
-# Langchain
+**LangChain** is a framework designed for building applications with large language models (LLMs) by chaining together various components. It supports a range of functionalities including memory, agents, and chat models, enabling developers to create context-aware applications.
-![Illustration](../assets/langchain.png)
+
+![Illustration](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/langchain_rag.png)
+
+LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities.
+
+Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs.
 
 ## Quick Start
 You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. Checkout Complete example here - [LangChain demo](../notebooks/langchain_example.ipynb)
@@ -26,20 +31,28 @@ print(docs[0].page_content)
 
 ## Documentation
 In the above example `LanceDB` vector store class object is created using `from_documents()` method  which is a `classmethod` and returns the initialized class object. 
+
 You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method.  
 
 The exhaustive list of parameters for `LanceDB` vector store are : 
-- `connection`: (Optional) `lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.  
+
-- `embedding`: Langchain embedding model.  
+|Name|type|Purpose|default|
-- `vector_key`: (Optional) Column name to use for vector's in the table. Defaults to `'vector'`.   
+|:----|:----|:----|:----|
-- `id_key`: (Optional) Column name to use for id's in the table. Defaults to `'id'`.  
+|`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.|`None`|
-- `text_key`: (Optional) Column name to use for text in the table. Defaults to `'text'`.  
+|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.|
-- `table_name`: (Optional) Name of your table in the database. Defaults to `'vectorstore'`.  
+|`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`|
-- `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
+|`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`|
-- `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
+|`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`|
-- `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
+|`text_key` |(Optional) `str` |Column name to use for text in the table.|`'text'`|
-- `reranker`: (Optional) The reranker to use for LanceDB.
+|`table_name` |(Optional) `str`| Name of your table in the database.|`'vectorstore'`|
-- `relevance_score_fn`: (Optional[Callable[[float], float]]) Langchain relevance score function to be used. Defaults to `None`. 
+|`api_key` |(Optional `str`) |API key to use for LanceDB cloud database.|`None`|
+|`region` |(Optional) `str`| Region to use for LanceDB cloud database.|Only for LanceDB Cloud : `None`.|
+|`mode` |(Optional) `str` |Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`|
+|`table`| (Optional) `Any`|You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`|
+|`distance`|(Optional) `str`|The choice of distance metric used to calculate the similarity between vectors.|`'l2'`|
+|`reranker` |(Optional) `Any`|The reranker to use for LanceDB.|`None`|
+|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`|
+|`limit`|`int`|Set the maximum number of results to return.|`DEFAULT_K` (it is 4)|
 
 ```python
 db_url = "db://lang_test" # url of db you created
@@ -51,19 +64,24 @@ vector_store = LanceDB(
     api_key=api_key, #(dont include for local API)
     region=region, #(dont include for local API)
     embedding=embeddings,
-    table_name='langchain_test' #Optional
+    table_name='langchain_test' # Optional
     )
 ```
 
 ### Methods 
 
 ##### add_texts()
-- `texts`: `Iterable` of strings to add to the vectorstore.
-- `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
-- `ids`: Optional `list` of ids to associate with the texts. 
-- `kwargs`: `Any`
 
-This method adds texts and stores respective embeddings automatically.
+This method turn texts into embedding and add it to the database.
+
+|Name|Purpose|defaults|
+|:---|:---|:---|
+|`texts`|`Iterable` of strings to add to the vectorstore.|Provided by user|
+|`metadatas`|Optional `list[dict()]` of metadatas associated with the texts.|`None`|
+|`ids`|Optional `list` of ids to associate with the texts.|`None`|
+|`kwargs`| Other keyworded arguments provided by the user. |-|
+
+It returns list of ids of the added texts.
 
 ```python
 vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
@@ -78,14 +96,25 @@ pd_df.to_csv("docsearch.csv", index=False)
 # you can also create a new vector store object using an older connection object:
 vector_store = LanceDB(connection=tbl, embedding=embeddings)
 ```
-##### create_index() 
-- `col_name`: `Optional[str] = None`
-- `vector_col`: `Optional[str] = None`
-- `num_partitions`: `Optional[int] = 256`
-- `num_sub_vectors`: `Optional[int] = 96`
-- `index_cache_size`: `Optional[int] = None`
 
-This method creates an index for the vector store. For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+------
+
+
+##### create_index() 
+
+This method creates a scalar(for non-vector cols) or a vector index on a table.
+
+|Name|type|Purpose|defaults|
+|:---|:---|:---|:---|
+|`vector_col`|`Optional[str]`| Provide if you want to create index on a vector column. |`None`|
+|`col_name`|`Optional[str]`| Provide if you want to create index on a non-vector column. |`None`|
+|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'l2', 'dot', 'cosine'. |`l2`|
+|`num_partitions`|`Optional[int]`|Number of partitions to use for the index.|`256`|
+|`num_sub_vectors`|`Optional[int]` |Number of sub-vectors to use for the index.|`96`|
+|`index_cache_size`|`Optional[int]` |Size of the index cache.|`None`|
+|`name`|`Optional[str]` |Name of the table to create index on.|`None`|
+
+For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
 
 ```python
 # for creating vector index
@@ -96,42 +125,63 @@ vector_store.create_index(col_name='text')
 
 ```
 
-##### similarity_search()
+------
-- `query`: `str`
-- `k`: `Optional[int] = None`
-- `filter`: `Optional[Dict[str, str]] = None`
-- `fts`: `Optional[bool] = False`
-- `name`: `Optional[str] = None`
-- `kwargs`: `Any`
 
-Return documents most similar to the query without relevance scores
+##### similarity_search()
+
+This method performs similarity search based on **text query**.
+
+| Name    | Type                 | Purpose | Default |
+|---------|----------------------|---------|---------|
+| `query` | `str`                |  A `str` representing the text query that you want to search for in the vector store.         | N/A     |
+| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
+| `fts`   | `Optional[bool]`     |   It indicates whether to perform a full-text search (FTS).      | `False`   |
+| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
+| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
+
+Return documents most similar to the query **without relevance scores**.
 
 ```python
 docs = docsearch.similarity_search(query)
 print(docs[0].page_content)
 ```
 
-##### similarity_search_by_vector()
+------
-- `embedding`: `List[float]`
-- `k`: `Optional[int] = None`
-- `filter`: `Optional[Dict[str, str]] = None`
-- `name`: `Optional[str] = None`
-- `kwargs`: `Any`
 
-Returns documents most similar to the query vector.
+##### similarity_search_by_vector()
+
+The method returns documents that are most similar to the specified **embedding (query) vector**. 
+
+| Name        | Type                      | Purpose | Default |
+|-------------|---------------------------|---------|---------|
+| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.         | N/A     |
+| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
+| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
+| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
+
+**It does not provide relevance scores.**
 
 ```python
 docs = docsearch.similarity_search_by_vector(query)
 print(docs[0].page_content)
 ```
 
-##### similarity_search_with_score()
+------
-- `query`: `str`
-- `k`: `Optional[int] = None`
-- `filter`: `Optional[Dict[str, str]] = None`
-- `kwargs`: `Any`
 
-Returns documents most similar to the query string with relevance scores, gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
+##### similarity_search_with_score()
+
+Returns documents most similar to the **query string** along with their relevance scores.
+
+| Name     | Type                      | Purpose | Default |
+|----------|---------------------------|---------|---------|
+| `query`  | `str`                     |A `str` representing the text query you want to search for in the vector store. This query will be converted into an embedding using the specified embedding function.         | N/A     |
+| `k`      | `Optional[int]`           | It specifies the number of documents to return.  | `None`    |
+| `filter` | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria. This allows you to narrow down the search results based on certain metadata attributes associated with the documents.       | `None`    |
+| `kwargs` | `Any`                     |  Other keyworded arguments provided by the user.       | N/A     |
+
+It gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
 
 ```python
 docs = docsearch.similarity_search_with_relevance_scores(query)
@@ -139,15 +189,21 @@ print("relevance score - ", docs[0][1])
 print("text- ", docs[0][0].page_content[:1000])
 ```
 
-##### similarity_search_by_vector_with_relevance_scores()
+------
-- `embedding`: `List[float]`
-- `k`: `Optional[int] = None`
-- `filter`: `Optional[Dict[str, str]] = None`
-- `name`: `Optional[str] = None`
-- `kwargs`: `Any`
 
-Return documents most similar to the query vector with relevance scores.
+##### similarity_search_by_vector_with_relevance_scores()
-Relevance score 
+
+Similarity search using **query vector**.
+
+| Name        | Type                      | Purpose | Default |
+|-------------|---------------------------|---------|---------|
+| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.        | N/A     |
+| `k`         | `Optional[int]`           |   It specifies the number of documents to return.       | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria.       | `None`    |
+| `name`      | `Optional[str]`           |   It is used for specifying the name of the table to query.       | `None`    |
+| `kwargs`    | `Any`                     |   Other keyworded arguments provided by the user.      | N/A     |
+
+The method returns documents most similar to the specified embedding (query) vector, along with their relevance scores.
 
 ```python
 docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)
@@ -155,20 +211,22 @@ print("relevance score - ", docs[0][1])
 print("text- ", docs[0][0].page_content[:1000])
 ```
 
-##### max_marginal_relevance_search()
+------
-- `query`: `str`
-- `k`: `Optional[int] = None`
-- `fetch_k` : Number of Documents to fetch to pass to MMR algorithm, `Optional[int] = None`
-- `lambda_mult`: Number between 0 and 1 that determines the degree
-                        of diversity among the results with 0 corresponding
-                        to maximum diversity and 1 to minimum diversity.
-                        Defaults to 0.5. `float = 0.5`
-- `filter`: `Optional[Dict[str, str]] = None`
-- `kwargs`: `Any`
 
-Returns docs selected using the maximal marginal relevance(MMR).
+##### max_marginal_relevance_search()
+
+This method returns docs selected using the maximal marginal relevance(MMR).
 Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.
 
+| Name | Type | Purpose | Default |
+|---------------|-----------------|-----------|---------|
+| `query`       | `str` | Text to look up documents similar to. | N/A     |
+| `k`   | `Optional[int]` | Number of Documents to return.| `4`       |
+| `fetch_k`| `Optional[int]`| Number of Documents to fetch to pass to MMR algorithm.| `None`    |
+| `lambda_mult` | `float`                   | Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. | `0.5`     |
+| `filter`| `Optional[Dict[str, str]]`| Filter by metadata. | `None`    |
+|`kwargs`| Other keyworded arguments provided by the user. |-|
+
 Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 
 
 ```python
@@ -186,12 +244,19 @@ result_texts = [doc.page_content for doc in result]
 print(result_texts)
 ```
 
-##### add_images()
+------
-- `uris` : File path to the image. `List[str]`.
-- `metadatas` : Optional list of metadatas. `(Optional[List[dict]], optional)`
-- `ids` : Optional list of IDs. `(Optional[List[str]], optional)`
 
-Adds images by automatically creating their embeddings and adds them to the vectorstore.
+##### add_images()
+
+This method ddds images by automatically creating their embeddings and adds them to the vectorstore.
+
+| Name       | Type                          | Purpose                        | Default |
+|------------|-------------------------------|--------------------------------|---------|
+| `uris`     | `List[str]`                   | File path to the image         | N/A     |
+| `metadatas`| `Optional[List[dict]]`        | Optional list of metadatas     | `None`    |
+| `ids`      | `Optional[List[str]]`         | Optional list of IDs           | `None`    |
+
+It returns list of IDs of the added images.
 
 ```python
 vec_store.add_images(uris=image_uris) 

docs/src/integrations/llamaIndex.md

@@ -125,7 +125,7 @@ The exhaustive list of parameters for `LanceDBVectorStore` vector store are :
         ```               
 - **_table_exists(self, tbl_name: `Optional[str]` = `None`) -> `bool`** : Returns `True` if `tbl_name` exists in database.
 - __create_index(  
-  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "L2",  
+  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "l2",  
 ) -> `None`__ : Creates a scalar(for non-vector cols) or a vector index on a table.
         Make sure your vector column has enough data before creating an index on it.
 

docs/src/integrations/phidata.md

@@ -0,0 +1,383 @@
+**phidata** is a framework for building **AI Assistants** with long-term memory, contextual knowledge, and the ability to take actions using function calling. It helps turn general-purpose LLMs into specialized assistants tailored to your use case by extending its capabilities using **memory**, **knowledge**, and **tools**. 
+
+- **Memory**: Stores chat history in a **database** and enables LLMs to have long-term conversations.
+- **Knowledge**: Stores information in a **vector database** and provides LLMs with business context. (Here we will use LanceDB)
+- **Tools**: Enable LLMs to take actions like pulling data from an **API**, **sending emails** or **querying a database**, etc.
+
+![example](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/phidata_assistant.png)
+
+Memory & knowledge make LLMs smarter while tools make them autonomous.
+
+LanceDB is a vector database and its integration into phidata makes it easy for us to provide a **knowledge base** to LLMs. It enables us to store information as [embeddings](../embeddings/understanding_embeddings.md) and search for the **results** similar to ours using **query**. 
+
+??? Question "What is Knowledge Base?"
+    Knowledge Base is a database of information that the Assistant can search to improve its responses. This information is stored in a vector database and provides LLMs with business context, which makes them respond in a context-aware manner.
+
+    While any type of storage can act as a knowledge base, vector databases offer the best solution for retrieving relevant results from dense information quickly.
+
+Let's see how using LanceDB inside phidata helps in making LLM more useful:
+
+## Prerequisites: install and import necessary dependencies
+
+**Create a virtual environment**
+
+1. install virtualenv package
+    ```python
+    pip install virtualenv
+    ```
+2. Create a directory for your project and go to the directory and create a virtual environment inside it.
+    ```python
+    mkdir phi
+    ```
+    ```python
+    cd phi
+    ```
+    ```python
+    python -m venv phidata_
+    ```
+
+**Activating virtual environment**
+
+1. from inside the project directory, run the following command to activate the virtual environment.
+    ```python
+    phidata_/Scripts/activate
+    ```
+
+**Install the following packages in the virtual environment**
+```python
+pip install lancedb phidata youtube_transcript_api openai ollama numpy pandas
+```
+
+**Create python files and import necessary libraries**
+
+You need to create two files - `transcript.py` and `ollama_assistant.py` or `openai_assistant.py`
+
+=== "openai_assistant.py"
+
+    ```python
+    import os, openai
+    from rich.prompt import Prompt
+    from phi.assistant import Assistant
+    from phi.knowledge.text import TextKnowledgeBase
+    from phi.vectordb.lancedb import LanceDb
+    from phi.llm.openai import OpenAIChat
+    from phi.embedder.openai import OpenAIEmbedder
+    from transcript import extract_transcript
+
+    if "OPENAI_API_KEY" not in os.environ:
+    # OR set the key here as a variable
+        openai.api_key = "sk-..."
+
+    # The code below creates a file "transcript.txt" in the directory, the txt file will be used below
+    youtube_url = "https://www.youtube.com/watch?v=Xs33-Gzl8Mo" 
+    segment_duration = 20
+    transcript_text,dict_transcript = extract_transcript(youtube_url,segment_duration)
+    ```
+
+=== "ollama_assistant.py"
+
+    ```python
+    from rich.prompt import Prompt
+    from phi.assistant import Assistant
+    from phi.knowledge.text import TextKnowledgeBase
+    from phi.vectordb.lancedb import LanceDb
+    from phi.llm.ollama import Ollama
+    from phi.embedder.ollama import OllamaEmbedder
+    from transcript import extract_transcript
+
+    # The code below creates a file "transcript.txt" in the directory, the txt file will be used below
+    youtube_url = "https://www.youtube.com/watch?v=Xs33-Gzl8Mo"
+    segment_duration = 20
+    transcript_text,dict_transcript = extract_transcript(youtube_url,segment_duration)
+    ```
+
+=== "transcript.py"
+
+    ``` python
+    from youtube_transcript_api import YouTubeTranscriptApi
+    import re
+
+    def smodify(seconds):
+        hours, remainder = divmod(seconds, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        return f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}"
+        
+    def extract_transcript(youtube_url,segment_duration):
+        # Extract video ID from the URL
+        video_id = re.search(r'(?<=v=)[\w-]+', youtube_url)
+        if not video_id:
+            video_id = re.search(r'(?<=be/)[\w-]+', youtube_url)
+        if not video_id:
+            return None
+
+        video_id = video_id.group(0)
+
+        # Attempt to fetch the transcript
+        try:
+            # Try to get the official transcript
+            transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=['en'])
+        except Exception:
+            # If no official transcript is found, try to get auto-generated transcript
+            try:
+                transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+                for transcript in transcript_list:
+                    transcript = transcript.translate('en').fetch()
+            except Exception:
+                return None
+
+        # Format the transcript into 120s chunks
+        transcript_text,dict_transcript = format_transcript(transcript,segment_duration)
+        # Open the file in write mode, which creates it if it doesn't exist
+        with open("transcript.txt", "w",encoding="utf-8") as file:
+            file.write(transcript_text)
+        return transcript_text,dict_transcript
+
+    def format_transcript(transcript,segment_duration):
+        chunked_transcript = []
+        chunk_dict = []
+        current_chunk = []
+        current_time = 0
+        # 2 minutes in seconds
+        start_time_chunk = 0  # To track the start time of the current chunk
+
+        for segment in transcript:
+            start_time = segment['start']
+            end_time_x = start_time + segment['duration']
+            text = segment['text']
+            
+            # Add text to the current chunk
+            current_chunk.append(text)
+            
+            # Update the current time with the duration of the current segment
+            # The duration of the current segment is given by segment['start'] - start_time_chunk
+            if current_chunk:
+                current_time = start_time - start_time_chunk
+            
+            # If current chunk duration reaches or exceeds 2 minutes, save the chunk
+            if current_time >= segment_duration:
+                # Use the start time of the first segment in the current chunk as the timestamp
+                chunked_transcript.append(f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}] " + " ".join(current_chunk))
+                current_chunk = re.sub(r'[\xa0\n]', lambda x: '' if x.group() == '\xa0' else ' ', "\n".join(current_chunk))
+                chunk_dict.append({"timestamp":f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}]", "text": "".join(current_chunk)})
+                current_chunk = []  # Reset the chunk
+                start_time_chunk = start_time + segment['duration'] # Update the start time for the next chunk
+                current_time = 0  # Reset current time
+
+        # Add any remaining text in the last chunk
+        if current_chunk:
+            chunked_transcript.append(f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}] " + " ".join(current_chunk))
+            current_chunk = re.sub(r'[\xa0\n]', lambda x: '' if x.group() == '\xa0' else ' ', "\n".join(current_chunk))
+            chunk_dict.append({"timestamp":f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}]", "text": "".join(current_chunk)})
+
+        return "\n\n".join(chunked_transcript), chunk_dict
+    ```
+
+!!! warning
+    If creating Ollama assistant, download and install Ollama [from here](https://ollama.com/) and then run the Ollama instance in the background. Also, download the required models using `ollama pull <model-name>`. Check out the models [here](https://ollama.com/library) 
+
+
+**Run the following command to deactivate the virtual environment if needed**
+```python
+deactivate
+```
+
+## **Step 1** - Create a Knowledge Base for AI Assistant using LanceDB
+
+=== "openai_assistant.py"
+
+    ```python
+    # Create knowledge Base with OpenAIEmbedder in LanceDB
+    knowledge_base = TextKnowledgeBase(
+        path="transcript.txt",
+        vector_db=LanceDb(
+            embedder=OpenAIEmbedder(api_key = openai.api_key),
+            table_name="transcript_documents",
+            uri="./t3mp/.lancedb",
+        ),
+        num_documents = 10
+    )
+    ```
+
+=== "ollama_assistant.py"
+
+    ```python
+    # Create knowledge Base with OllamaEmbedder in LanceDB
+    knowledge_base = TextKnowledgeBase(
+        path="transcript.txt",
+        vector_db=LanceDb(
+            embedder=OllamaEmbedder(model="nomic-embed-text",dimensions=768),
+            table_name="transcript_documents",
+            uri="./t2mp/.lancedb",
+        ),
+        num_documents = 10
+    )
+    ```
+Check out the list of **embedders** supported by **phidata** and their usage [here](https://docs.phidata.com/embedder/introduction).
+
+Here we have used `TextKnowledgeBase`, which loads text/docx files to the knowledge base.
+
+Let's see all the parameters that `TextKnowledgeBase` takes -
+
+| Name| Type | Purpose | Default |
+|:----|:-----|:--------|:--------|
+|`path`|`Union[str, Path]`| Path to text file(s). It can point to a single text file or a directory of text files.| provided by user |
+|`formats`|`List[str]`| File formats accepted by this knowledge base. |`[".txt"]`|
+|`vector_db`|`VectorDb`| Vector Database for the Knowledge Base. phidata provides a wrapper around many vector DBs, you can import it like this - `from phi.vectordb.lancedb import LanceDb` | provided by user |
+|`num_documents`|`int`| Number of results (documents/vectors) that vector search should return. |`5`|
+|`reader`|`TextReader`| phidata provides many types of reader objects which read data, clean it and create chunks of data, encapsulate each chunk inside an object of the `Document` class, and return **`List[Document]`**.  | `TextReader()` |
+|`optimize_on`|`int`| It is used to specify the number of documents on which to optimize the vector database. Supposed to create an index. |`1000`|
+
+??? Tip "Wonder! What is `Document` class?"
+    We know that, before storing the data in vectorDB, we need to split the data into smaller chunks upon which embeddings will be created and these embeddings along with the chunks will be stored in vectorDB. When the user queries over the vectorDB, some of these embeddings will be returned as the result based on the semantic similarity with the query.
+    
+    When the user queries over vectorDB, the queries are converted into embeddings, and a nearest neighbor search is performed over these query embeddings which returns the embeddings that correspond to most semantically similar chunks(parts of our data) present in vectorDB. 
+
+    Here, a “Document” is a class in phidata. Since there is an option to let phidata create and manage embeddings, it splits our data into smaller chunks(as expected). It does not directly create embeddings on it. Instead, it takes each chunk and encapsulates it inside the object of the `Document` class along with various other metadata related to the chunk. Then embeddings are created on these `Document` objects and stored in vectorDB.
+
+    ```python
+    class Document(BaseModel):
+        """Model for managing a document"""
+
+        content: str # <--- here data of chunk is stored 
+        id: Optional[str] = None
+        name: Optional[str] = None
+        meta_data: Dict[str, Any] = {}
+        embedder: Optional[Embedder] = None
+        embedding: Optional[List[float]] = None
+        usage: Optional[Dict[str, Any]] = None
+    ```
+
+However, using phidata you can load many other types of data in the knowledge base(other than text). Check out [phidata Knowledge Base](https://docs.phidata.com/knowledge/introduction) for more information.
+
+Let's dig deeper into the `vector_db` parameter and see what parameters `LanceDb` takes -
+
+| Name| Type | Purpose | Default |
+|:----|:-----|:--------|:--------|
+|`embedder`|`Embedder`| phidata provides many Embedders that abstract the interaction with embedding APIs and utilize it to generate embeddings. Check out other embedders [here](https://docs.phidata.com/embedder/introduction) | `OpenAIEmbedder` |
+|`distance`|`List[str]`| The choice of distance metric used to calculate the similarity between vectors, which directly impacts search results and performance in vector databases. |`Distance.cosine`|
+|`connection`|`lancedb.db.LanceTable`| LanceTable can be accessed through `.connection`. You can connect to an existing table of LanceDB, created outside of phidata, and utilize it. If not provided, it creates a new table using `table_name` parameter and adds it to `connection`. |`None`|
+|`uri`|`str`| It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. | `"/tmp/lancedb"` |
+|`table_name`|`str`|  If `connection` is not provided, it initializes and connects to a new **LanceDB table** with a specified(or default) name in the database present at `uri`. |`"phi"`|
+|`nprobes`|`int`| It refers to the number of partitions that the search algorithm examines to find the nearest neighbors of a given query vector. Higher values will yield better recall (more likely to find vectors if they exist) at the expense of latency. |`20`|
+
+
+!!! note
+    Since we just initialized the KnowledgeBase. The VectorDB table that corresponds to this Knowledge Base is not yet populated with our data. It will be populated in **Step 3**, once we perform the `load` operation. 
+    
+    You can check the state of the LanceDB table using - `knowledge_base.vector_db.connection.to_pandas()`
+
+Now that the Knowledge Base is initialized, , we can go to **step 2**.
+
+## **Step 2** -  Create an assistant with our choice of LLM and reference to the knowledge base.
+
+
+=== "openai_assistant.py"
+
+    ```python
+    # define an assistant with gpt-4o-mini llm and reference to the knowledge base created above
+    assistant = Assistant(
+        llm=OpenAIChat(model="gpt-4o-mini", max_tokens=1000, temperature=0.3,api_key = openai.api_key),
+        description="""You are an Expert in explaining youtube video transcripts. You are a bot that takes transcript of a video and answer the question based on it.
+        
+        This is transcript for the above timestamp: {relevant_document}
+        The user input is: {user_input}
+        generate highlights only when asked.
+        When asked to generate highlights from the video, understand the context for each timestamp and create key highlight points, answer in following way - 
+        [timestamp] - highlight 1
+        [timestamp] - highlight 2
+        ... so on
+        
+        Your task is to understand the user question, and provide an answer using the provided contexts. Your answers are correct, high-quality, and written by an domain expert. If the provided context does not contain the answer, simply state,'The provided context does not have the answer.'""",
+        knowledge_base=knowledge_base,
+        add_references_to_prompt=True,
+    )
+    ```
+
+=== "ollama_assistant.py"
+
+    ```python
+    # define an assistant with llama3.1 llm and reference to the knowledge base created above
+    assistant = Assistant(
+        llm=Ollama(model="llama3.1"),
+        description="""You are an Expert in explaining youtube video transcripts. You are a bot that takes transcript of a video and answer the question based on it.
+
+        This is transcript for the above timestamp: {relevant_document}
+        The user input is: {user_input}
+        generate highlights only when asked.
+        When asked to generate highlights from the video, understand the context for each timestamp and create key highlight points, answer in following way - 
+        [timestamp] - highlight 1
+        [timestamp] - highlight 2
+        ... so on
+
+        Your task is to understand the user question, and provide an answer using the provided contexts. Your answers are correct, high-quality, and written by an domain expert. If the provided context does not contain the answer, simply state,'The provided context does not have the answer.'""",
+        knowledge_base=knowledge_base,
+        add_references_to_prompt=True,
+    )
+    ```
+
+Assistants add **memory**, **knowledge**, and **tools** to LLMs. Here we will add only **knowledge** in this example. 
+
+Whenever we will give a query to LLM, the assistant will retrieve relevant information from our **Knowledge Base**(table in LanceDB) and pass it to LLM along with the user query in a structured way. 
+
+- The `add_references_to_prompt=True` always adds information from the knowledge base to the prompt, regardless of whether it is relevant to the question.
+
+To know more about an creating assistant in phidata, check out [phidata docs](https://docs.phidata.com/assistants/introduction) here.
+
+## **Step 3** - Load data to Knowledge Base.
+
+```python
+# load out data into the knowledge_base (populating the LanceTable)
+assistant.knowledge_base.load(recreate=False)
+```
+The above code loads the data to the Knowledge Base(LanceDB Table) and now it is ready to be used by the assistant. 
+
+| Name| Type | Purpose | Default |
+|:----|:-----|:--------|:--------|
+|`recreate`|`bool`| If True, it drops the existing table and recreates the table in the vectorDB. |`False`|
+|`upsert`|`bool`| If True and the vectorDB supports upsert, it will upsert documents to the vector db. | `False` |
+|`skip_existing`|`bool`| If True, skips documents that already exist in the vectorDB when inserting. |`True`|
+
+??? tip "What is upsert?"
+    Upsert is a database operation that combines "update" and "insert". It updates existing records if a document with the same identifier does exist, or inserts new records if no matching record exists. This is useful for maintaining the most current information without manually checking for existence.
+
+During the Load operation, phidata directly interacts with the LanceDB library and performs the loading of the table with our data in the following steps -     
+
+1. **Creates** and **initializes** the table if it does not exist.
+
+2. Then it **splits** our data into smaller **chunks**.
+
+    ??? question "How do they create chunks?"
+        **phidata** provides many types of **Knowledge Bases** based on the type of data. Most of them :material-information-outline:{ title="except LlamaIndexKnowledgeBase and LangChainKnowledgeBase"} has a property method called `document_lists` of type `Iterator[List[Document]]`. During the load operation, this property method is invoked. It traverses on the data provided by us (in this case, a text file(s)) using `reader`. Then it **reads**, **creates chunks**, and **encapsulates** each chunk inside a `Document` object and yields **lists of `Document` objects** that contain our data.
+
+3. Then **embeddings** are created on these chunks are **inserted** into the LanceDB Table 
+
+    ??? question "How do they insert your data as different rows in LanceDB Table?"
+        The chunks of your data are in the form - **lists of `Document` objects**. It was yielded in the step above.
+
+        for each `Document` in `List[Document]`, it does the following operations:
+        
+        - Creates embedding on `Document`.
+        - Cleans the **content attribute**(chunks of our data is here) of `Document`.
+        - Prepares data by creating `id` and loading `payload` with the metadata related to this chunk. (1)
+            { .annotate }
+
+            1.  Three columns will be added to the table - `"id"`, `"vector"`, and `"payload"` (payload contains various metadata including **`content`**)
+                
+        - Then add this data to LanceTable. 
+
+4. Now the internal state of `knowledge_base` is changed (embeddings are created and loaded in the table ) and it **ready to be used by assistant**.
+
+## **Step 4** - Start a cli chatbot with access to the Knowledge base  
+
+```python
+# start cli chatbot with knowledge base
+assistant.print_response("Ask me about something from the knowledge base")
+while True:
+    message = Prompt.ask(f"[bold] :sunglasses: User [/bold]")
+    if message in ("exit", "bye"):
+        break
+    assistant.print_response(message, markdown=True)
+```  
+
+
+For more information and amazing cookbooks of phidata, read the [phidata documentation](https://docs.phidata.com/introduction) and also visit [LanceDB x phidata docmentation](https://docs.phidata.com/vectordb/lancedb).

docs/src/integrations/voxel51.md

@@ -1,13 +1,73 @@
 # FiftyOne
 
-FiftyOne is an open source toolkit for building high-quality datasets and computer vision models. It provides an API to create LanceDB tables and run similarity queries, both programmatically in Python and via point-and-click in the App.
+FiftyOne is an open source toolkit that enables users to curate better data and build better models. It includes tools for data exploration, visualization, and management, as well as features for collaboration and sharing. 
+
+Any developers, data scientists, and researchers who work with computer vision and machine learning can use FiftyOne to improve the quality of their datasets and deliver insights about their models.
+
 
 ![example](../assets/voxel.gif)
 
-## Basic recipe
+**FiftyOne** provides an API to create LanceDB tables and run similarity queries, both **programmatically in Python** and via **point-and-click in the App**.
 
-The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne
+Let's get started and see how to use **LanceDB** to create a **similarity index** on your FiftyOne datasets.
-datasets:
+
+## Overview 
+
+**[Embeddings](../embeddings/understanding_embeddings.md)** are foundational to all of the **vector search** features. In FiftyOne, embeddings are managed by the [**FiftyOne Brain**](https://docs.voxel51.com/user_guide/brain.html) that provides powerful machine learning techniques designed to transform how you curate your data from an art into a measurable science. 
+
+!!!question "Have you ever wanted to find the images most similar to an image in your dataset?"
+    The **FiftyOne Brain** makes computing **visual similarity** really easy. You can compute the similarity of samples in your dataset using an embedding model and store the results in the **brain key**. 
+
+    You can then sort your samples by similarity or use this information to find potential duplicate images.
+
+Here we will be doing the following : 
+
+1. **Create Index** - In order to run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function. 
+
+    -  In the function, specify the **model** you want to use to generate the embedding vectors, and what **vector search engine** you want to use on the **backend** (here LanceDB). 
+    
+    !!!tip
+        You can also give the similarity index a name(`brain_key`), which is useful if you want to run vector searches against multiple indexes.
+
+2. **Query** - Once you have generated your similarity index, you can query your dataset with `sort_by_similarity()`. The query can be any of the following:
+
+    - An ID (sample or patch)
+    - A query vector of same dimension as the index
+    - A list of IDs (samples or patches)
+    - A text prompt (search semantically)
+
+## Prerequisites: install necessary dependencies
+
+1. **Create and activate a virtual environment**
+
+    Install virtualenv package and run the following command in your project directory.
+    ```python
+    python -m venv fiftyone_
+    ```
+    From inside the project directory run the following to activate the virtual environment.
+    === "Windows"
+
+        ```python
+        fiftyone_/Scripts/activate
+        ```
+
+    === "macOS/Linux"
+
+        ```python
+        source fiftyone_/Scripts/activate
+        ```
+
+2. **Install the following packages in the virtual environment**
+
+    To install FiftyOne, ensure you have activated any virtual environment that you are using, then run
+    ```python
+    pip install fiftyone
+    ```
+
+
+## Understand basic workflow
+
+The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne datasets:
 
 1. Load a dataset into FiftyOne.
 
@@ -19,14 +79,10 @@ datasets:
 
 5. If desired, delete the table.
 
-The example below demonstrates this workflow.
+## Quick Example
 
-!!! Note
+Let's jump on a quick example that demonstrates this workflow.
 
-    Install the LanceDB Python client to run the code shown below.
-    ```
-    pip install lancedb
-    ```
 
 ```python
 
@@ -36,7 +92,10 @@ import fiftyone.zoo as foz
 
 # Step 1: Load your data into FiftyOne
 dataset = foz.load_zoo_dataset("quickstart")
+```
+Make sure you install torch ([guide here](https://pytorch.org/get-started/locally/)) before proceeding. 
 
+```python
 # Steps 2 and 3: Compute embeddings and create a similarity index
 lancedb_index = fob.compute_similarity(
     dataset, 
@@ -45,8 +104,11 @@ lancedb_index = fob.compute_similarity(
     backend="lancedb",
 )
 ```
-Once the similarity index has been generated, we can query our data in FiftyOne
+
-by specifying the `brain_key`:
+!!! note
+    Running the code above will download the clip model (2.6Gb)
+
+Once the similarity index has been generated, we can query our data in FiftyOne by specifying the `brain_key`:
 
 ```python
 # Step 4: Query your data
@@ -56,7 +118,22 @@ view = dataset.sort_by_similarity(
     brain_key="lancedb_index",
     k=10,  # limit to 10 most similar samples
 )
+```
+The returned result are of type - `DatasetView`.
 
+!!! note
+    `DatasetView` does not hold its contents in-memory. Views simply store the rule(s) that are applied to extract the content of interest from the underlying Dataset when the view is iterated/aggregated on.
+
+    This means, for example, that the contents of a `DatasetView` may change as the underlying Dataset is modified.
+
+??? question "Can you query a view instead of dataset?"
+    Yes, you can also query a view.
+
+    Performing a similarity search on a `DatasetView` will only return results from the view; if the view contains samples that were not included in the index, they will never be included in the result.
+
+    This means that you can index an entire Dataset once and then perform searches on subsets of the dataset by constructing views that contain the images of interest.
+
+```python
 # Step 5 (optional): Cleanup
 
 # Delete the LanceDB table
@@ -66,4 +143,90 @@ lancedb_index.cleanup()
 dataset.delete_brain_run("lancedb_index")
 ```
 
+
+## Using LanceDB backend
+By default, calling `compute_similarity()` or `sort_by_similarity()` will use an sklearn backend.
+
+To use the LanceDB backend, simply set the optional `backend` parameter of `compute_similarity()` to `"lancedb"`:
+
+```python
+import fiftyone.brain as fob
+#... rest of the code
+fob.compute_similarity(..., backend="lancedb", ...)
+```
+
+Alternatively, you can configure FiftyOne to use the LanceDB backend by setting the following environment variable.
+
+In your terminal, set the environment variable using:
+=== "Windows"
+
+    ```python
+    $Env:FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND="lancedb" //powershell
+
+    set FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb //cmd
+    ```
+
+=== "macOS/Linux"
+
+    ```python
+    export FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb
+    ```
+
+!!! note
+    This will only run during the terminal session. Once terminal is closed, environment variable is deleted.
+
+Alternatively, you can **permanently** configure FiftyOne to use the LanceDB backend creating a `brain_config.json` at `~/.fiftyone/brain_config.json`. The JSON file may contain any desired subset of config fields that you wish to customize.
+
+```json
+{
+    "default_similarity_backend": "lancedb"
+}
+```
+This will override the default `brain_config` and will set it according to your customization. You can check the configuration by running the following code : 
+
+```python
+import fiftyone.brain as fob
+# Print your current brain config
+print(fob.brain_config)
+```
+
+## LanceDB config parameters
+
+The LanceDB backend supports query parameters that can be used to customize your similarity queries. These parameters include:
+
+| Name| Purpose | Default |
+|:----|:--------|:--------|
+|**table_name**|The name of the LanceDB table to use. If none is provided, a new table will be created|`None`|
+|**metric**|The embedding distance metric to use when creating a new table. The supported values are ("cosine", "euclidean")|`"cosine"`|
+|**uri**| The database URI to use. In this Database URI, tables will be created. |`"/tmp/lancedb"`|
+
+There are two ways to specify/customize the parameters:
+
+1. **Using `brain_config.json` file** 
+
+    ```json
+    {
+        "similarity_backends": {
+            "lancedb": {
+                "table_name": "your-table",
+                "metric": "euclidean",
+                "uri": "/tmp/lancedb"
+            }
+        }
+    }
+    ```
+
+2. **Directly passing to `compute_similarity()` to configure a specific new index** : 
+
+    ```python
+    lancedb_index = fob.compute_similarity(
+        ...
+        backend="lancedb",
+        brain_key="lancedb_index",
+        table_name="your-table",
+        metric="euclidean",
+        uri="/tmp/lancedb",
+    )
+    ```
+
 For a much more in depth walkthrough of the integration, visit the LanceDB x Voxel51 [docs page](https://docs.voxel51.com/integrations/lancedb.html).

docs/src/javascript/README.md

@@ -41,7 +41,6 @@ To build everything fresh:
 
 ```bash
 npm install
-npm run tsc
 npm run build
 ```
 
@@ -51,18 +50,6 @@ Then you should be able to run the tests with:
 npm test
 ```
 
-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
 ### Fix lints
 
 To run the linter and have it automatically fix all errors

docs/src/javascript/classes/DefaultWriteOptions.md

@@ -38,4 +38,4 @@ A [WriteMode](../enums/WriteMode.md) to use on this operation
 
 #### Defined in
 
-[index.ts:1019](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1019)
+[index.ts:1359](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1359)

docs/src/javascript/classes/LocalConnection.md

@@ -30,6 +30,7 @@ A connection to a LanceDB database.
 - [dropTable](LocalConnection.md#droptable)
 - [openTable](LocalConnection.md#opentable)
 - [tableNames](LocalConnection.md#tablenames)
+- [withMiddleware](LocalConnection.md#withmiddleware)
 
 ## Constructors
 
@@ -46,7 +47,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:489](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L489)
+[index.ts:739](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L739)
 
 ## Properties
 
@@ -56,7 +57,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:487](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L487)
+[index.ts:737](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L737)
 
 ___
 
@@ -74,7 +75,7 @@ ___
 
 #### Defined in
 
-[index.ts:486](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L486)
+[index.ts:736](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L736)
 
 ## Accessors
 
@@ -92,7 +93,7 @@ ___
 
 #### Defined in
 
-[index.ts:494](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L494)
+[index.ts:744](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L744)
 
 ## Methods
 
@@ -113,7 +114,7 @@ Creates a new Table, optionally initializing it with new data.
 | Name | Type |
 | :------ | :------ |
 | `name` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
-| `data?` | `Record`\<`string`, `unknown`\>[] |
+| `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
 | `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 | `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
 
@@ -127,7 +128,7 @@ Creates a new Table, optionally initializing it with new data.
 
 #### Defined in
 
-[index.ts:542](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L542)
+[index.ts:788](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L788)
 
 ___
 
@@ -158,7 +159,7 @@ ___
 
 #### Defined in
 
-[index.ts:576](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L576)
+[index.ts:822](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L822)
 
 ___
 
@@ -184,7 +185,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:630](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L630)
+[index.ts:876](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L876)
 
 ___
 
@@ -210,7 +211,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:510](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L510)
+[index.ts:760](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L760)
 
 ▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
@@ -239,7 +240,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:518](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L518)
+[index.ts:768](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L768)
 
 ▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
@@ -266,7 +267,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:522](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L522)
+[index.ts:772](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L772)
 
 ___
 
@@ -286,4 +287,36 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:501](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L501)
+[index.ts:751](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L751)
+
+___
+
+### withMiddleware
+
+▸ **withMiddleware**(`middleware`): [`Connection`](../interfaces/Connection.md)
+
+Instrument the behavior of this Connection with middleware.
+
+The middleware will be called in the order they are added.
+
+Currently this functionality is only supported for remote Connections.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `middleware` | `HttpMiddleware` |
+
+#### Returns
+
+[`Connection`](../interfaces/Connection.md)
+
+- this Connection instrumented by the passed middleware
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[withMiddleware](../interfaces/Connection.md#withmiddleware)
+
+#### Defined in
+
+[index.ts:880](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L880)

docs/src/javascript/classes/LocalTable.md

@@ -37,6 +37,8 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Methods
 
 - [add](LocalTable.md#add)
+- [addColumns](LocalTable.md#addcolumns)
+- [alterColumns](LocalTable.md#altercolumns)
 - [checkElectron](LocalTable.md#checkelectron)
 - [cleanupOldVersions](LocalTable.md#cleanupoldversions)
 - [compactFiles](LocalTable.md#compactfiles)
@@ -44,13 +46,16 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [createIndex](LocalTable.md#createindex)
 - [createScalarIndex](LocalTable.md#createscalarindex)
 - [delete](LocalTable.md#delete)
+- [dropColumns](LocalTable.md#dropcolumns)
 - [filter](LocalTable.md#filter)
 - [getSchema](LocalTable.md#getschema)
 - [indexStats](LocalTable.md#indexstats)
 - [listIndices](LocalTable.md#listindices)
+- [mergeInsert](LocalTable.md#mergeinsert)
 - [overwrite](LocalTable.md#overwrite)
 - [search](LocalTable.md#search)
 - [update](LocalTable.md#update)
+- [withMiddleware](LocalTable.md#withmiddleware)
 
 ## Constructors
 
@@ -74,7 +79,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:642](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L642)
+[index.ts:892](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L892)
 
 • **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
@@ -95,7 +100,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:649](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L649)
+[index.ts:899](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L899)
 
 ## Properties
 
@@ -105,7 +110,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:639](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L639)
+[index.ts:889](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L889)
 
 ___
 
@@ -115,7 +120,7 @@ ___
 
 #### Defined in
 
-[index.ts:638](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L638)
+[index.ts:888](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L888)
 
 ___
 
@@ -125,7 +130,7 @@ ___
 
 #### Defined in
 
-[index.ts:637](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L637)
+[index.ts:887](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L887)
 
 ___
 
@@ -143,7 +148,7 @@ ___
 
 #### Defined in
 
-[index.ts:640](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L640)
+[index.ts:890](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L890)
 
 ___
 
@@ -153,7 +158,7 @@ ___
 
 #### Defined in
 
-[index.ts:636](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L636)
+[index.ts:886](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L886)
 
 ___
 
@@ -179,7 +184,7 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:688](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L688)
+[index.ts:938](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L938)
 
 ## Accessors
 
@@ -197,7 +202,7 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:668](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L668)
+[index.ts:918](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L918)
 
 ___
 
@@ -215,7 +220,7 @@ ___
 
 #### Defined in
 
-[index.ts:849](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L849)
+[index.ts:1171](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1171)
 
 ## Methods
 
@@ -229,7 +234,7 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
@@ -243,7 +248,59 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:696](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L696)
+[index.ts:946](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L946)
+
+___
+
+### addColumns
+
+▸ **addColumns**(`newColumnTransforms`): `Promise`\<`void`\>
+
+Add new columns with defined values.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `newColumnTransforms` | \{ `name`: `string` ; `valueSql`: `string`  }[] | pairs of column names and the SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns in the table. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[addColumns](../interfaces/Table.md#addcolumns)
+
+#### Defined in
+
+[index.ts:1195](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1195)
+
+___
+
+### alterColumns
+
+▸ **alterColumns**(`columnAlterations`): `Promise`\<`void`\>
+
+Alter the name or nullability of columns.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `columnAlterations` | [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[] | One or more alterations to apply to columns. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[alterColumns](../interfaces/Table.md#altercolumns)
+
+#### Defined in
+
+[index.ts:1201](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1201)
 
 ___
 
@@ -257,7 +314,7 @@ ___
 
 #### Defined in
 
-[index.ts:861](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L861)
+[index.ts:1183](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1183)
 
 ___
 
@@ -280,7 +337,7 @@ Clean up old versions of the table, freeing disk space.
 
 #### Defined in
 
-[index.ts:808](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L808)
+[index.ts:1130](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1130)
 
 ___
 
@@ -307,16 +364,22 @@ Metrics about the compaction operation.
 
 #### Defined in
 
-[index.ts:831](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L831)
+[index.ts:1153](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1153)
 
 ___
 
 ### countRows
 
-▸ **countRows**(): `Promise`\<`number`\>
+▸ **countRows**(`filter?`): `Promise`\<`number`\>
 
 Returns the number of rows in this table.
 
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `filter?` | `string` |
+
 #### Returns
 
 `Promise`\<`number`\>
@@ -327,7 +390,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:749](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L749)
+[index.ts:1021](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1021)
 
 ___
 
@@ -357,13 +420,13 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:734](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L734)
+[index.ts:1003](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1003)
 
 ___
 
 ### createScalarIndex
 
-▸ **createScalarIndex**(`column`, `replace`): `Promise`\<`void`\>
+▸ **createScalarIndex**(`column`, `replace?`): `Promise`\<`void`\>
 
 Create a scalar index on this Table for the given column
 
@@ -372,7 +435,7 @@ Create a scalar index on this Table for the given column
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `column` | `string` | The column to index |
-| `replace` | `boolean` | If false, fail if an index already exists on the column Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
+| `replace?` | `boolean` | If false, fail if an index already exists on the column it is always set to true for remote connections Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
 
 #### Returns
 
@@ -392,7 +455,7 @@ await table.createScalarIndex('my_col')
 
 #### Defined in
 
-[index.ts:742](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L742)
+[index.ts:1011](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1011)
 
 ___
 
@@ -418,7 +481,38 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:758](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L758)
+[index.ts:1030](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1030)
+
+___
+
+### dropColumns
+
+▸ **dropColumns**(`columnNames`): `Promise`\<`void`\>
+
+Drop one or more columns from the dataset
+
+This is a metadata-only operation and does not remove the data from the
+underlying storage. In order to remove the data, you must subsequently
+call ``compact_files`` to rewrite the data without the removed columns and
+then call ``cleanup_files`` to remove the old files.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `columnNames` | `string`[] | The names of the columns to drop. These can be nested column references (e.g. "a.b.c") or top-level column names (e.g. "a"). |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[dropColumns](../interfaces/Table.md#dropcolumns)
+
+#### Defined in
+
+[index.ts:1205](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1205)
 
 ___
 
@@ -438,9 +532,13 @@ Creates a filter query to find all rows matching the specified criteria
 
 [`Query`](Query.md)\<`T`\>
 
+#### Implementation of
+
+[Table](../interfaces/Table.md).[filter](../interfaces/Table.md#filter)
+
 #### Defined in
 
-[index.ts:684](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L684)
+[index.ts:934](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L934)
 
 ___
 
@@ -454,13 +552,13 @@ ___
 
 #### Defined in
 
-[index.ts:854](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L854)
+[index.ts:1176](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1176)
 
 ___
 
 ### indexStats
 
-▸ **indexStats**(`indexUuid`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+▸ **indexStats**(`indexName`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
 
 Get statistics about an index.
 
@@ -468,7 +566,7 @@ Get statistics about an index.
 
 | Name | Type |
 | :------ | :------ |
-| `indexUuid` | `string` |
+| `indexName` | `string` |
 
 #### Returns
 
@@ -480,7 +578,7 @@ Get statistics about an index.
 
 #### Defined in
 
-[index.ts:845](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L845)
+[index.ts:1167](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1167)
 
 ___
 
@@ -500,7 +598,57 @@ List the indicies on this table.
 
 #### Defined in
 
-[index.ts:841](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L841)
+[index.ts:1163](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1163)
+
+___
+
+### mergeInsert
+
+▸ **mergeInsert**(`on`, `data`, `args`): `Promise`\<`void`\>
+
+Runs a "merge insert" operation on the table
+
+This operation can add rows, update rows, and remove rows all in a single
+transaction. It is a very generic tool that can be used to create
+behaviors like "insert if not exists", "update or insert (i.e. upsert)",
+or even replace a portion of existing data with new data (e.g. replace
+all data where month="january")
+
+The merge insert operation works by combining new data from a
+**source table** with existing data in a **target table** by using a
+join.  There are three categories of records.
+
+"Matched" records are records that exist in both the source table and
+the target table. "Not matched" records exist only in the source table
+(e.g. these are new data) "Not matched by source" records exist only
+in the target table (this is old data)
+
+The MergeInsertArgs can be used to customize what should happen for
+each category of data.
+
+Please note that the data may appear to be reordered as part of this
+operation.  This is because updated rows will be deleted from the
+dataset and then reinserted at the end with the new values.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `on` | `string` | a column to join on. This is how records from the source table and target table are matched. |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | the new data to insert |
+| `args` | [`MergeInsertArgs`](../interfaces/MergeInsertArgs.md) | parameters controlling how the operation should behave |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[mergeInsert](../interfaces/Table.md#mergeinsert)
+
+#### Defined in
+
+[index.ts:1065](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1065)
 
 ___
 
@@ -514,7 +662,7 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
@@ -528,7 +676,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:716](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L716)
+[index.ts:977](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L977)
 
 ___
 
@@ -554,7 +702,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:676](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L676)
+[index.ts:926](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L926)
 
 ___
 
@@ -580,4 +728,36 @@ Update rows in this table.
 
 #### Defined in
 
-[index.ts:771](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L771)
+[index.ts:1043](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1043)
+
+___
+
+### withMiddleware
+
+▸ **withMiddleware**(`middleware`): [`Table`](../interfaces/Table.md)\<`T`\>
+
+Instrument the behavior of this Table with middleware.
+
+The middleware will be called in the order they are added.
+
+Currently this functionality is only supported for remote tables.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `middleware` | `HttpMiddleware` |
+
+#### Returns
+
+[`Table`](../interfaces/Table.md)\<`T`\>
+
+- this Table instrumented by the passed middleware
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[withMiddleware](../interfaces/Table.md#withmiddleware)
+
+#### Defined in
+
+[index.ts:1209](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1209)

docs/src/javascript/classes/MakeArrowTableOptions.md

@@ -0,0 +1,82 @@
+[vectordb](../README.md) / [Exports](../modules.md) / MakeArrowTableOptions
+
+# Class: MakeArrowTableOptions
+
+Options to control the makeArrowTable call.
+
+## Table of contents
+
+### Constructors
+
+- [constructor](MakeArrowTableOptions.md#constructor)
+
+### Properties
+
+- [dictionaryEncodeStrings](MakeArrowTableOptions.md#dictionaryencodestrings)
+- [embeddings](MakeArrowTableOptions.md#embeddings)
+- [schema](MakeArrowTableOptions.md#schema)
+- [vectorColumns](MakeArrowTableOptions.md#vectorcolumns)
+
+## Constructors
+
+### constructor
+
+• **new MakeArrowTableOptions**(`values?`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `values?` | `Partial`\<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)\> |
+
+#### Defined in
+
+[arrow.ts:98](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L98)
+
+## Properties
+
+### dictionaryEncodeStrings
+
+• **dictionaryEncodeStrings**: `boolean` = `false`
+
+If true then string columns will be encoded with dictionary encoding
+
+Set this to true if your string columns tend to repeat the same values
+often.  For more precise control use the `schema` property to specify the
+data type for individual columns.
+
+If `schema` is provided then this property is ignored.
+
+#### Defined in
+
+[arrow.ts:96](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L96)
+
+___
+
+### embeddings
+
+• `Optional` **embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`any`\>
+
+#### Defined in
+
+[arrow.ts:85](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L85)
+
+___
+
+### schema
+
+• `Optional` **schema**: `Schema`\<`any`\>
+
+#### Defined in
+
+[arrow.ts:63](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L63)
+
+___
+
+### vectorColumns
+
+• **vectorColumns**: `Record`\<`string`, `VectorColumnOptions`\>
+
+#### Defined in
+
+[arrow.ts:81](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L81)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:22](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L22)
 
 ## Properties
 
@@ -50,17 +50,17 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:20](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L20)
 
 ___
 
 ### \_openai
 
-• `Private` `Readonly` **\_openai**: `any`
+• `Private` `Readonly` **\_openai**: `OpenAI`
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:56](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L56)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:43](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L43)

docs/src/javascript/classes/Query.md

@@ -19,6 +19,7 @@ A builder for nearest neighbor queries for LanceDB.
 ### Properties
 
 - [\_embeddings](Query.md#_embeddings)
+- [\_fastSearch](Query.md#_fastsearch)
 - [\_filter](Query.md#_filter)
 - [\_limit](Query.md#_limit)
 - [\_metricType](Query.md#_metrictype)
@@ -34,6 +35,7 @@ A builder for nearest neighbor queries for LanceDB.
 ### Methods
 
 - [execute](Query.md#execute)
+- [fastSearch](Query.md#fastsearch)
 - [filter](Query.md#filter)
 - [isElectron](Query.md#iselectron)
 - [limit](Query.md#limit)
@@ -65,7 +67,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[query.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L38)
+[query.ts:39](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L39)
 
 ## Properties
 
@@ -75,7 +77,17 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[query.ts:36](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L36)
+[query.ts:37](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L37)
+
+___
+
+### \_fastSearch
+
+• `Private` **\_fastSearch**: `boolean`
+
+#### Defined in
+
+[query.ts:36](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L36)
 
 ___
 
@@ -85,7 +97,7 @@ ___
 
 #### Defined in
 
-[query.ts:33](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L33)
+[query.ts:33](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L33)
 
 ___
 
@@ -95,7 +107,7 @@ ___
 
 #### Defined in
 
-[query.ts:29](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L29)
+[query.ts:29](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L29)
 
 ___
 
@@ -105,7 +117,7 @@ ___
 
 #### Defined in
 
-[query.ts:34](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L34)
+[query.ts:34](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L34)
 
 ___
 
@@ -115,7 +127,7 @@ ___
 
 #### Defined in
 
-[query.ts:31](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L31)
+[query.ts:31](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L31)
 
 ___
 
@@ -125,7 +137,7 @@ ___
 
 #### Defined in
 
-[query.ts:35](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L35)
+[query.ts:35](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L35)
 
 ___
 
@@ -135,7 +147,7 @@ ___
 
 #### Defined in
 
-[query.ts:26](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L26)
+[query.ts:26](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L26)
 
 ___
 
@@ -145,7 +157,7 @@ ___
 
 #### Defined in
 
-[query.ts:28](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L28)
+[query.ts:28](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L28)
 
 ___
 
@@ -155,7 +167,7 @@ ___
 
 #### Defined in
 
-[query.ts:30](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L30)
+[query.ts:30](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L30)
 
 ___
 
@@ -165,7 +177,7 @@ ___
 
 #### Defined in
 
-[query.ts:32](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L32)
+[query.ts:32](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L32)
 
 ___
 
@@ -175,7 +187,7 @@ ___
 
 #### Defined in
 
-[query.ts:27](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L27)
+[query.ts:27](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L27)
 
 ___
 
@@ -201,7 +213,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[query.ts:87](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L87)
+[query.ts:90](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L90)
 
 ## Methods
 
@@ -223,7 +235,30 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[query.ts:115](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L115)
+[query.ts:127](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L127)
+
+___
+
+### fastSearch
+
+▸ **fastSearch**(`value`): [`Query`](Query.md)\<`T`\>
+
+Skip searching un-indexed data. This can make search faster, but will miss
+any data that is not yet indexed.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `boolean` |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[query.ts:119](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L119)
 
 ___
 
@@ -245,7 +280,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[query.ts:82](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L82)
+[query.ts:85](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L85)
 
 ___
 
@@ -259,7 +294,7 @@ ___
 
 #### Defined in
 
-[query.ts:142](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L142)
+[query.ts:155](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L155)
 
 ___
 
@@ -268,6 +303,7 @@ ___
 ▸ **limit**(`value`): [`Query`](Query.md)\<`T`\>
 
 Sets the number of results that will be returned
+default value is 10
 
 #### Parameters
 
@@ -281,7 +317,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[query.ts:55](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L55)
+[query.ts:58](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L58)
 
 ___
 
@@ -307,7 +343,7 @@ MetricType for the different options
 
 #### Defined in
 
-[query.ts:102](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L102)
+[query.ts:105](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L105)
 
 ___
 
@@ -329,7 +365,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[query.ts:73](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L73)
+[query.ts:76](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L76)
 
 ___
 
@@ -349,7 +385,7 @@ ___
 
 #### Defined in
 
-[query.ts:107](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L107)
+[query.ts:110](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L110)
 
 ___
 
@@ -371,7 +407,7 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[query.ts:64](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L64)
+[query.ts:67](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L67)
 
 ___
 
@@ -393,4 +429,4 @@ Return only the specified columns.
 
 #### Defined in
 
-[query.ts:93](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L93)
+[query.ts:96](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L96)