update

Solves -  https://github.com/lancedb/lancedb/issues/968

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.12
+current_version = 0.4.14
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/docs_test.yml

@@ -18,7 +18,7 @@ on:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
@@ -28,6 +28,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Print CPU capabilities
+      run: cat /proc/cpuinfo
     - name: Install dependecies needed for ubuntu
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
@@ -64,6 +66,8 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
+    - name: Print CPU capabilities
+      run: cat /proc/cpuinfo
     - name: Set up Node
       uses: actions/setup-node@v4
       with:

.github/workflows/node.yml

@@ -20,7 +20,8 @@ env:
   # "1" means line tables only, which is useful for panic tracebacks.
   #
   # Use native CPU to accelerate tests if possible, especially for f16
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
+  # target-cpu=haswell fixes failing ci build
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:

.github/workflows/npm-publish.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: node/package-lock.json
       - name: Install dependencies
         run: |
@@ -31,7 +31,7 @@ jobs:
           npm run tsc
           npm pack
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: node-package
           path: |
@@ -61,12 +61,41 @@ jobs:
       - name: Build MacOS native node modules
         run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-darwin
+          name: node-native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
 
+  nodejs-macos:
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-14
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build MacOS native nodejs modules
+        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-darwin-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
@@ -103,12 +132,63 @@ jobs:
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-linux
+          name: node-native-linux
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
+  nodejs-linux:
+    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-16vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-linux-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
+      # The generic files are the same in all distros so we just pick
+      # one to do the upload.
+      - name: Upload Generic Artifacts
+        if: ${{ matrix.config.arch == 'x86_64' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-dist
+          path: |
+            nodejs/dist/*
+            !nodejs/dist/*.node
+
   node-windows:
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
@@ -136,25 +216,60 @@ jobs:
       - name: Build Windows native node modules
         run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
       - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-windows
+          name: node-native-windows
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
+  nodejs-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows
+          path: |
+            nodejs/dist/*.node
+
   release:
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: node-*
       - name: Display structure of downloaded files
         run: ls -R
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
@@ -164,6 +279,45 @@ jobs:
             npm publish $filename
           done
 
+  release-nodejs:
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+      - uses: actions/download-artifact@v4
+        name: Download arch-specific binaries
+        with:
+          pattern: nodejs-*
+          path: nodejs/nodejs-artifacts
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: find .
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+      - name: Install napi-rs
+        run: npm install -g @napi-rs/cli
+      - name: Prepare artifacts
+        run: npx napi artifacts -d nodejs-artifacts
+      - name: Display structure of staged files
+        run: find npm
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        run: npm publish --access public
+
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
@@ -178,3 +332,18 @@ jobs:
       - uses: ./.github/workflows/update_package_lock
         with:
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+
+  update-package-lock-nodejs:
+    needs: [release-nodejs]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/update_package_lock_nodejs/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock_nodejs
+description: "Update nodejs's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./nodejs
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run_nodejs.yml

@@ -0,0 +1,19 @@
+name: Update NodeJs package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.gitignore

@@ -34,6 +34,7 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
+nodejs/lancedb/native*
 dist
 
 ## Rust

Cargo.toml

@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.10.2", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.10.2" }
-lance-linalg = { "version" = "=0.10.2" }
-lance-testing = { "version" = "=0.10.2" }
+lance = { "version" = "=0.10.5", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.10.5" }
+lance-linalg = { "version" = "=0.10.5" }
+lance-testing = { "version" = "=0.10.5" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"
@@ -28,13 +28,16 @@ arrow-schema = "50.0"
 arrow-arith = "50.0"
 arrow-cast = "50.0"
 async-trait = "0"
-chrono = "0.4.23"
+chrono = "0.4.35"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
 object_store = "0.9.0"
+pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
+regex = "1.10"
+lazy_static = "1"

ci/build_linux_artifacts_nodejs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+ARCH=${1:-x86_64}
+
+# We pass down the current user so that when we later mount the local files
+# into the container, the files are accessible by the current user.
+pushd ci/manylinux_nodejs
+docker build \
+    -t lancedb-nodejs-manylinux \
+    --build-arg="ARCH=$ARCH" \
+    --build-arg="DOCKER_USER=$(id -u)" \
+    --progress=plain \
+    .
+popd
+
+# We turn on memory swap to avoid OOM killer
+docker run \
+    -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
+    lancedb-nodejs-manylinux \
+    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -0,0 +1,34 @@
+# Builds the macOS artifacts (nodejs binaries).
+# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/lancedb
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd nodejs
+    echo "Building nodejs library for $1"
+    export RUST_TARGET=$1
+    npm run build-release
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts_nodejs.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (nodejs binaries).
+# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/lancedb"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "nodejs"
+    Write-Host "Building nodejs library for $target"
+    $env:RUST_TARGET=$target
+    npm run build-release
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/manylinux_nodejs/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd nodejs
+npm ci
+npm run build-release

ci/manylinux_nodejs/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

docs/mkdocs.yml

@@ -27,7 +27,6 @@ theme:
     - content.tabs.link
     - content.action.edit
     - toc.follow
-    # - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
@@ -47,10 +46,11 @@ plugins:
           paths: [../python]
           options:
             docstring_style: numpy
-          heading_level: 4
+            heading_level: 3
             show_source: true
             show_symbol_type_in_heading: true
             show_signature_annotations: true
+            show_root_heading: true
             members_order: source
           import:
             # for cross references
@@ -104,9 +104,18 @@ nav:
               - Overview: hybrid_search/hybrid_search.md
               - Comparing Rerankers: hybrid_search/eval.md
               - Airbnb financial data example: notebooks/hybrid_search.ipynb
+          - Reranking:
+              - Quickstart: reranking/index.md
+              - Cohere Reranker: reranking/cohere.md
+              - Linear Combination Reranker: reranking/linear_combination.md
+              - Cross Encoder Reranker: reranking/cross_encoder.md
+              - ColBERT Reranker: reranking/colbert.md
+              - OpenAI Reranker: reranking/openai.md
+              - Building Custom Rerankers: reranking/custom_reranker.md
           - Filtering: sql.md
           - Versioning & Reproducibility: notebooks/reproducibility.ipynb
           - Configuring Storage: guides/storage.md
+          - Sync -> Async Migration Guide: migration.md
       - 🧬 Managing embeddings:
           - Overview: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
@@ -140,19 +149,20 @@ nav:
               - Serverless Website Chatbot: examples/serverless_website_chatbot.md
               - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
               - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+          - 🦀 Rust:
+              - Overview: examples/examples_rust.md
       - 🔧 CLI & Config: cli_config.md
       - 💭 FAQs: faq.md
       - ⚙️ API reference:
           - 🐍 Python: python/python.md
           - 👾 JavaScript: javascript/modules.md
-    - 🦀 Rust: https://docs.rs/vectordb/latest/vectordb/
+          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
       - ☁️ LanceDB Cloud:
           - Overview: cloud/index.md
           - API reference:
               - 🐍 Python: python/saas-python.md
               - 👾 JavaScript: javascript/saas-modules.md
 
-
   - Quick start: basic.md
   - Concepts:
       - Vector search: concepts/vector_search.md
@@ -168,9 +178,18 @@ nav:
           - Overview: hybrid_search/hybrid_search.md
           - Comparing Rerankers: hybrid_search/eval.md
           - Airbnb financial data example: notebooks/hybrid_search.ipynb
+      - Reranking:
+          - Quickstart: reranking/index.md
+          - Cohere Reranker: reranking/cohere.md
+          - Linear Combination Reranker: reranking/linear_combination.md
+          - Cross Encoder Reranker: reranking/cross_encoder.md
+          - ColBERT Reranker: reranking/colbert.md
+          - OpenAI Reranker: reranking/openai.md
+          - Building Custom Rerankers: reranking/custom_reranker.md
       - Filtering: sql.md
       - Versioning & Reproducibility: notebooks/reproducibility.ipynb
       - Configuring Storage: guides/storage.md
+      - Sync -> Async Migration Guide: migration.md
   - Managing Embeddings:
       - Overview: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
@@ -189,21 +208,21 @@ nav:
       - Pydantic: python/pydantic.md
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
-- Python examples:
+  - Examples:
       - examples/index.md
       - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
       - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
       - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
       - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
       - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-- Javascript examples:
-  - Overview: examples/examples_js.md
-  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+      - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
       - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
       - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
   - API reference:
+      - Overview: api_reference.md
       - Python: python/python.md
       - Javascript: javascript/modules.md
+      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
   - LanceDB Cloud:
       - Overview: cloud/index.md
       - API reference:

docs/src/ann_indexes.md

@@ -46,12 +46,34 @@ Lance supports `IVF_PQ` index type by default.
     --8<-- "docs/src/ann_indexes.ts:ingest"
     ```
 
-- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
+    ```
+
+    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
+
+The following IVF_PQ paramters can be specified:
+
+- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
   We also support "cosine" and "dot" distance as well.
-- **num_partitions** (default: 256): The number of partitions of the index.
-- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-  For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-  a single PQ code.
+- **num_partitions**: The number of partitions in the index. The default is the square root
+  of the number of rows.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
+
+- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
+  a single PQ code. The default is the dimension of the vector divided by 16.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -134,6 +156,14 @@ There are a couple of parameters that can be used to fine-tune the search:
     --8<-- "docs/src/ann_indexes.ts:search1"
     ```
 
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
+    ```
+
+    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
+
 The search will return the data requested in addition to the distance of each item.
 
 ### Filtering (where clause)

docs/src/api_reference.md

@@ -0,0 +1,7 @@
+# API Reference
+
+The API reference for the LanceDB client SDKs are available at the following locations:
+
+- [Python](python/python.md)
+- [JavaScript](javascript/modules.md)
+- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -3,7 +3,7 @@
 !!! info "LanceDB can be run in a number of ways:"
 
     * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Connected to directly from a client application like a Jupyter notebook for analytical workloads
+    * Directly from a client application like a Jupyter notebook for analytical workloads
     * Deployed as a remote serverless database
 
 ![](assets/lancedb_embedded_explanation.png)
@@ -24,13 +24,11 @@
 
 === "Rust"
 
-    !!! warning "Rust SDK is experimental, might introduce breaking changes in the near future"
-
     ```shell
-    cargo add vectordb
+    cargo add lancedb
     ```
 
-    !!! info "To use the vectordb create, you first need to install protobuf."
+    !!! info "To use the lancedb create, you first need to install protobuf."
 
     === "macOS"
 
@@ -44,18 +42,27 @@
         sudo apt install -y protobuf-compiler libssl-dev
         ```
 
-    !!! info "Please also make sure you're using the same version of Arrow as in the [vectordb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
+    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
 
 ## Connect to a database
 
 === "Python"
 
     ```python
-      import lancedb
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
+    --8<-- "python/python/tests/docs/test_basic.py:imports"
+    --8<-- "python/python/tests/docs/test_basic.py:connect"
+
+    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
     ```
 
+    !!! note "Asynchronous Python API"
+
+        The asynchronous Python API is new and has some slight differences compared
+        to the synchronous API.  Feel free to start using the asynchronous version.
+        Once all features have migrated we will start to move the synchronous API to
+        use the same syntax as the asynchronous API.  To help with this migration we
+        have created a [migration guide](migration.md) detailing the differences.
+
 === "Typescript"
 
     ```typescript
@@ -81,17 +88,17 @@ If you need a reminder of the uri, you can call `db.uri()`.
 
 ## Create a table
 
-### Directly insert data to a new table
+### Create a table from initial data
 
 If you have data to insert into the table at creation time, you can simultaneously create a
-table and insert the data to it.
+table and insert the data into it. The schema of the data will be used as the schema of the
+table.
 
 === "Python"
 
     ```python
-    tbl = db.create_table("my_table",
-                    data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                          {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    --8<-- "python/python/tests/docs/test_basic.py:create_table"
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
     ```
 
     If the table already exists, LanceDB will raise an error by default.
@@ -101,10 +108,8 @@ table and insert the data to it.
     You can also pass in a pandas DataFrame directly:
 
     ```python
-    import pandas as pd
-    df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-    tbl = db.create_table("table_from_df", data=df)
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
     ```
 
 === "Typescript"
@@ -120,28 +125,33 @@ table and insert the data to it.
 === "Rust"
 
     ```rust
-    use arrow_schema::{DataType, Schema, Field};
-    use arrow_array::{RecordBatch, RecordBatchIterator};
-
     --8<-- "rust/lancedb/examples/simple.rs:create_table"
     ```
 
-    If the table already exists, LanceDB will raise an error by default.
+    If the table already exists, LanceDB will raise an error by default.  See
+    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
+    for details on how to overwrite (or open) existing tables instead.
 
-!!! info "Under the hood, LanceDB converts the input data into an Apache Arrow table and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
+    !!! Providing table records in Rust
+
+        The Rust SDK currently expects data to be provided as an Arrow
+        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
+        Support for additional formats (such as serde or polars) is on the roadmap.
+
+!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
 
 ### Create an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
 In this case, you can create an empty table and specify the schema, so that you can add
-data to the table at a later time (such that it conforms to the schema).
+data to the table at a later time (as long as it conforms to the schema). This is
+similar to a `CREATE TABLE` statement in SQL.
 
 === "Python"
 
       ```python
-      import pyarrow as pa
-      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
-      tbl = db.create_table("empty_table", schema=schema)
+      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
+      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
       ```
 
 === "Typescript"
@@ -163,7 +173,8 @@ Once created, you can open a table as follows:
 === "Python"
 
     ```python
-    tbl = db.open_table("my_table")
+    --8<-- "python/python/tests/docs/test_basic.py:open_table"
+    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
     ```
 
 === "Typescript"
@@ -175,7 +186,7 @@ Once created, you can open a table as follows:
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:open_with_existing_file"
+    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
     ```
 
 If you forget the name of your table, you can always get a listing of all table names:
@@ -183,7 +194,8 @@ If you forget the name of your table, you can always get a listing of all table
 === "Python"
 
     ```python
-    print(db.table_names())
+    --8<-- "python/python/tests/docs/test_basic.py:table_names"
+    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
     ```
 
 === "Javascript"
@@ -205,15 +217,8 @@ After a table has been created, you can always add more data to it as follows:
 === "Python"
 
     ```python
-
-    # Option 1: Add a list of dicts to a table
-    data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
-    tbl.add(data)
-
-    # Option 2: Add a pandas DataFrame to a table
-    df = pd.DataFrame(data)
-    tbl.add(data)
+    --8<-- "python/python/tests/docs/test_basic.py:add_data"
+    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
     ```
 
 === "Typescript"
@@ -235,7 +240,8 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
 === "Python"
 
     ```python
-    tbl.search([100, 100]).limit(2).to_pandas()
+    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
+    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
     ```
 
     This returns a pandas DataFrame with the results.
@@ -254,6 +260,14 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
     --8<-- "rust/lancedb/examples/simple.rs:search"
     ```
 
+    !!! Query vectors in Rust
+        Rust does not yet support automatic execution of embedding functions.  You will need to
+        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
+        https://github.com/lancedb/lancedb/issues/994
+
+        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
+        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
+
 By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
 For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
 LanceDB allows you to create an ANN index on a table as follows:
@@ -261,7 +275,8 @@ LanceDB allows you to create an ANN index on a table as follows:
 === "Python"
 
     ```py
-    tbl.create_index()
+    --8<-- "python/python/tests/docs/test_basic.py:create_index"
+    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
     ```
 
 === "Typescript"
@@ -277,7 +292,7 @@ LanceDB allows you to create an ANN index on a table as follows:
     ```
 
 !!! note "Why do I need to create an index manually?"
-    LanceDB does not automatically create the ANN index, for two reasons. The first is that it's optimized
+LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
 for really fast retrievals via a disk-based index, and the second is that data and query workloads can
 be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
 to fine-tune index size, query latency and accuracy. See the section on
@@ -292,7 +307,8 @@ This can delete any number of rows that match the filter.
 === "Python"
 
     ```python
-    tbl.delete('item = "fizz"')
+    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
+    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
     ```
 
 === "Typescript"
@@ -308,8 +324,9 @@ This can delete any number of rows that match the filter.
     ```
 
 The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause on a search. They can be as simple or complex as needed.
-To see what expressions are supported, see the [SQL filters](sql.md) section.
+as the `where()` clause (`only_if()` in Rust) on a search. They can be as
+simple or complex as needed. To see what expressions are supported, see the
+[SQL filters](sql.md) section.
 
 === "Python"
 
@@ -319,6 +336,10 @@ To see what expressions are supported, see the [SQL filters](sql.md) section.
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
+=== "Rust"
+
+      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
+
 ## Drop a table
 
 Use the `drop_table()` method on the database to remove a table.
@@ -326,7 +347,8 @@ Use the `drop_table()` method on the database to remove a table.
 === "Python"
 
       ```python
-      db.drop_table("my_table")
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.

docs/src/concepts/index_ivfpq.md

@@ -31,7 +31,7 @@ As an example, consider starting with 128-dimensional vector consisting of 32-bi
 
 While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
 
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
 
 ![](../assets/ivfpq_ivf_desc.webp)
 

docs/src/embeddings/default_embedding_functions.md

@@ -19,14 +19,148 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
 
 
+??? "Check out available sentence-transformer models here!"
+    ```markdown
+    - sentence-transformers/all-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-mpnet-base-v2
+    - sentence-transformers/gtr-t5-base
+    - sentence-transformers/LaBSE
+    - sentence-transformers/all-MiniLM-L6-v2
+    - sentence-transformers/bert-base-nli-max-tokens
+    - sentence-transformers/bert-base-nli-mean-tokens
+    - sentence-transformers/bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
+    - sentence-transformers/bert-large-nli-cls-token
+    - sentence-transformers/bert-large-nli-max-tokens
+    - sentence-transformers/bert-large-nli-mean-tokens
+    - sentence-transformers/bert-large-nli-stsb-mean-tokens
+    - sentence-transformers/distilbert-base-nli-max-tokens
+    - sentence-transformers/distilbert-base-nli-mean-tokens
+    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
+    - sentence-transformers/distilroberta-base-msmarco-v1
+    - sentence-transformers/distilroberta-base-msmarco-v2
+    - sentence-transformers/nli-bert-base-cls-pooling
+    - sentence-transformers/nli-bert-base-max-pooling
+    - sentence-transformers/nli-bert-base
+    - sentence-transformers/nli-bert-large-cls-pooling
+    - sentence-transformers/nli-bert-large-max-pooling
+    - sentence-transformers/nli-bert-large
+    - sentence-transformers/nli-distilbert-base-max-pooling
+    - sentence-transformers/nli-distilbert-base
+    - sentence-transformers/nli-roberta-base
+    - sentence-transformers/nli-roberta-large
+    - sentence-transformers/roberta-base-nli-mean-tokens
+    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
+    - sentence-transformers/roberta-large-nli-mean-tokens
+    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
+    - sentence-transformers/stsb-bert-base
+    - sentence-transformers/stsb-bert-large
+    - sentence-transformers/stsb-distilbert-base
+    - sentence-transformers/stsb-roberta-base
+    - sentence-transformers/stsb-roberta-large
+    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
+    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
+    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
+    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
+    - sentence-transformers/bert-base-nli-cls-token
+    - sentence-transformers/all-distilroberta-v1
+    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
+    - sentence-transformers/multi-qa-distilbert-cos-v1
+    - sentence-transformers/multi-qa-distilbert-dot-v1
+    - sentence-transformers/multi-qa-mpnet-base-cos-v1
+    - sentence-transformers/multi-qa-mpnet-base-dot-v1
+    - sentence-transformers/nli-distilroberta-base-v2
+    - sentence-transformers/all-MiniLM-L6-v1
+    - sentence-transformers/all-mpnet-base-v1
+    - sentence-transformers/all-mpnet-base-v2
+    - sentence-transformers/all-roberta-large-v1
+    - sentence-transformers/allenai-specter
+    - sentence-transformers/average_word_embeddings_glove.6B.300d
+    - sentence-transformers/average_word_embeddings_glove.840B.300d
+    - sentence-transformers/average_word_embeddings_komninos
+    - sentence-transformers/average_word_embeddings_levy_dependency
+    - sentence-transformers/clip-ViT-B-32-multilingual-v1
+    - sentence-transformers/clip-ViT-B-32
+    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
+    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
+    - sentence-transformers/distilroberta-base-paraphrase-v1
+    - sentence-transformers/distiluse-base-multilingual-cased-v1
+    - sentence-transformers/distiluse-base-multilingual-cased-v2
+    - sentence-transformers/distiluse-base-multilingual-cased
+    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
+    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
+    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
+    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
+    - sentence-transformers/gtr-t5-large
+    - sentence-transformers/gtr-t5-xl
+    - sentence-transformers/gtr-t5-xxl
+    - sentence-transformers/msmarco-MiniLM-L-12-v3
+    - sentence-transformers/msmarco-MiniLM-L-6-v3
+    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
+    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
+    - sentence-transformers/msmarco-bert-base-dot-v5
+    - sentence-transformers/msmarco-bert-co-condensor
+    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
+    - sentence-transformers/msmarco-distilbert-base-tas-b
+    - sentence-transformers/msmarco-distilbert-base-v2
+    - sentence-transformers/msmarco-distilbert-base-v3
+    - sentence-transformers/msmarco-distilbert-base-v4
+    - sentence-transformers/msmarco-distilbert-cos-v5
+    - sentence-transformers/msmarco-distilbert-dot-v5
+    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
+    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
+    - sentence-transformers/msmarco-distilroberta-base-v2
+    - sentence-transformers/msmarco-roberta-base-ance-firstp
+    - sentence-transformers/msmarco-roberta-base-v2
+    - sentence-transformers/msmarco-roberta-base-v3
+    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
+    - sentence-transformers/nli-mpnet-base-v2
+    - sentence-transformers/nli-roberta-base-v2
+    - sentence-transformers/nq-distilbert-base-v1
+    - sentence-transformers/paraphrase-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-MiniLM-L3-v2
+    - sentence-transformers/paraphrase-MiniLM-L6-v2
+    - sentence-transformers/paraphrase-TinyBERT-L6-v2
+    - sentence-transformers/paraphrase-albert-base-v2
+    - sentence-transformers/paraphrase-albert-small-v2
+    - sentence-transformers/paraphrase-distilroberta-base-v1
+    - sentence-transformers/paraphrase-distilroberta-base-v2
+    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
+    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
+    - sentence-transformers/quora-distilbert-base
+    - sentence-transformers/quora-distilbert-multilingual
+    - sentence-transformers/sentence-t5-base
+    - sentence-transformers/sentence-t5-large
+    - sentence-transformers/sentence-t5-xxl
+    - sentence-transformers/sentence-t5-xl
+    - sentence-transformers/stsb-distilroberta-base-v2
+    - sentence-transformers/stsb-mpnet-base-v2
+    - sentence-transformers/stsb-roberta-base-v2
+    - sentence-transformers/stsb-xlm-r-multilingual
+    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
+    - sentence-transformers/clip-ViT-L-14
+    - sentence-transformers/clip-ViT-B-16
+    - sentence-transformers/use-cmlm-multilingual
+    - sentence-transformers/all-MiniLM-L12-v1
+    ```
+
+!!! info
+    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
+    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
+
+!!! note "BAAI Embeddings example"
+    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
     ```python
     db = lancedb.connect("/tmp/db")
     registry = EmbeddingFunctionRegistry.get_instance()
-func = registry.get("sentence-transformers").create(device="cpu")
+    model = registry.get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
 
     class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
 
     table = db.create_table("words", schema=Words)
     table.add(
@@ -40,6 +174,8 @@ query = "greetings"
     actual = table.search(query).limit(1).to_pydantic(Words)[0]
     print(actual.text)
     ```
+Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
+
 
 ### OpenAI embeddings
 LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
@@ -224,7 +360,6 @@ This embedding function supports ingesting images as both bytes and urls. You ca
 !!! info
     LanceDB supports ingesting images directly from accessible links.
 
-
 ```python
 
 db = lancedb.connect(tmp_path)
@@ -290,4 +425,67 @@ print(actual.label)
 
 ```
 
+### Imagebind embeddings
+We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
+
+This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"imagebind_huge"` | Name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
+
+Below is an example demonstrating how the API works:
+
+```python
+db = lancedb.connect(tmp_path)
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("imagebind").create()
+
+class ImageBindModel(LanceModel):
+    text: str
+    image_uri: str = func.SourceField()
+    audio_path: str
+    vector: Vector(func.ndims()) = func.VectorField()
+
+# add locally accessible image paths
+text_list=["A dog.", "A car", "A bird"]
+image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
+audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
+
+# Load data
+inputs = [
+    {"text": a, "audio_path": b, "image_uri": c}
+    for a, b, c in zip(text_list, audio_paths, image_paths)
+]
+
+#create table and add data
+table = db.create_table("img_bind", schema=ImageBindModel)
+table.add(inputs)
+```
+
+Now, we can search using any modality:
+
+#### image search
+```python
+query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
+actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "dog")
+```
+#### audio search
+
+```python
+query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
+actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "car")
+```
+#### Text search
+You can add any input query and fetch the result as follows:
+```python
+query = "an animal which flies and tweets" 
+actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "bird")
+```
+
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/examples/examples_rust.md

@@ -0,0 +1,3 @@
+# Examples: Rust
+
+Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/index.md

@@ -2,10 +2,11 @@
 
 ## Recipes and example code
 
-LanceDB provides language APIs, allowing you to embed a database in your language of choice. We currently provide Python and Javascript APIs, with the Rust API and examples actively being worked on and will be available soon.
+LanceDB provides language APIs, allowing you to embed a database in your language of choice.
 
 * 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](exampled_js.md) examples
+* 👾 [JavaScript](examples_js.md) examples
+* 🦀 Rust examples (coming soon)
 
 ## Applications powered by LanceDB
 

docs/src/extra_js/init_ask_ai_widget.js

@@ -1,11 +1,79 @@
+// Creates an SVG robot icon (from Lucide)
+function robotSVG() {
+  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+  svg.setAttribute("width", "24");
+  svg.setAttribute("height", "24");
+  svg.setAttribute("viewBox", "0 0 24 24");
+  svg.setAttribute("fill", "none");
+  svg.setAttribute("stroke", "currentColor");
+  svg.setAttribute("stroke-width", "2");
+  svg.setAttribute("stroke-linecap", "round");
+  svg.setAttribute("stroke-linejoin", "round");
+  svg.setAttribute("class", "lucide lucide-bot-message-square");
+
+  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path1.setAttribute("d", "M12 6V2H8");
+  svg.appendChild(path1);
+
+  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
+  svg.appendChild(path2);
+
+  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path3.setAttribute("d", "M2 12h2");
+  svg.appendChild(path3);
+
+  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path4.setAttribute("d", "M9 11v2");
+  svg.appendChild(path4);
+
+  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path5.setAttribute("d", "M15 11v2");
+  svg.appendChild(path5);
+
+  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path6.setAttribute("d", "M20 12h2");
+  svg.appendChild(path6);
+
+  return svg
+}
+
+// Creates the Fluidic Chatbot buttom
+function fluidicButton() {
+  var btn = document.createElement("a");
+  btn.href = "https://asklancedb.com";
+  btn.target = "_blank";
+  btn.style.position = "fixed";
+  btn.style.fontWeight = "bold";
+  btn.style.fontSize = ".8rem";
+  btn.style.right = "10px";
+  btn.style.bottom = "10px";
+  btn.style.width = "80px";
+  btn.style.height = "80px";
+  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
+  btn.style.color = "white";
+  btn.style.borderRadius = "5px";
+  btn.style.display = "flex";
+  btn.style.flexDirection = "column";
+  btn.style.justifyContent = "center";
+  btn.style.alignItems = "center";
+  btn.style.zIndex = "1000";
+  btn.style.opacity = "0";
+  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
+  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
+
+  setTimeout(function() {
+      btn.style.opacity = "1";
+      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
+  }, 0);
+
+  return btn
+}
+
 document.addEventListener("DOMContentLoaded", function() {
-    var script = document.createElement("script");
-    script.src = "https://widget.kapa.ai/kapa-widget.bundle.js";
-    script.setAttribute("data-website-id", "c5881fae-cec0-490b-b45e-d83d131d4f25");
-    script.setAttribute("data-project-name", "LanceDB");
-    script.setAttribute("data-project-color", "#000000");
-    script.setAttribute("data-project-logo", "https://avatars.githubusercontent.com/u/108903835?s=200&v=4");
-  script.setAttribute("data-modal-example-questions","Help me create an IVF_PQ index,How do I do an exhaustive search?,How do I create a LanceDB table?,Can I use my own embedding function?");
-    script.async = true;
-    document.head.appendChild(script);
+  var btn = fluidicButton()
+  btn.appendChild(robotSVG());
+  var text = document.createTextNode("Ask AI");
+  btn.appendChild(text);
+  document.body.appendChild(btn);
 });

docs/src/faq.md

@@ -16,7 +16,7 @@ As we mention in our talk titled “[Lance, a modern columnar data format](https
 
 ### Why build in Rust? 🦀
 
-We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python and JavaScript client libraries to interact with the database. Our Rust API is a little rough around the edges right now, but is fast becoming on par with the Python and JS APIs.
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
 
 ### What is the difference between LanceDB OSS and LanceDB Cloud?
 
@@ -44,7 +44,7 @@ For large-scale (>1M) or higher dimension vectors, it is beneficial to create an
 
 ### Does LanceDB support full-text search?
 
-Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
 
 ### How can I speed up data inserts?
 

docs/src/fts.md

@@ -1,6 +1,6 @@
 # Full-text search
 
-LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for JavaScript users as well.
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
 A hybrid search solution combining vector and full-text search is also on the way.
 
@@ -75,6 +75,36 @@ applied on top of the full text search results. This can be invoked via the fami
 table.search("puppy").limit(10).where("meta='foo'").to_list()
 ```
 
+## Sorting
+
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example, 
+
+```
+table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
+
+(table.search("terms", ordering_field_name="sort_by_field")
+ .limit(20)
+ .to_list())
+```
+
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
+
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see 
+    an error during indexing that looks like 
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
+
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
+
+
 ## Phrase queries vs. terms queries
 
 For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
@@ -131,4 +161,3 @@ table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
 2. We currently only support local filesystem paths for the FTS index.
    This is a tantivy limitation. We've implemented an object store plugin
    but there's no way in tantivy-py to specify to use it.
-

docs/src/hybrid_search/hybrid_search.md

@@ -5,6 +5,9 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
 
+!!! note
+    You need to create a full-text search index before performing a hybrid search. You can create a full-text search index using the `create_fts_index()` method of the table object.
+
 ```python
 import os
 
@@ -55,188 +58,7 @@ By default, LanceDB uses `LinearCombinationReranker(weight=0.7)` to combine and
 
 
 ## Available Rerankers
-LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Here's a list of available re-rankers:
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Visit the [rerankers](../reranking/) page for more information on each re-ranker.
 
-### Linear Combination Reranker
-This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
-
-
-```python
-from lancedb.rerankers import LinearCombinationReranker
-
-reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search
-
-results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `weight`: `float`, default `0.7`:
-    The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
-* `fill`: `float`, default `1.0`:
-        The score to give to results that are only in one of the two result sets.This is treated as penalty, so a higher value means a lower score.
-        TODO: We should just hardcode this-- its pretty confusing as we invert scores to calculate final score
-* `return_score` : str, default `"relevance"`
-        options are "relevance" or "all"
-        The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score.
-
-### Cohere Reranker
-This re-ranker uses the [Cohere](https://cohere.ai/) API to combine the results of semantic and full-text search. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll need to set the `COHERE_API_KEY` environment variable to use this re-ranker.
-
-```python
-from lancedb.rerankers import CohereReranker
-
-reranker = CohereReranker()
-
-results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `model_name` : str, default `"rerank-english-v2.0"`
-        The name of the cross encoder model to use. Available cohere models are:
-        - rerank-english-v2.0
-        - rerank-multilingual-v2.0
-* `column` : str, default `"text"`
-        The name of the column to use as input to the cross encoder model.
-* `top_n` : str, default `None`
-        The number of results to return. If None, will return all results.
-
-!!! Note
-    Only returns `_relevance_score`. Does not support `return_score = "all"`.
-
-### Cross Encoder Reranker
-This reranker uses the [Sentence Transformers](https://www.sbert.net/) library to combine the results of semantic and full-text search. You can use it by passing `CrossEncoderReranker()` to the `rerank()` method.
-
-```python
-from lancedb.rerankers import CrossEncoderReranker
-
-reranker = CrossEncoderReranker()
-
-results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-
-### Arguments
-----------------
-* `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
-        The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
-* `column` : str, default `"text"`
-        The name of the column to use as input to the cross encoder model.
-* `device` : str, default `None`
-        The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu".
-
-!!! Note
-    Only returns `_relevance_score`. Does not support `return_score = "all"`.
-
-
-### ColBERT Reranker
-This reranker uses the ColBERT model to combine the results of semantic and full-text search. You can use it by passing `ColbertrReranker()` to the `rerank()` method. 
-
-ColBERT reranker model calculates relevance of given docs against the query and don't take existing fts and vector search scores into account, so it currently only supports `return_score="relevance"`. By default, it looks for `text` column to rerank the results. But you can specify the column name to use as input to the cross encoder model as described below.
-
-```python
-from lancedb.rerankers import ColbertReranker
-
-reranker = ColbertReranker()
-
-results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
-        The name of the cross encoder model to use.
-* `column` : `str`, default `"text"`
-        The name of the column to use as input to the cross encoder model.
-* `return_score` : `str`, default `"relevance"`
-        options are `"relevance"` or `"all"`. Only `"relevance"` is supported for now.
-
-!!! Note
-    Only returns `_relevance_score`. Does not support `return_score = "all"`.
-
-### OpenAI Reranker
-This reranker uses the OpenAI API to combine the results of semantic and full-text search. You can use it by passing `OpenaiReranker()` to the `rerank()` method.
-
-!!! Note
-    This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
-
-!!! Tip
-    - You might run out of token limit so set the search `limits` based on your token limit.
-    - It is recommended to use gpt-4-turbo-preview, the default model, older models might lead to undesired behaviour
-
-```python
-from lancedb.rerankers import OpenaiReranker
-
-reranker = OpenaiReranker()
-
-results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `model_name` : `str`, default `"gpt-4-turbo-preview"`
-    The name of the cross encoder model to use.
-* `column` : `str`, default `"text"`
-    The name of the column to use as input to the cross encoder model.
-* `return_score` : `str`, default `"relevance"`
-    options are "relevance" or "all". Only "relevance" is supported for now.
-* `api_key` : `str`, default `None`
-    The API key to use. If None, will use the OPENAI_API_KEY environment variable.
-
-
-## Building Custom Rerankers
-You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
-
-The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
-
-```python
-
-from lancedb.rerankers import Reranker
-import pyarrow as pa
-
-class MyReranker(Reranker):
-    def __init__(self, param1, param2, ..., return_score="relevance"):
-        super().__init__(return_score)
-        self.param1 = param1
-        self.param2 = param2
-    
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
-        # Use the built-in merging function
-        combined_result = self.merge_results(vector_results, fts_results)
-        
-        # Do something with the combined results
-        # ...
-
-        # Return the combined results
-        return combined_result
-
-```
-
-### Example of a Custom Reranker
-For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
-
-```python
-
-from typing import List, Union
-import pandas as pd
-from lancedb.rerankers import CohereReranker
-
-class MofidifiedCohereReranker(CohereReranker):
-    def __init__(self, filters: Union[str, List[str]], **kwargs):
-        super().__init__(**kwargs)
-        filters = filters if isinstance(filters, list) else [filters]
-        self.filters = filters
-
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
-        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
-        df = combined_result.to_pandas()
-        for filter in self.filters:
-            df = df.query("not text.str.contains(@filter)")
-
-        return pa.Table.from_pandas(df)
-
-```
-
-!!! tip
-    The `vector_results` and `fts_results` are pyarrow tables. You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.
+## Custom Rerankers
+You can also create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. Visit the [custom rerankers](../reranking/custom_reranker.md) page for more information on creating custom rerankers.

docs/src/index.md

@@ -28,7 +28,7 @@ LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverles
 
 * Fast production-scale vector similarity, full-text & hybrid search and a SQL query interface (via [DataFusion](https://github.com/apache/arrow-datafusion))
 
-* Native Python and Javascript/Typescript support
+* Python, Javascript/Typescript, and Rust support
 
 * Store, query & manage multi-modal data (text, images, videos, point clouds, etc.), not just the embeddings and metadata
 
@@ -54,3 +54,4 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references
 * [JavaScript API Reference](javascript/modules.md): JavaScript OSS and Cloud API references
+* [Rust API Reference](https://docs.rs/lancedb/latest/lancedb/index.html): Rust API reference

docs/src/migration.md

@@ -0,0 +1,76 @@
+# Rust-backed Client Migration Guide
+
+In an effort to ensure all clients have the same set of capabilities we have begun migrating the
+python and node clients onto a common Rust base library. In python, this new client is part of
+the same lancedb package, exposed as an asynchronous client. Once the asynchronous client has
+reached full functionality we will begin migrating the synchronous library to be a thin wrapper
+around the asynchronous client.
+
+This guide describes the differences between the two APIs and will hopefully assist users
+that would like to migrate to the new API.
+
+## Closeable Connections
+
+The Connection now has a `close` method. You can call this when
+you are done with the connection to eagerly free resources. Currently
+this is limited to freeing/closing the HTTP connection for remote
+connections. In the future we may add caching or other resources to
+native connections so this is probably a good practice even if you
+aren't using remote connections.
+
+In addition, the connection can be used as a context manager which may
+be a more convenient way to ensure the connection is closed.
+
+```python
+import lancedb
+
+async def my_async_fn():
+    with await lancedb.connect_async("my_uri") as db:
+        print(await db.table_names())
+```
+
+It is not mandatory to call the `close` method. If you do not call it
+then the connection will be closed when the object is garbage collected.
+
+## Closeable Table
+
+The Table now also has a `close` method, similar to the connection. This
+can be used to eagerly free the cache used by a Table object. Similar to
+the connection, it can be used as a context manager and it is not mandatory
+to call the `close` method.
+
+### Changes to Table APIs
+
+- Previously `Table.schema` was a property. Now it is an async method.
+- The method `Table.__len__` was removed and `len(table)` will no longer
+  work. Use `Table.count_rows` instead.
+
+### Creating Indices
+
+The `Table.create_index` method is now used for creating both vector indices
+and scalar indices. It currently requires a column name to be specified (the
+column to index). Vector index defaults are now smarter and scale better with
+the size of the data.
+
+To specify index configuration details you will need to specify which kind of
+index you are using.
+
+### Querying
+
+The `Table.search` method has been renamed to `AsyncTable.vector_search` for
+clarity.
+
+## Features not yet supported
+
+The following features are not yet supported by the asynchronous API. However,
+we plan to support them soon.
+
+- You cannot specify an embedding function when creating or opening a table.
+  You must calculate embeddings yourself if using the asynchronous API
+- The merge insert operation is not supported in the asynchronous API
+- Cleanup / compact / optimize indices are not supported in the asynchronous API
+- add / alter columns is not supported in the asynchronous API
+- The asynchronous API does not yet support any full text search or reranking
+  search
+- Remote connections to LanceDb Cloud are not yet supported.
+- The method Table.head is not yet supported.

docs/src/python/python.md

@@ -8,17 +8,20 @@ This section contains the API reference for the OSS Python API.
 pip install lancedb
 ```
 
-## Connection
+The following methods describe the synchronous API client. There
+is also an [asynchronous API client](#connections-asynchronous).
+
+## Connections (Synchronous)
 
 ::: lancedb.connect
 
 ::: lancedb.db.DBConnection
 
-## Table
+## Tables (Synchronous)
 
 ::: lancedb.table.Table
 
-## Querying
+## Querying (Synchronous)
 
 ::: lancedb.query.Query
 
@@ -87,3 +90,41 @@ pip install lancedb
 ::: lancedb.rerankers.cross_encoder.CrossEncoderReranker
 
 ::: lancedb.rerankers.openai.OpenaiReranker
+
+## Connections (Asynchronous)
+
+Connections represent a connection to a LanceDb database and
+can be used to create, list, or open tables.
+
+::: lancedb.connect_async
+
+::: lancedb.db.AsyncConnection
+
+## Tables (Asynchronous)
+
+Table hold your actual data as a collection of records / rows.
+
+::: lancedb.table.AsyncTable
+
+## Indices (Asynchronous)
+
+Indices can be created on a table to speed up queries. This section
+lists the indices that LanceDb supports.
+
+::: lancedb.index.BTree
+
+::: lancedb.index.IvfPq
+
+## Querying (Asynchronous)
+
+Queries allow you to return data from your database. Basic queries can be
+created with the [AsyncTable.query][lancedb.table.AsyncTable.query] method
+to return the entire (typically filtered) table. Vector searches return the
+rows nearest to a query vector and can be created with the
+[AsyncTable.vector_search][lancedb.table.AsyncTable.vector_search] method.
+
+::: lancedb.query.AsyncQueryBase
+
+::: lancedb.query.AsyncQuery
+
+::: lancedb.query.AsyncVectorQuery

docs/src/reranking/cohere.md

@@ -0,0 +1,75 @@
+# Cohere Reranker
+
+This re-ranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+
+
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import CohereReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = CohereReranker(api_key="key")
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `"rerank-english-v2.0"` | The name of the reranker model to use. Available cohere models are: rerank-english-v2.0, rerank-multilingual-v2.0 |
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
+| `api_key` | `str` | `None` | The API key for the Cohere API. If not provided, the `COHERE_API_KEY` environment variable is used. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/colbert.md

@@ -0,0 +1,71 @@
+# ColBERT Reranker
+
+This re-ranker uses ColBERT model to rerank the search results. You can use this re-ranker by passing `ColbertReranker()` to the `rerank()` method. 
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import ColbertReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = ColbertReranker()
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `"colbert-ir/colbertv2.0"` | The name of the reranker model to use.|
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/cross_encoder.md

@@ -0,0 +1,70 @@
+# Cross Encoder Reranker
+
+This re-ranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this re-ranker by passing `CrossEncoderReranker()` to the `rerank()` method. 
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import CrossEncoderReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = CrossEncoderReranker()
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `""cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.|
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/custom_reranker.md

@@ -0,0 +1,89 @@
+## Building Custom Rerankers
+You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Optionally, you can also implement the `rerank_vector()` and `rerank_fts()` methods if you want to support reranking for vector and FTS search separately.
+Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
+
+The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
+
+```python
+
+from lancedb.rerankers import Reranker
+import pyarrow as pa
+
+class MyReranker(Reranker):
+    def __init__(self, param1, param2, ..., return_score="relevance"):
+        super().__init__(return_score)
+        self.param1 = param1
+        self.param2 = param2
+    
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
+        # Use the built-in merging function
+        combined_result = self.merge_results(vector_results, fts_results)
+        
+        # Do something with the combined results
+        # ...
+
+        # Return the combined results
+        return combined_result
+    
+    def rerank_vector(self, query: str, vector_results: pa.Table):
+        # Do something with the vector results
+        # ...
+
+        # Return the vector results
+        return vector_results
+    
+    def rerank_fts(self, query: str, fts_results: pa.Table):
+        # Do something with the FTS results
+        # ...
+
+        # Return the FTS results
+        return fts_results
+
+```
+
+### Example of a Custom Reranker
+For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
+
+```python
+
+from typing import List, Union
+import pandas as pd
+from lancedb.rerankers import CohereReranker
+
+class ModifiedCohereReranker(CohereReranker):
+    def __init__(self, filters: Union[str, List[str]], **kwargs):
+        super().__init__(**kwargs)
+        filters = filters if isinstance(filters, list) else [filters]
+        self.filters = filters
+
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
+        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
+        df = combined_result.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
+    
+    def rerank_vector(self, query: str, vector_results: pa.Table)-> pa.Table:
+        vector_results = super().rerank_vector(query, vector_results)
+        df = vector_results.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
+    
+    def rerank_fts(self, query: str, fts_results: pa.Table)-> pa.Table:
+        fts_results = super().rerank_fts(query, fts_results)
+        df = fts_results.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
+
+```
+
+!!! tip
+    The `vector_results` and `fts_results` are pyarrow tables. Lean more about pyarrow tables [here](https://arrow.apache.org/docs/python). It can be convered to other data types like pandas dataframe, pydict, pylist etc.
+
+    For example, You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.
+

docs/src/reranking/index.md

@@ -0,0 +1,61 @@
+Reranking is the process of reordering a list of items based on some criteria. In the context of search, reranking is used to reorder the search results returned by a search engine based on some criteria. This can be useful when the initial ranking of the search results is not satisfactory or when the user has provided additional information that can be used to improve the ranking of the search results.
+
+LanceDB comes with some built-in rerankers. Some of the rerankers that are available in LanceDB are:
+
+| Reranker | Description | Supported Query Types |
+| --- | --- | --- |
+| `LinearCombinationReranker` | Reranks search results based on a linear combination of FTS and vector search scores | Hybrid |
+| `CohereReranker` | Uses cohere Reranker API to rerank results | Vector, FTS, Hybrid |
+| `CrossEncoderReranker` | Uses a cross-encoder model to rerank search results | Vector, FTS, Hybrid |
+| `ColbertReranker` | Uses a colbert model to rerank search results | Vector, FTS, Hybrid |
+| `OpenaiReranker`(Experimental) | Uses OpenAI's chat model to rerank search results | Vector, FTS, Hybrid |
+
+
+## Using a Reranker
+Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder.
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import CohereReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", data)
+reranker = CohereReranker(api_key="your_api_key")
+
+# Run vector search with a reranker
+result = tbl.query("hello").rerank(reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.query("hello", query_type="fts").rerank(reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text")
+result = tbl.query("hello", query_type="hybrid").rerank(reranker).to_list()
+```
+
+## Available Rerankers
+LanceDB comes with some built-in rerankers. Here are some of the rerankers that are available in LanceDB:
+
+- [Cohere Reranker](./cohere.md)
+- [Cross Encoder Reranker](./cross_encoder.md)
+- [ColBERT Reranker](./colbert.md)
+- [OpenAI Reranker](./openai.md)
+- [Linear Combination Reranker](./linear_combination.md)
+
+## Creating Custom Rerankers
+
+LanceDB also you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. This is covered in more detail in the [Creating Custom Rerankers](./custom_reranker.md) section.
+

docs/src/reranking/linear_combination.md

@@ -0,0 +1,52 @@
+# Linear Combination Reranker
+
+This is the default re-ranker used by LanceDB hybrid search. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+
+!!! note
+    Supported Query Types: Hybrid
+
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import LinearCombinationReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = LinearCombinationReranker()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score. |
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) |

docs/src/reranking/openai.md

@@ -0,0 +1,73 @@
+# OpenAI Reranker (Experimental)
+
+This re-ranker uses OpenAI chat model to rerank the search results. You can use this re-ranker by passing `OpenAI()` to the `rerank()` method. 
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+!!! warning
+    This re-ranker is experimental. OpenAI doesn't have a dedicated reranking model, so we are using the chat model for reranking. 
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import OpenaiReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = OpenaiReranker()
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `"gpt-4-turbo-preview"` | The name of the reranker model to use.|
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `api_key` | str | `None` | The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/search.md

@@ -22,7 +22,7 @@ Currently, LanceDB supports the following metrics:
 ## Exhaustive search (kNN)
 
 If you do not create a vector index, LanceDB exhaustively scans the _entire_ vector space
-and compute the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
+and computes the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
 
 <!-- Setup Code
 ```python
@@ -85,7 +85,7 @@ To perform scalable vector retrieval with acceptable latencies, it's common to b
 While the exhaustive search is guaranteed to always return 100% recall, the approximate nature of
 an ANN search means that using an index often involves a trade-off between recall and latency.
 
-See the [IVF_PQ index](./concepts/index_ivfpq.md.md) for a deeper description of how `IVF_PQ`
+See the [IVF_PQ index](./concepts/index_ivfpq.md) for a deeper description of how `IVF_PQ`
 indexes work in LanceDB.
 
 ## Output search results
@@ -184,4 +184,3 @@ Let's create a LanceDB table with a nested schema:
 
     Note that in this case the extra `_distance` field is discarded since
     it's not part of the LanceSchema.
-

docs/test/md_testing.py

@@ -15,6 +15,7 @@ excluded_globs = [
     "../src/ann_indexes.md",
     "../src/basic.md",
     "../src/hybrid_search/hybrid_search.md",
+    "../src/reranking/*.md",
 ]
 
 python_prefix = "py"

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.12",
+  "version": "0.4.14",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.12",
+      "version": "0.4.14",
       "cpu": [
         "x64",
         "arm64"
@@ -52,11 +52,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.12",
-        "@lancedb/vectordb-darwin-x64": "0.4.12",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.12",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.12",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.12"
+        "@lancedb/vectordb-darwin-arm64": "0.4.14",
+        "@lancedb/vectordb-darwin-x64": "0.4.14",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.14",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.14",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.14"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -334,9 +334,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.12.tgz",
-      "integrity": "sha512-38/rkJRlWXkPWXuj9onzvbrhnIWcIUQjgEp5G9v5ixPosBowm7A4j8e2Q8CJMsVSNcVX2JLqwWVldiWegZFuYw==",
+      "version": "0.4.14",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.14.tgz",
+      "integrity": "sha512-fw6mf6UhFf4j2kKdFcw0P+SOiIqmRbt+YQSgDbF4BFU3OUSW0XyfETIj9cUMQbSwPFsofhlGp5BRpCd7W9noew==",
       "cpu": [
         "arm64"
       ],
@@ -345,22 +345,10 @@
         "darwin"
       ]
     },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.12.tgz",
-      "integrity": "sha512-psE48dztyO450hXWdv9Rl9aayM2HQ1uF9wErfC0gKmDUh1N0NdVq2viDuFpZxnmCis/nvGwKlYiYT9OnYNCJ9g==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.12.tgz",
-      "integrity": "sha512-xwkgF6MiF5aAdG9JG8v4ke652YxUJrhs9z4OrsEfrENnvsIQd2C5UyKMepVLdvij4BI/XPFRFWXdjPvP7S9rTA==",
+      "version": "0.4.14",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.14.tgz",
+      "integrity": "sha512-1+LFI8vU+f/lnGy1s3XCySuV4oj3ZUW03xtmedGBW8nv/Y/jWXP0OYJCRI72eu+dLIdu0tCPsEiu8Hl+o02t9g==",
       "cpu": [
         "arm64"
       ],
@@ -369,22 +357,10 @@
         "linux"
       ]
     },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.12.tgz",
-      "integrity": "sha512-gJqYR0aymrS+C60xc4EQPzmQ5/69XfeFv2ofBvAj7qW+c6BcnoAcfVl+7s1IrcWeGz251sm5cD5Lx4AzJd89dA==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.12.tgz",
-      "integrity": "sha512-LhCzpyEeBUyO6L2fuVqeP3mW8kYDryyU9PNqcM01m88sZB1Do6AlwiM+GjPRQ0SpzD0LK9oxQqSmJrdcNGqjbw==",
+      "version": "0.4.14",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.14.tgz",
+      "integrity": "sha512-fpuNMZ4aHSpZC3ztp5a0Wh18N6DpCx5EPWhS7bGA5XulGc0l+sZAJHfHwalx76ys//0Ns1z7cuKJhZpSa4SrdQ==",
       "cpu": [
         "x64"
       ],

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.4.12",
+  "version": "0.4.14",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -88,10 +88,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.12",
-    "@lancedb/vectordb-darwin-x64": "0.4.12",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.12",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.12",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.12"
+    "@lancedb/vectordb-darwin-arm64": "0.4.14",
+    "@lancedb/vectordb-darwin-x64": "0.4.14",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.14",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.14",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.14"
   }
 }

node/src/index.ts

@@ -24,6 +24,7 @@ import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
 import { type Literal, toSQL } from './util'
+import { type HttpMiddleware } from './middleware'
 
 const {
   databaseNew,
@@ -176,6 +177,10 @@ export async function connect (
     opts = { uri: arg }
   } else {
     // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
+    const keys = Object.keys(arg)
+    if (keys.length === 1 && keys[0] === 'uri' && typeof arg.uri === 'string') {
+      opts = { uri: arg.uri }
+    } else {
       opts = Object.assign(
         {
           uri: '',
@@ -187,6 +192,7 @@ export async function connect (
         arg
       )
     }
+  }
 
   if (opts.uri.startsWith('db://')) {
     // Remote connection
@@ -297,6 +303,18 @@ export interface Connection {
    * @param name The name of the table to drop.
    */
   dropTable(name: string): Promise<void>
+
+  /**
+   * Instrument the behavior of this Connection with middleware.
+   *
+   * The middleware will be called in the order they are added.
+   *
+   * Currently this functionality is only supported for remote Connections.
+   *
+   * @param {HttpMiddleware} - Middleware which will instrument the Connection.
+   * @returns - this Connection instrumented by the passed middleware
+   */
+  withMiddleware(middleware: HttpMiddleware): Connection
 }
 
 /**
@@ -536,6 +554,18 @@ export interface Table<T = number[]> {
    *                    names (e.g. "a").
    */
   dropColumns(columnNames: string[]): Promise<void>
+
+  /**
+   * Instrument the behavior of this Table with middleware.
+   *
+   * The middleware will be called in the order they are added.
+   *
+   * Currently this functionality is only supported for remote tables.
+   *
+   * @param {HttpMiddleware} - Middleware which will instrument the Table.
+   * @returns - this Table instrumented by the passed middleware
+   */
+  withMiddleware(middleware: HttpMiddleware): Table<T>
 }
 
 /**
@@ -790,6 +820,10 @@ export class LocalConnection implements Connection {
   async dropTable (name: string): Promise<void> {
     await databaseDropTable.call(this._db, name)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Connection {
+    return this
+  }
 }
 
 export class LocalTable<T = number[]> implements Table<T> {
@@ -1100,6 +1134,10 @@ export class LocalTable<T = number[]> implements Table<T> {
   async dropColumns (columnNames: string[]): Promise<void> {
     return tableDropColumns.call(this._tbl, columnNames)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Table<T> {
+    return this
+  }
 }
 
 export interface CleanupStats {

node/src/middleware.ts

@@ -0,0 +1,58 @@
+// Copyright 2024 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Middleware for Remote LanceDB Connection or Table
+ */
+export interface HttpMiddleware {
+  /**
+   * A callback that can be used to instrument the behavior of http requests to remote
+   * tables. It can be used to add headers, modify the request, or even short-circuit
+   * the request and return a response without making the request to the remote endpoint.
+   * It can also be used to modify the response from the remote endpoint.
+   *
+   * @param {RemoteResponse} res - Request to the remote endpoint
+   * @param {onRemoteRequestNext} next - Callback to advance the middleware chain
+   */
+  onRemoteRequest(
+    req: RemoteRequest,
+    next: (req: RemoteRequest) => Promise<RemoteResponse>,
+  ): Promise<RemoteResponse>
+};
+
+export enum Method {
+  GET,
+  POST
+}
+
+/**
+ * A LanceDB Remote HTTP Request
+ */
+export interface RemoteRequest {
+  uri: string
+  method: Method
+  headers: Map<string, string>
+  params?: Map<string, string>
+  body?: any
+}
+
+/**
+ * A LanceDB Remote HTTP Response
+ */
+export interface RemoteResponse {
+  status: number
+  statusText: string
+  headers: Map<string, string>
+  body: () => Promise<any>
+}

node/src/remote/client.ts

@@ -12,13 +12,101 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import axios, { type AxiosResponse } from 'axios'
+import axios, { type AxiosResponse, type ResponseType } from 'axios'
 
 import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
 
+import { type RemoteResponse, type RemoteRequest, Method } from '../middleware'
+
+interface HttpLancedbClientMiddleware {
+  onRemoteRequest(
+    req: RemoteRequest,
+    next: (req: RemoteRequest) => Promise<RemoteResponse>,
+  ): Promise<RemoteResponse>
+}
+
+/**
+ * Invoke the middleware chain and at the end call the remote endpoint
+ */
+async function callWithMiddlewares (
+  req: RemoteRequest,
+  middlewares: HttpLancedbClientMiddleware[],
+  opts?: MiddlewareInvocationOptions
+): Promise<RemoteResponse> {
+  async function call (
+    i: number,
+    req: RemoteRequest
+  ): Promise<RemoteResponse> {
+    // if we have reached the end of the middleware chain, make the request
+    if (i > middlewares.length) {
+      const headers = Object.fromEntries(req.headers.entries())
+      const params = Object.fromEntries(req.params?.entries() ?? [])
+      const timeout = 10000
+      let res
+      if (req.method === Method.POST) {
+        res = await axios.post(
+          req.uri,
+          req.body,
+          {
+            headers,
+            params,
+            timeout,
+            responseType: opts?.responseType
+          }
+        )
+      } else {
+        res = await axios.get(
+          req.uri,
+          {
+            headers,
+            params,
+            timeout
+          }
+        )
+      }
+
+      return toLanceRes(res)
+    }
+
+    // call next middleware in chain
+    return await middlewares[i - 1].onRemoteRequest(
+      req,
+      async (req) => {
+        return await call(i + 1, req)
+      }
+    )
+  }
+
+  return await call(1, req)
+}
+
+interface MiddlewareInvocationOptions {
+  responseType?: ResponseType
+}
+
+/**
+ * Marshall the library response into a LanceDB response
+ */
+function toLanceRes (res: AxiosResponse): RemoteResponse {
+  const headers = new Map()
+  for (const h in res.headers) {
+    headers.set(h, res.headers[h])
+  }
+
+  return {
+    status: res.status,
+    statusText: res.statusText,
+    headers,
+    body: async () => {
+      return res.data
+    }
+  }
+}
+
 export class HttpLancedbClient {
   private readonly _url: string
   private readonly _apiKey: () => string
+  private readonly _middlewares: HttpLancedbClientMiddleware[]
 
   public constructor (
     url: string,
@@ -27,6 +115,7 @@ export class HttpLancedbClient {
   ) {
     this._url = url
     this._apiKey = () => apiKey
+    this._middlewares = []
   }
 
   get uri (): string {
@@ -43,8 +132,8 @@ export class HttpLancedbClient {
     columns?: string[],
     filter?: string
   ): Promise<ArrowTable<any>> {
-    const response = await axios.post(
-              `${this._url}/v1/table/${tableName}/query/`,
+    const result = await this.post(
+      `/v1/table/${tableName}/query/`,
       {
         vector,
         k,
@@ -54,63 +143,50 @@ export class HttpLancedbClient {
         filter,
         prefilter
       },
-              {
-                headers: {
-                  'Content-Type': 'application/json',
-                  'x-api-key': this._apiKey(),
-                  ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-                },
-                responseType: 'arraybuffer',
-                timeout: 10000
-              }
-    ).catch((err) => {
-      console.error('error: ', err)
-      if (err.response === undefined) {
-        throw new Error(`Network Error: ${err.message as string}`)
-      }
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
+      undefined,
+      undefined,
+      'arraybuffer'
     )
-    }
-
-    const table = tableFromIPC(response.data)
+    const table = tableFromIPC(await result.body())
     return table
   }
 
   /**
    * Sent GET request.
    */
-  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
-    const response = await axios.get(
-      `${this._url}${path}`,
-      {
-        headers: {
+  public async get (path: string, params?: Record<string, string>): Promise<RemoteResponse> {
+    const req = {
+      uri: `${this._url}${path}`,
+      method: Method.GET,
+      headers: new Map(Object.entries({
         'Content-Type': 'application/json',
         'x-api-key': this._apiKey(),
         ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-        },
-        params,
-        timeout: 10000
+      })),
+      params: new Map(Object.entries(params ?? {}))
     }
-    ).catch((err) => {
+
+    let response
+    try {
+      response = await callWithMiddlewares(req, this._middlewares)
+      return response
+    } catch (err: any) {
       console.error('error: ', err)
       if (err.response === undefined) {
         throw new Error(`Network Error: ${err.message as string}`)
       }
-      return err.response
-    })
+
+      response = toLanceRes(err.response)
+    }
+
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
+      const errorData = new TextDecoder().decode(await response.body())
       throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
+        `Server Error, status: ${response.status}, ` +
+        `message: ${response.statusText}: ${errorData}`
       )
     }
+
     return response
   }
 
@@ -120,35 +196,65 @@ export class HttpLancedbClient {
   public async post (
     path: string,
     data?: any,
-    params?: Record<string, string | number>,
-    content?: string | undefined
-  ): Promise<AxiosResponse> {
-    const response = await axios.post(
-        `${this._url}${path}`,
-        data,
-        {
-          headers: {
+    params?: Record<string, string>,
+    content?: string | undefined,
+    responseType?: ResponseType | undefined
+  ): Promise<RemoteResponse> {
+    const req = {
+      uri: `${this._url}${path}`,
+      method: Method.POST,
+      headers: new Map(Object.entries({
         'Content-Type': content ?? 'application/json',
         'x-api-key': this._apiKey(),
         ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-          },
-          params,
-          timeout: 30000
+      })),
+      params: new Map(Object.entries(params ?? {})),
+      body: data
     }
-    ).catch((err) => {
+
+    let response
+    try {
+      response = await callWithMiddlewares(req, this._middlewares, { responseType })
+
+      // return response
+    } catch (err: any) {
       console.error('error: ', err)
       if (err.response === undefined) {
         throw new Error(`Network Error: ${err.message as string}`)
       }
-      return err.response
-    })
+      response = toLanceRes(err.response)
+    }
+
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
+      const errorData = new TextDecoder().decode(await response.body())
       throw new Error(
-          `Server Error, status: ${response.status as number}, ` +
-          `message: ${response.statusText as string}: ${errorData}`
+        `Server Error, status: ${response.status}, ` +
+        `message: ${response.statusText}: ${errorData}`
       )
     }
+
     return response
   }
+
+  /**
+   * Instrument this client with middleware
+   * @param mw - The middleware that instruments the client
+   * @returns - an instance of this client instrumented with the middleware
+   */
+  public withMiddleware (mw: HttpLancedbClientMiddleware): HttpLancedbClient {
+    const wrapped = this.clone()
+    wrapped._middlewares.push(mw)
+    return wrapped
+  }
+
+  /**
+   * Make a clone of this client
+   */
+  private clone (): HttpLancedbClient {
+    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._dbName)
+    for (const mw of this._middlewares) {
+      clone._middlewares.push(mw)
+    }
+    return clone
+  }
 }

node/src/remote/index.ts

@@ -39,12 +39,13 @@ import {
   fromTableToStreamBuffer
 } from '../arrow'
 import { toSQL } from '../util'
+import { type HttpMiddleware } from '../middleware'
 
 /**
  * Remote connection.
  */
 export class RemoteConnection implements Connection {
-  private readonly _client: HttpLancedbClient
+  private _client: HttpLancedbClient
   private readonly _dbName: string
 
   constructor (opts: ConnectionOptions) {
@@ -84,10 +85,11 @@ export class RemoteConnection implements Connection {
     limit: number = 10
   ): Promise<string[]> {
     const response = await this._client.get('/v1/table/', {
-      limit,
+      limit: `${limit}`,
       page_token: pageToken
     })
-    return response.data.tables
+    const body = await response.body()
+    return body.tables
   }
 
   async openTable (name: string): Promise<Table>
@@ -163,7 +165,7 @@ export class RemoteConnection implements Connection {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
 
@@ -177,6 +179,17 @@ export class RemoteConnection implements Connection {
   async dropTable (name: string): Promise<void> {
     await this._client.post(`/v1/table/${name}/drop/`)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Connection {
+    const wrapped = this.clone()
+    wrapped._client = wrapped._client.withMiddleware(middleware)
+    return wrapped
+  }
+
+  private clone (): RemoteConnection {
+    const clone: RemoteConnection = Object.create(RemoteConnection.prototype)
+    return Object.assign(clone, this)
+  }
 }
 
 export class RemoteQuery<T = number[]> extends Query<T> {
@@ -229,7 +242,7 @@ export class RemoteQuery<T = number[]> extends Query<T> {
 // we are using extend until we have next next version release
 // Table and Connection has both been refactored to interfaces
 export class RemoteTable<T = number[]> implements Table<T> {
-  private readonly _client: HttpLancedbClient
+  private _client: HttpLancedbClient
   private readonly _embeddings?: EmbeddingFunction<T>
   private readonly _name: string
 
@@ -256,15 +269,15 @@ export class RemoteTable<T = number[]> implements Table<T> {
   get schema (): Promise<any> {
     return this._client
       .post(`/v1/table/${this._name}/describe/`)
-      .then((res) => {
+      .then(async (res) => {
         if (res.status !== 200) {
           throw new Error(
             `Server Error, status: ${res.status}, ` +
               // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-              `message: ${res.statusText}: ${res.data}`
+              `message: ${res.statusText}: ${await res.body()}`
           )
         }
-        return res.data?.schema
+        return (await res.body())?.schema
       })
   }
 
@@ -320,7 +333,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
@@ -346,7 +359,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
     return tbl.numRows
@@ -372,7 +385,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
     return tbl.numRows
@@ -415,7 +428,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
@@ -436,14 +449,14 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
 
   async countRows (): Promise<number> {
     const result = await this._client.post(`/v1/table/${this._name}/describe/`)
-    return result.data?.stats?.num_rows
+    return (await result.body())?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {
@@ -476,7 +489,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const results = await this._client.post(
       `/v1/table/${this._name}/index/list/`
     )
-    return results.data.indexes?.map((index: any) => ({
+    return (await results.body()).indexes?.map((index: any) => ({
       columns: index.columns,
       name: index.index_name,
       uuid: index.index_uuid
@@ -487,9 +500,10 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const results = await this._client.post(
       `/v1/table/${this._name}/index/${indexUuid}/stats/`
     )
+    const body = await results.body()
     return {
-      numIndexedRows: results.data.num_indexed_rows,
-      numUnindexedRows: results.data.num_unindexed_rows
+      numIndexedRows: body?.num_indexed_rows,
+      numUnindexedRows: body?.num_unindexed_rows
     }
   }
 
@@ -504,4 +518,15 @@ export class RemoteTable<T = number[]> implements Table<T> {
   async dropColumns (columnNames: string[]): Promise<void> {
     throw new Error('Drop columns is not yet supported in LanceDB Cloud.')
   }
+
+  withMiddleware(middleware: HttpMiddleware): Table<T> {
+    const wrapped = this.clone()
+    wrapped._client = wrapped._client.withMiddleware(middleware)
+    return wrapped
+  }
+
+  private clone (): RemoteTable<T> {
+    const clone: RemoteTable<T> = Object.create(RemoteTable.prototype)
+    return Object.assign(clone, this)
+  }
 }

node/src/sanitize.ts

@@ -79,7 +79,7 @@ import {
 import type { IntBitWidth, TimeBitWidth } from "apache-arrow/type";
 
 function sanitizeMetadata(
-  metadataLike?: unknown
+  metadataLike?: unknown,
 ): Map<string, string> | undefined {
   if (metadataLike === undefined || metadataLike === null) {
     return undefined;
@@ -90,7 +90,7 @@ function sanitizeMetadata(
   for (const item of metadataLike) {
     if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
       throw Error(
-        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values"
+        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
       );
     }
   }
@@ -105,7 +105,7 @@ function sanitizeInt(typeLike: object) {
     typeof typeLike.isSigned !== "boolean"
   ) {
     throw Error(
-      "Expected an Int Type to have a `bitWidth` and `isSigned` property"
+      "Expected an Int Type to have a `bitWidth` and `isSigned` property",
     );
   }
   return new Int(typeLike.isSigned, typeLike.bitWidth as IntBitWidth);
@@ -128,7 +128,7 @@ function sanitizeDecimal(typeLike: object) {
     typeof typeLike.bitWidth !== "number"
   ) {
     throw Error(
-      "Expected a Decimal Type to have `scale`, `precision`, and `bitWidth` properties"
+      "Expected a Decimal Type to have `scale`, `precision`, and `bitWidth` properties",
     );
   }
   return new Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
@@ -149,7 +149,7 @@ function sanitizeTime(typeLike: object) {
     typeof typeLike.bitWidth !== "number"
   ) {
     throw Error(
-      "Expected a Time type to have `unit` and `bitWidth` properties"
+      "Expected a Time type to have `unit` and `bitWidth` properties",
     );
   }
   return new Time(typeLike.unit, typeLike.bitWidth as TimeBitWidth);
@@ -172,7 +172,7 @@ function sanitizeTypedTimestamp(
     | typeof TimestampNanosecond
     | typeof TimestampMicrosecond
     | typeof TimestampMillisecond
-    | typeof TimestampSecond
+    | typeof TimestampSecond,
 ) {
   let timezone = null;
   if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
@@ -191,7 +191,7 @@ function sanitizeInterval(typeLike: object) {
 function sanitizeList(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
-      "Expected a List type to have an array-like `children` property"
+      "Expected a List type to have an array-like `children` property",
     );
   }
   if (typeLike.children.length !== 1) {
@@ -203,7 +203,7 @@ function sanitizeList(typeLike: object) {
 function sanitizeStruct(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
-      "Expected a Struct type to have an array-like `children` property"
+      "Expected a Struct type to have an array-like `children` property",
     );
   }
   return new Struct(typeLike.children.map((child) => sanitizeField(child)));
@@ -216,47 +216,47 @@ function sanitizeUnion(typeLike: object) {
     typeof typeLike.mode !== "number"
   ) {
     throw Error(
-      "Expected a Union type to have `typeIds` and `mode` properties"
+      "Expected a Union type to have `typeIds` and `mode` properties",
     );
   }
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
-      "Expected a Union type to have an array-like `children` property"
+      "Expected a Union type to have an array-like `children` property",
     );
   }
 
   return new Union(
     typeLike.mode,
     typeLike.typeIds as any,
-    typeLike.children.map((child) => sanitizeField(child))
+    typeLike.children.map((child) => sanitizeField(child)),
   );
 }
 
 function sanitizeTypedUnion(
   typeLike: object,
-  UnionType: typeof DenseUnion | typeof SparseUnion
+  UnionType: typeof DenseUnion | typeof SparseUnion,
 ) {
   if (!("typeIds" in typeLike)) {
     throw Error(
-      "Expected a DenseUnion/SparseUnion type to have a `typeIds` property"
+      "Expected a DenseUnion/SparseUnion type to have a `typeIds` property",
     );
   }
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
-      "Expected a DenseUnion/SparseUnion type to have an array-like `children` property"
+      "Expected a DenseUnion/SparseUnion type to have an array-like `children` property",
     );
   }
 
   return new UnionType(
     typeLike.typeIds as any,
-    typeLike.children.map((child) => sanitizeField(child))
+    typeLike.children.map((child) => sanitizeField(child)),
   );
 }
 
 function sanitizeFixedSizeBinary(typeLike: object) {
   if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
     throw Error(
-      "Expected a FixedSizeBinary type to have a `byteWidth` property"
+      "Expected a FixedSizeBinary type to have a `byteWidth` property",
     );
   }
   return new FixedSizeBinary(typeLike.byteWidth);
@@ -268,7 +268,7 @@ function sanitizeFixedSizeList(typeLike: object) {
   }
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
-      "Expected a FixedSizeList type to have an array-like `children` property"
+      "Expected a FixedSizeList type to have an array-like `children` property",
     );
   }
   if (typeLike.children.length !== 1) {
@@ -276,14 +276,14 @@ function sanitizeFixedSizeList(typeLike: object) {
   }
   return new FixedSizeList(
     typeLike.listSize,
-    sanitizeField(typeLike.children[0])
+    sanitizeField(typeLike.children[0]),
   );
 }
 
 function sanitizeMap(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
-      "Expected a Map type to have an array-like `children` property"
+      "Expected a Map type to have an array-like `children` property",
     );
   }
   if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
@@ -291,7 +291,7 @@ function sanitizeMap(typeLike: object) {
   }
   return new Map_(
     typeLike.children.map((field) => sanitizeField(field)) as any,
-    typeLike.keysSorted
+    typeLike.keysSorted,
   );
 }
 
@@ -319,7 +319,7 @@ function sanitizeDictionary(typeLike: object) {
     sanitizeType(typeLike.dictionary),
     sanitizeType(typeLike.indices) as any,
     typeLike.id,
-    typeLike.isOrdered
+    typeLike.isOrdered,
   );
 }
 
@@ -454,7 +454,7 @@ function sanitizeField(fieldLike: unknown): Field {
     !("nullable" in fieldLike)
   ) {
     throw Error(
-      "The field passed in is missing a `type`/`name`/`nullable` property"
+      "The field passed in is missing a `type`/`name`/`nullable` property",
     );
   }
   const type = sanitizeType(fieldLike.type);
@@ -473,6 +473,13 @@ function sanitizeField(fieldLike: unknown): Field {
   return new Field(name, type, nullable, metadata);
 }
 
+/**
+ * Convert something schemaLike into a Schema instance
+ *
+ * This method is often needed even when the caller is using a Schema
+ * instance because they might be using a different instance of apache-arrow
+ * than lancedb is using.
+ */
 export function sanitizeSchema(schemaLike: unknown): Schema {
   if (schemaLike instanceof Schema) {
     return schemaLike;
@@ -482,7 +489,7 @@ export function sanitizeSchema(schemaLike: unknown): Schema {
   }
   if (!("fields" in schemaLike)) {
     throw Error(
-      "The schema passed in does not appear to be a schema (no 'fields' property)"
+      "The schema passed in does not appear to be a schema (no 'fields' property)",
     );
   }
   let metadata;
@@ -491,11 +498,11 @@ export function sanitizeSchema(schemaLike: unknown): Schema {
   }
   if (!Array.isArray(schemaLike.fields)) {
     throw Error(
-      "The schema passed in had a 'fields' property but it was not an array"
+      "The schema passed in had a 'fields' property but it was not an array",
     );
   }
   const sanitizedFields = schemaLike.fields.map((field) =>
-    sanitizeField(field)
+    sanitizeField(field),
   );
   return new Schema(sanitizedFields, metadata);
 }

node/src/test/test.ts

@@ -128,6 +128,11 @@ describe('LanceDB client', function () {
       assertResults(results)
       results = await table.where('id % 2 = 0').execute()
       assertResults(results)
+
+      // Should reject a bad filter
+      await expect(table.filter('id % 2 = 0 AND').execute()).to.be.rejectedWith(
+        /.*sql parser error: Expected an expression:, found: EOF.*/
+      )
     })
 
     it('uses a filter / where clause', async function () {
@@ -283,7 +288,8 @@ describe('LanceDB client', function () {
 
     it('create a table from an Arrow Table', async function () {
       const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
+      // Also test the connect function with an object
+      const con = await lancedb.connect({ uri: dir })
 
       const i32s = new Int32Array(new Array<number>(10))
       const i32 = makeVector(i32s)
@@ -745,11 +751,11 @@ describe('LanceDB client', function () {
         num_sub_vectors: 2
       })
       await expect(createIndex).to.be.rejectedWith(
-        /VectorIndex requires the column data type to be fixed size list of float32s/
+        "index cannot be created on the column `name` which has data type Utf8"
       )
     })
 
-    it('it should fail when the column is not a vector', async function () {
+    it('it should fail when num_partitions is invalid', async function () {
       const uri = await createTestDB(32, 300)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')

nodejs/Cargo.toml

@@ -14,12 +14,10 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow-ipc.workspace = true
 futures.workspace = true
-lance-linalg.workspace = true
-lance.workspace = true
 lancedb = { path = "../rust/lancedb" }
 napi = { version = "2.15", default-features = false, features = [
     "napi7",
-    "async"
+    "async",
 ] }
 napi-derive = "2"
 

nodejs/__test__/table.test.ts

@@ -27,6 +27,7 @@ import {
   Float64,
 } from "apache-arrow";
 import { makeArrowTable } from "../dist/arrow";
+import { Index } from "../dist/indices";
 
 describe("Given a table", () => {
   let tmpDir: tmp.DirResult;
@@ -65,21 +66,36 @@ describe("Given a table", () => {
     expect(table.isOpen()).toBe(false);
     expect(table.countRows()).rejects.toThrow("Table some_table is closed");
   });
+
+  it("should let me update values", async () => {
+    await table.add([{ id: 1 }]);
+    expect(await table.countRows("id == 1")).toBe(1);
+    expect(await table.countRows("id == 7")).toBe(0);
+    await table.update({ id: "7" });
+    expect(await table.countRows("id == 1")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    await table.add([{ id: 2 }]);
+    // Test Map as input
+    await table.update(new Map(Object.entries({ id: "10" })), {
+      where: "id % 2 == 0",
+    });
+    expect(await table.countRows("id == 2")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    expect(await table.countRows("id == 10")).toBe(1);
+  });
 });
 
-describe("Test creating index", () => {
+describe("When creating an index", () => {
   let tmpDir: tmp.DirResult;
   const schema = new Schema([
     new Field("id", new Int32(), true),
     new Field("vec", new FixedSizeList(32, new Field("item", new Float32()))),
   ]);
+  let tbl: Table;
+  let queryVec: number[];
 
-  beforeEach(() => {
+  beforeEach(async () => {
     tmpDir = tmp.dirSync({ unsafeCleanup: true });
-  });
-  afterEach(() => tmpDir.removeCallback());
-
-  test("create vector index with no column", async () => {
     const db = await connect(tmpDir.name);
     const data = makeArrowTable(
       Array(300)
@@ -94,47 +110,80 @@ describe("Test creating index", () => {
         schema,
       },
     );
-    const tbl = await db.createTable("test", data);
-    await tbl.createIndex().build();
+    queryVec = data.toArray()[5].vec.toJSON();
+    tbl = await db.createTable("test", data);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should create a vector index on vector columns", async () => {
+    await tbl.createIndex("vec");
 
     // check index directory
     const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
     expect(fs.readdirSync(indexDir)).toHaveLength(1);
-    // TODO: check index type.
+    const indices = await tbl.listIndices();
+    expect(indices.length).toBe(1);
+    expect(indices[0]).toEqual({
+      indexType: "IvfPq",
+      columns: ["vec"],
+    });
 
     // Search without specifying the column
-    const queryVector = data.toArray()[5].vec.toJSON();
-    const rst = await tbl.query().nearestTo(queryVector).limit(2).toArrow();
+    let rst = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(queryVec)
+      .distanceType("DoT")
+      .toArrow();
+    expect(rst.numRows).toBe(2);
+
+    // Search using `vectorSearch`
+    rst = await tbl.vectorSearch(queryVec).limit(2).toArrow();
     expect(rst.numRows).toBe(2);
 
     // Search with specifying the column
-    const rst2 = await tbl.search(queryVector, "vec").limit(2).toArrow();
+    const rst2 = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(queryVec)
+      .column("vec")
+      .toArrow();
     expect(rst2.numRows).toBe(2);
     expect(rst.toString()).toEqual(rst2.toString());
   });
 
-  test("no vector column available", async () => {
-    const db = await connect(tmpDir.name);
-    const tbl = await db.createTable(
-      "no_vec",
-      makeArrowTable([
-        { id: 1, val: 2 },
-        { id: 2, val: 3 },
-      ]),
-    );
-    await expect(tbl.createIndex().build()).rejects.toThrow(
-      "No vector column found",
-    );
+  it("should allow parameters to be specified", async () => {
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({
+        numPartitions: 10,
+      }),
+    });
 
-    await tbl.createIndex("val").build();
-    const indexDir = path.join(tmpDir.name, "no_vec.lance", "_indices");
+    // TODO: Verify parameters when we can load index config as part of list indices
+  });
+
+  it("should allow me to replace (or not) an existing index", async () => {
+    await tbl.createIndex("id");
+    // Default is replace=true
+    await tbl.createIndex("id");
+    await expect(tbl.createIndex("id", { replace: false })).rejects.toThrow(
+      "already exists",
+    );
+    await tbl.createIndex("id", { replace: true });
+  });
+
+  test("should create a scalar index on scalar columns", async () => {
+    await tbl.createIndex("id");
+    const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
     expect(fs.readdirSync(indexDir)).toHaveLength(1);
 
-    for await (const r of tbl.query().filter("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(1);
+    for await (const r of tbl.query().where("id > 1").select(["id"])) {
+      expect(r.numRows).toBe(298);
     }
   });
 
+  // TODO: Move this test to the query API test (making sure we can reject queries
+  // when the dimension is incorrect)
   test("two columns with different dimensions", async () => {
     const db = await connect(tmpDir.name);
     const schema = new Schema([
@@ -164,71 +213,48 @@ describe("Test creating index", () => {
     );
 
     // Only build index over v1
-    await expect(tbl.createIndex().build()).rejects.toThrow(
-      /.*More than one vector columns found.*/,
-    );
-    tbl
-      .createIndex("vec")
-      // eslint-disable-next-line @typescript-eslint/naming-convention
-      .ivf_pq({ num_partitions: 2, num_sub_vectors: 2 })
-      .build();
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({ numPartitions: 2, numSubVectors: 2 }),
+    });
 
     const rst = await tbl
       .query()
+      .limit(2)
       .nearestTo(
         Array(32)
           .fill(1)
           .map(() => Math.random()),
       )
-      .limit(2)
       .toArrow();
     expect(rst.numRows).toBe(2);
 
     // Search with specifying the column
     await expect(
       tbl
-        .search(
+        .query()
+        .limit(2)
+        .nearestTo(
           Array(64)
             .fill(1)
             .map(() => Math.random()),
-          "vec",
         )
-        .limit(2)
+        .column("vec")
         .toArrow(),
-    ).rejects.toThrow(/.*does not match the dimension.*/);
+    ).rejects.toThrow(/.* query dim=64, expected vector dim=32.*/);
 
     const query64 = Array(64)
       .fill(1)
       .map(() => Math.random());
-    const rst64Query = await tbl.query().nearestTo(query64).limit(2).toArrow();
-    const rst64Search = await tbl.search(query64, "vec2").limit(2).toArrow();
+    const rst64Query = await tbl.query().limit(2).nearestTo(query64).toArrow();
+    const rst64Search = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(query64)
+      .column("vec2")
+      .toArrow();
     expect(rst64Query.toString()).toEqual(rst64Search.toString());
     expect(rst64Query.numRows).toBe(2);
   });
-
-  test("create scalar index", async () => {
-    const db = await connect(tmpDir.name);
-    const data = makeArrowTable(
-      Array(300)
-        .fill(1)
-        .map((_, i) => ({
-          id: i,
-          vec: Array(32)
-            .fill(1)
-            .map(() => Math.random()),
-        })),
-      {
-        schema,
-      },
-    );
-    const tbl = await db.createTable("test", data);
-    await tbl.createIndex("id").build();
-
-    // check index directory
-    const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
-    expect(fs.readdirSync(indexDir)).toHaveLength(1);
-    // TODO: check index type.
-  });
 });
 
 describe("Read consistency interval", () => {
@@ -348,3 +374,48 @@ describe("schema evolution", function () {
     expect(await table.schema()).toEqual(expectedSchema);
   });
 });
+
+describe("when dealing with versioning", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("can travel in time", async () => {
+    // Setup
+    const con = await connect(tmpDir.name);
+    const table = await con.createTable("vectors", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    const version = await table.version();
+    await table.add([{ id: 2n, vector: [0.1, 0.2] }]);
+    expect(await table.countRows()).toBe(2);
+    // Make sure we can rewind
+    await table.checkout(version);
+    expect(await table.countRows()).toBe(1);
+    // Can't add data in time travel mode
+    await expect(table.add([{ id: 3n, vector: [0.1, 0.2] }])).rejects.toThrow(
+      "table cannot be modified when a specific version is checked out",
+    );
+    // Can go back to normal mode
+    await table.checkoutLatest();
+    expect(await table.countRows()).toBe(2);
+    // Should be able to add data again
+    await table.add([{ id: 2n, vector: [0.1, 0.2] }]);
+    expect(await table.countRows()).toBe(3);
+    // Now checkout and restore
+    await table.checkout(version);
+    await table.restore();
+    expect(await table.countRows()).toBe(1);
+    // Should be able to add data
+    await table.add([{ id: 2n, vector: [0.1, 0.2] }]);
+    expect(await table.countRows()).toBe(2);
+    // Can't use restore if not checked out
+    await expect(table.restore()).rejects.toThrow(
+      "checkout before running restore",
+    );
+  });
+});

nodejs/eslint.config.js

@@ -4,14 +4,25 @@
 const eslint = require("@eslint/js");
 const tseslint = require("typescript-eslint");
 const eslintConfigPrettier = require("eslint-config-prettier");
+const jsdoc = require("eslint-plugin-jsdoc");
 
 module.exports = tseslint.config(
   eslint.configs.recommended,
+  jsdoc.configs["flat/recommended"],
   eslintConfigPrettier,
   ...tseslint.configs.recommended,
   {
     rules: {
       "@typescript-eslint/naming-convention": "error",
+      "jsdoc/require-returns": "off",
+      "jsdoc/require-param": "off",
+      "jsdoc/require-jsdoc": [
+        "error",
+        {
+          publicOnly: true,
         },
+      ],
+    },
+    plugins: jsdoc,
   },
 );

nodejs/lancedb/arrow.ts

@@ -31,6 +31,7 @@ import {
   DataType,
   Binary,
   Float32,
+  type makeTable,
 } from "apache-arrow";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
 import { sanitizeSchema } from "./sanitize";
@@ -105,6 +106,9 @@ export class MakeArrowTableOptions {
  * An enhanced version of the {@link makeTable} function from Apache Arrow
  * that supports nested fields and embeddings columns.
  *
+ * (typically you do not need to call this function.  It will be called automatically
+ * when creating a table or adding data to it)
+ *
  * This function converts an array of Record<String, any> (row-major JS objects)
  * to an Arrow Table (a columnar structure)
  *
@@ -128,14 +132,7 @@ export class MakeArrowTableOptions {
  *  - Buffer => Binary
  *  - Record<String, any> => Struct
  *  - Array<any> => List
- *
- * @param data input data
- * @param options options to control the makeArrowTable call.
- *
  * @example
- *
- * ```ts
- *
  * import { fromTableToBuffer, makeArrowTable } from "../arrow";
  * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
  *
@@ -307,7 +304,9 @@ export function makeEmptyTable(schema: Schema): ArrowTable {
   return makeArrowTable([], { schema });
 }
 
-// Helper function to convert Array<Array<any>> to a variable sized list array
+/**
+ * Helper function to convert Array<Array<any>> to a variable sized list array
+ */
 // @ts-expect-error (Vector<unknown> is not assignable to Vector<any>)
 function makeListVector(lists: unknown[][]): Vector<unknown> {
   if (lists.length === 0 || lists[0].length === 0) {
@@ -333,7 +332,7 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
   return listBuilder.finish().toVector();
 }
 
-// Helper function to convert an Array of JS values to an Arrow Vector
+/** Helper function to convert an Array of JS values to an Arrow Vector */
 function makeVector(
   values: unknown[],
   type?: DataType,
@@ -374,6 +373,7 @@ function makeVector(
   }
 }
 
+/** Helper function to apply embeddings to an input table */
 async function applyEmbeddings<T>(
   table: ArrowTable,
   embeddings?: EmbeddingFunction<T>,
@@ -466,7 +466,7 @@ async function applyEmbeddings<T>(
   return newTable;
 }
 
-/*
+/**
  * Convert an Array of records into an Arrow Table, optionally applying an
  * embeddings function to it.
  *
@@ -493,7 +493,7 @@ export async function convertToTable<T>(
   return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
 }
 
-// Creates the Arrow Type for a Vector column with dimension `dim`
+/** Creates the Arrow Type for a Vector column with dimension `dim` */
 function newVectorType<T extends Float>(
   dim: number,
   innerType: T,
@@ -565,6 +565,14 @@ export async function fromTableToBuffer<T>(
   return Buffer.from(await writer.toUint8Array());
 }
 
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
 export async function fromDataToBuffer<T>(
   data: Data,
   embeddings?: EmbeddingFunction<T>,
@@ -599,6 +607,9 @@ export async function fromTableToStreamBuffer<T>(
   return Buffer.from(await writer.toUint8Array());
 }
 
+/**
+ * Reorder the columns in `batch` so that they agree with the field order in `schema`
+ */
 function alignBatch(batch: RecordBatch, schema: Schema): RecordBatch {
   const alignedChildren = [];
   for (const field of schema.fields) {
@@ -621,6 +632,9 @@ function alignBatch(batch: RecordBatch, schema: Schema): RecordBatch {
   return new RecordBatch(schema, newData);
 }
 
+/**
+ * Reorder the columns in `table` so that they agree with the field order in `schema`
+ */
 function alignTable(table: ArrowTable, schema: Schema): ArrowTable {
   const alignedBatches = table.batches.map((batch) =>
     alignBatch(batch, schema),
@@ -628,7 +642,9 @@ function alignTable(table: ArrowTable, schema: Schema): ArrowTable {
   return new ArrowTable(schema, alignedBatches);
 }
 
-// Creates an empty Arrow Table
+/**
+ * Create an empty table with the given schema
+ */
 export function createEmptyTable(schema: Schema): ArrowTable {
   return new ArrowTable(sanitizeSchema(schema));
 }

nodejs/lancedb/connection.ts

@@ -78,7 +78,8 @@ export class Connection {
     return this.inner.isOpen();
   }
 
-  /** Close the connection, releasing any underlying resources.
+  /**
+   * Close the connection, releasing any underlying resources.
    *
    * It is safe to call this method multiple times.
    *
@@ -93,11 +94,12 @@ export class Connection {
     return this.inner.display();
   }
 
-  /** List all the table names in this database.
+  /**
+   * List all the table names in this database.
    *
    * Tables will be returned in lexicographical order.
-   *
-   * @param options Optional parameters to control the listing.
+   * @param {Partial<TableNamesOptions>} options - options to control the
+   * paging / start point
    */
   async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
     return this.inner.tableNames(options?.startAfter, options?.limit);
@@ -105,9 +107,7 @@ export class Connection {
 
   /**
    * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this table
+   * @param {string} name - The name of the table
    */
   async openTable(name: string): Promise<Table> {
     const innerTable = await this.inner.openTable(name);
@@ -116,9 +116,9 @@ export class Connection {
 
   /**
    * Creates a new Table and initialize it with new data.
-   *
    * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {Record<string, unknown>[] | ArrowTable} data - Non-empty Array of Records
+   * to be inserted into the table
    */
   async createTable(
     name: string,
@@ -145,9 +145,8 @@ export class Connection {
 
   /**
    * Creates a new empty Table
-   *
    * @param {string} name - The name of the table.
-   * @param schema - The schema of the table
+   * @param {Schema} schema - The schema of the table
    */
   async createEmptyTable(
     name: string,
@@ -169,7 +168,7 @@ export class Connection {
 
   /**
    * Drop an existing table.
-   * @param name The name of the table to drop.
+   * @param {string} name The name of the table to drop.
    */
   async dropTable(name: string): Promise<void> {
     return this.inner.dropTable(name);

nodejs/lancedb/embedding/embedding_function.ts

@@ -62,6 +62,7 @@ export interface EmbeddingFunction<T> {
   embed: (data: T[]) => Promise<number[][]>;
 }
 
+/** Test if the input seems to be an embedding function */
 export function isEmbeddingFunction<T>(
   value: unknown,
 ): value is EmbeddingFunction<T> {

nodejs/lancedb/embedding/index.ts

@@ -0,0 +1,2 @@
+export { EmbeddingFunction, isEmbeddingFunction } from "./embedding_function";
+export { OpenAIEmbeddingFunction } from "./openai";

nodejs/lancedb/index.ts

@@ -19,14 +19,33 @@ import {
 } from "./native.js";
 
 export {
-  ConnectionOptions,
   WriteOptions,
-  Query,
-  MetricType,
+  WriteMode,
+  AddColumnsSql,
+  ColumnAlteration,
+  ConnectionOptions,
 } from "./native.js";
-export { Connection } from "./connection";
-export { Table } from "./table";
-export { IvfPQOptions, IndexBuilder } from "./indexer";
+export {
+  makeArrowTable,
+  MakeArrowTableOptions,
+  Data,
+  VectorColumnOptions,
+} from "./arrow";
+export {
+  Connection,
+  CreateTableOptions,
+  TableNamesOptions,
+} from "./connection";
+export {
+  ExecutableQuery,
+  Query,
+  QueryBase,
+  VectorQuery,
+  RecordBatchIterator,
+} from "./query";
+export { Index, IndexOptions, IvfPqOptions } from "./indices";
+export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
+export * as embedding from "./embedding";
 
 /**
  * Connect to a LanceDB instance at the given URI.
@@ -36,9 +55,8 @@ export { IvfPQOptions, IndexBuilder } from "./indexer";
  * - `/path/to/database` - local database
  * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
  * - `db://host:port` - remote database (LanceDB cloud)
- *
- * @param uri The uri of the database. If the database uri starts with `db://` then it connects to a remote database.
- *
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
  * @see {@link ConnectionOptions} for more details on the URI format.
  */
 export async function connect(

nodejs/lancedb/indexer.ts

@@ -1,105 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// TODO: Re-enable this as part of https://github.com/lancedb/lancedb/pull/1052
-/* eslint-disable @typescript-eslint/naming-convention */
-
-import {
-  MetricType,
-  IndexBuilder as NativeBuilder,
-  Table as NativeTable,
-} from "./native";
-
-/** Options to create `IVF_PQ` index */
-export interface IvfPQOptions {
-  /** Number of IVF partitions. */
-  num_partitions?: number;
-
-  /** Number of sub-vectors in PQ coding. */
-  num_sub_vectors?: number;
-
-  /** Number of bits used for each PQ code.
-   */
-  num_bits?: number;
-
-  /** Metric type to calculate the distance between vectors.
-   *
-   * Supported metrics: `L2`, `Cosine` and `Dot`.
-   */
-  metric_type?: MetricType;
-
-  /** Number of iterations to train K-means.
-   *
-   * Default is 50. The more iterations it usually yield better results,
-   * but it takes longer to train.
-   */
-  max_iterations?: number;
-
-  sample_rate?: number;
-}
-
-/**
- * Building an index on LanceDB {@link Table}
- *
- * @see {@link Table.createIndex} for detailed usage.
- */
-export class IndexBuilder {
-  private inner: NativeBuilder;
-
-  constructor(tbl: NativeTable) {
-    this.inner = tbl.createIndex();
-  }
-
-  /** Instruct the builder to build an `IVF_PQ` index */
-  ivf_pq(options?: IvfPQOptions): IndexBuilder {
-    this.inner.ivfPq(
-      options?.metric_type,
-      options?.num_partitions,
-      options?.num_sub_vectors,
-      options?.num_bits,
-      options?.max_iterations,
-      options?.sample_rate,
-    );
-    return this;
-  }
-
-  /** Instruct the builder to build a Scalar index. */
-  scalar(): IndexBuilder {
-    this.scalar();
-    return this;
-  }
-
-  /** Set the column(s) to create index on top of. */
-  column(col: string): IndexBuilder {
-    this.inner.column(col);
-    return this;
-  }
-
-  /** Set to true to replace existing index. */
-  replace(val: boolean): IndexBuilder {
-    this.inner.replace(val);
-    return this;
-  }
-
-  /** Specify the name of the index. Optional */
-  name(n: string): IndexBuilder {
-    this.inner.name(n);
-    return this;
-  }
-
-  /** Building the index. */
-  async build() {
-    await this.inner.build();
-  }
-}

nodejs/lancedb/indices.ts

@@ -0,0 +1,203 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Index as LanceDbIndex } from "./native";
+
+/**
+ * Options to create an `IVF_PQ` index
+ */
+export interface IvfPqOptions {
+  /**
+   * The number of IVF partitions to create.
+   *
+   * This value should generally scale with the number of rows in the dataset.
+   * By default the number of partitions is the square root of the number of
+   * rows.
+   *
+   * If this value is too large then the first part of the search (picking the
+   * right partition) will be slow.  If this value is too small then the second
+   * part of the search (searching within a partition) will be slow.
+   */
+  numPartitions?: number;
+
+  /**
+   * Number of sub-vectors of PQ.
+   *
+   * This value controls how much the vector is compressed during the quantization step.
+   * The more sub vectors there are the less the vector is compressed.  The default is
+   * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+   * by 16 we use the dimension divded by 8.
+   *
+   * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+   * us to use efficient SIMD instructions.
+   *
+   * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+   * will likely result in poor performance.
+   */
+  numSubVectors?: number;
+
+  /**
+   * Distance type to use to build the index.
+   *
+   * Default value is "l2".
+   *
+   * This is used when training the index to calculate the IVF partitions
+   * (vectors are grouped in partitions with similar vectors according to this
+   * distance type) and to calculate a subvector's code during quantization.
+   *
+   * The distance type used to train an index MUST match the distance type used
+   * to search the index.  Failure to do so will yield inaccurate results.
+   *
+   * The following distance types are available:
+   *
+   * "l2" - Euclidean distance. This is a very common distance metric that
+   * accounts for both magnitude and direction when determining the distance
+   * between vectors. L2 distance has a range of [0, ∞).
+   *
+   * "cosine" - Cosine distance.  Cosine distance is a distance metric
+   * calculated from the cosine similarity between two vectors. Cosine
+   * similarity is a measure of similarity between two non-zero vectors of an
+   * inner product space. It is defined to equal the cosine of the angle
+   * between them.  Unlike L2, the cosine distance is not affected by the
+   * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+   *
+   * Note: the cosine distance is undefined when one (or both) of the vectors
+   * are all zeros (there is no direction).  These vectors are invalid and may
+   * never be returned from a vector search.
+   *
+   * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+   * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   */
+  distanceType?: "l2" | "cosine" | "dot";
+
+  /**
+   * Max iteration to train IVF kmeans.
+   *
+   * When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
+   * controls how many iterations of kmeans to run.
+   *
+   * Increasing this might improve the quality of the index but in most cases these extra
+   * iterations have diminishing returns.
+   *
+   * The default value is 50.
+   */
+  maxIterations?: number;
+
+  /**
+   * The number of vectors, per partition, to sample when training IVF kmeans.
+   *
+   * When an IVF PQ index is trained, we need to calculate partitions.  These are groups
+   * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+   *
+   * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+   * random sample of the data.  This parameter controls the size of the sample.  The total
+   * number of vectors used to train the index is `sample_rate * num_partitions`.
+   *
+   * Increasing this value might improve the quality of the index but in most cases the
+   * default should be sufficient.
+   *
+   * The default value is 256.
+   */
+  sampleRate?: number;
+}
+
+export class Index {
+  private readonly inner: LanceDbIndex;
+  private constructor(inner: LanceDbIndex) {
+    this.inner = inner;
+  }
+
+  /**
+   * Create an IvfPq index
+   *
+   * This index stores a compressed (quantized) copy of every vector.  These vectors
+   * are grouped into partitions of similar vectors.  Each partition keeps track of
+   * a centroid which is the average value of all vectors in the group.
+   *
+   * During a query the centroids are compared with the query vector to find the closest
+   * partitions.  The compressed vectors in these partitions are then searched to find
+   * the closest vectors.
+   *
+   * The compression scheme is called product quantization.  Each vector is divided into
+   * subvectors and then each subvector is quantized into a small number of bits.  the
+   * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+   * between index size (and thus search speed) and index accuracy.
+   *
+   * The partitioning process is called IVF and the `num_partitions` parameter controls how
+   * many groups to create.
+   *
+   * Note that training an IVF PQ index on a large dataset is a slow operation and
+   * currently is also a memory intensive operation.
+   */
+  static ivfPq(options?: Partial<IvfPqOptions>) {
+    return new Index(
+      LanceDbIndex.ivfPq(
+        options?.distanceType,
+        options?.numPartitions,
+        options?.numSubVectors,
+        options?.maxIterations,
+        options?.sampleRate,
+      ),
+    );
+  }
+
+  /**
+   * Create a btree index
+   *
+   * A btree index is an index on a scalar columns.  The index stores a copy of the column
+   * in sorted order.  A header entry is created for each block of rows (currently the
+   * block size is fixed at 4096).  These header entries are stored in a separate
+   * cacheable structure (a btree).  To search for data the header is used to determine
+   * which blocks need to be read from disk.
+   *
+   * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+   * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+   * the correct row ids.
+   *
+   * This index is good for scalar columns with mostly distinct values and does best when
+   * the query is highly selective.
+   *
+   * The btree index does not currently have any parameters though parameters such as the
+   * block size may be added in the future.
+   */
+  static btree() {
+    return new Index(LanceDbIndex.btree());
+  }
+}
+
+export interface IndexOptions {
+  /**
+   * Advanced index configuration
+   *
+   * This option allows you to specify a specfic index to create and also
+   * allows you to pass in configuration for training the index.
+   *
+   * See the static methods on Index for details on the various index types.
+   *
+   * If this is not supplied then column data type(s) and column statistics
+   * will be used to determine the most useful kind of index to create.
+   */
+  config?: Index;
+  /**
+   * Whether to replace the existing index
+   *
+   * If this is false, and another index already exists on the same columns
+   * and the same name, then an error will be returned.  This is true even if
+   * that index is out of date.
+   *
+   * The default is true
+   */
+  replace?: boolean;
+}

nodejs/lancedb/native.d.ts

@@ -1,133 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-export const enum IndexType {
-  Scalar = 0,
-  IvfPq = 1
-}
-export const enum MetricType {
-  L2 = 0,
-  Cosine = 1,
-  Dot = 2
-}
-/**
- *  A definition of a column alteration. The alteration changes the column at
- * `path` to have the new name `name`, to be nullable if `nullable` is true,
- * and to have the data type `data_type`. At least one of `rename` or `nullable`
- * must be provided.
- */
-export interface ColumnAlteration {
-  /**
-   * The path to the column to alter. This is a dot-separated path to the column.
-   * If it is a top-level column then it is just the name of the column. If it is
-   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
-   * `c` nested inside a column `b` nested inside a column `a`.
-   */
-  path: string
-  /**
-   * The new name of the column. If not provided then the name will not be changed.
-   * This must be distinct from the names of all other columns in the table.
-   */
-  rename?: string
-  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
-  nullable?: boolean
-}
-/** A definition of a new column to add to a table. */
-export interface AddColumnsSql {
-  /** The name of the new column. */
-  name: string
-  /**
-   * The values to populate the new column with, as a SQL expression.
-   * The expression can reference other columns in the table.
-   */
-  valueSql: string
-}
-export interface ConnectionOptions {
-  apiKey?: string
-  hostOverride?: string
-  /**
-   * (For LanceDB OSS only): The interval, in seconds, at which to check for
-   * updates to the table from other processes. If None, then consistency is not
-   * checked. For performance reasons, this is the default. For strong
-   * consistency, set this to zero seconds. Then every read will check for
-   * updates from other processes. As a compromise, you can set this to a
-   * non-zero value for eventual consistency. If more than that interval
-   * has passed since the last check, then the table will be checked for updates.
-   * Note: this consistency only applies to read operations. Write operations are
-   * always consistent.
-   */
-  readConsistencyInterval?: number
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  mode?: WriteMode
-}
-export function connect(uri: string, options: ConnectionOptions): Promise<Connection>
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string, options: ConnectionOptions): Promise<Connection>
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** List all tables in the dataset. */
-  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer, mode: string): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string): Promise<Table>
-  openTable(name: string): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class IndexBuilder {
-  replace(v: boolean): void
-  column(c: string): void
-  name(name: string): void
-  ivfPq(metricType?: MetricType | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): void
-  scalar(): void
-  build(): Promise<void>
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-export class Query {
-  column(column: string): void
-  filter(filter: string): void
-  select(columns: Array<string>): void
-  limit(limit: number): void
-  prefilter(prefilter: boolean): void
-  nearestTo(vector: Float32Array): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  executeStream(): Promise<RecordBatchIterator>
-}
-export class Table {
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Promise<Buffer>
-  add(buf: Buffer, mode: string): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<number>
-  delete(predicate: string): Promise<void>
-  createIndex(): IndexBuilder
-  query(): Query
-  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
-  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
-  dropColumns(columns: Array<string>): Promise<void>
-}

nodejs/lancedb/native.js

@@ -1,308 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/* prettier-ignore */
-
-/* auto-generated by NAPI-RS */
-
-const { existsSync, readFileSync } = require('fs')
-const { join } = require('path')
-
-const { platform, arch } = process
-
-let nativeBinding = null
-let localFileExisted = false
-let loadError = null
-
-function isMusl() {
-  // For Node 10
-  if (!process.report || typeof process.report.getReport !== 'function') {
-    try {
-      const lddPath = require('child_process').execSync('which ldd').toString().trim()
-      return readFileSync(lddPath, 'utf8').includes('musl')
-    } catch (e) {
-      return true
-    }
-  } else {
-    const { glibcVersionRuntime } = process.report.getReport().header
-    return !glibcVersionRuntime
-  }
-}
-
-switch (platform) {
-  case 'android':
-    switch (arch) {
-      case 'arm64':
-        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.android-arm64.node'))
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.android-arm64.node')
-          } else {
-            nativeBinding = require('lancedb-android-arm64')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'arm':
-        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.android-arm-eabi.node'))
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.android-arm-eabi.node')
-          } else {
-            nativeBinding = require('lancedb-android-arm-eabi')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on Android ${arch}`)
-    }
-    break
-  case 'win32':
-    switch (arch) {
-      case 'x64':
-        localFileExisted = existsSync(
-          join(__dirname, 'lancedb-nodejs.win32-x64-msvc.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.win32-x64-msvc.node')
-          } else {
-            nativeBinding = require('lancedb-win32-x64-msvc')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'ia32':
-        localFileExisted = existsSync(
-          join(__dirname, 'lancedb-nodejs.win32-ia32-msvc.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.win32-ia32-msvc.node')
-          } else {
-            nativeBinding = require('lancedb-win32-ia32-msvc')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'arm64':
-        localFileExisted = existsSync(
-          join(__dirname, 'lancedb-nodejs.win32-arm64-msvc.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.win32-arm64-msvc.node')
-          } else {
-            nativeBinding = require('lancedb-win32-arm64-msvc')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on Windows: ${arch}`)
-    }
-    break
-  case 'darwin':
-    localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.darwin-universal.node'))
-    try {
-      if (localFileExisted) {
-        nativeBinding = require('./lancedb-nodejs.darwin-universal.node')
-      } else {
-        nativeBinding = require('lancedb-darwin-universal')
-      }
-      break
-    } catch {}
-    switch (arch) {
-      case 'x64':
-        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.darwin-x64.node'))
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.darwin-x64.node')
-          } else {
-            nativeBinding = require('lancedb-darwin-x64')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'arm64':
-        localFileExisted = existsSync(
-          join(__dirname, 'lancedb-nodejs.darwin-arm64.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.darwin-arm64.node')
-          } else {
-            nativeBinding = require('lancedb-darwin-arm64')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on macOS: ${arch}`)
-    }
-    break
-  case 'freebsd':
-    if (arch !== 'x64') {
-      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
-    }
-    localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.freebsd-x64.node'))
-    try {
-      if (localFileExisted) {
-        nativeBinding = require('./lancedb-nodejs.freebsd-x64.node')
-      } else {
-        nativeBinding = require('lancedb-freebsd-x64')
-      }
-    } catch (e) {
-      loadError = e
-    }
-    break
-  case 'linux':
-    switch (arch) {
-      case 'x64':
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, 'lancedb-nodejs.linux-x64-musl.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./lancedb-nodejs.linux-x64-musl.node')
-            } else {
-              nativeBinding = require('lancedb-linux-x64-musl')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, 'lancedb-nodejs.linux-x64-gnu.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./lancedb-nodejs.linux-x64-gnu.node')
-            } else {
-              nativeBinding = require('lancedb-linux-x64-gnu')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        }
-        break
-      case 'arm64':
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, 'lancedb-nodejs.linux-arm64-musl.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./lancedb-nodejs.linux-arm64-musl.node')
-            } else {
-              nativeBinding = require('lancedb-linux-arm64-musl')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, 'lancedb-nodejs.linux-arm64-gnu.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./lancedb-nodejs.linux-arm64-gnu.node')
-            } else {
-              nativeBinding = require('lancedb-linux-arm64-gnu')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        }
-        break
-      case 'arm':
-        localFileExisted = existsSync(
-          join(__dirname, 'lancedb-nodejs.linux-arm-gnueabihf.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.linux-arm-gnueabihf.node')
-          } else {
-            nativeBinding = require('lancedb-linux-arm-gnueabihf')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'riscv64':
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, 'lancedb-nodejs.linux-riscv64-musl.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./lancedb-nodejs.linux-riscv64-musl.node')
-            } else {
-              nativeBinding = require('lancedb-linux-riscv64-musl')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, 'lancedb-nodejs.linux-riscv64-gnu.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./lancedb-nodejs.linux-riscv64-gnu.node')
-            } else {
-              nativeBinding = require('lancedb-linux-riscv64-gnu')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        }
-        break
-      case 's390x':
-        localFileExisted = existsSync(
-          join(__dirname, 'lancedb-nodejs.linux-s390x-gnu.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./lancedb-nodejs.linux-s390x-gnu.node')
-          } else {
-            nativeBinding = require('lancedb-linux-s390x-gnu')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on Linux: ${arch}`)
-    }
-    break
-  default:
-    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
-}
-
-if (!nativeBinding) {
-  if (loadError) {
-    throw loadError
-  }
-  throw new Error(`Failed to load native binding`)
-}
-
-const { Connection, IndexType, MetricType, IndexBuilder, RecordBatchIterator, Query, Table, WriteMode, connect } = nativeBinding
-
-module.exports.Connection = Connection
-module.exports.IndexType = IndexType
-module.exports.MetricType = MetricType
-module.exports.IndexBuilder = IndexBuilder
-module.exports.RecordBatchIterator = RecordBatchIterator
-module.exports.Query = Query
-module.exports.Table = Table
-module.exports.WriteMode = WriteMode
-module.exports.connect = connect

nodejs/lancedb/query.ts

@@ -17,18 +17,15 @@ import {
   RecordBatchIterator as NativeBatchIterator,
   Query as NativeQuery,
   Table as NativeTable,
+  VectorQuery as NativeVectorQuery,
 } from "./native";
-
-class RecordBatchIterator implements AsyncIterator<RecordBatch> {
+import { type IvfPqOptions } from "./indices";
+export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
 
-  constructor(
-    inner?: NativeBatchIterator,
-    promise?: Promise<NativeBatchIterator>,
-  ) {
+  constructor(promise?: Promise<NativeBatchIterator>) {
     // TODO: check promise reliably so we dont need to pass two arguments.
-    this.inner = inner;
     this.promisedInner = promise;
   }
 
@@ -53,82 +50,113 @@ class RecordBatchIterator implements AsyncIterator<RecordBatch> {
 }
 /* eslint-enable */
 
-/** Query executor */
-export class Query implements AsyncIterable<RecordBatch> {
-  private readonly inner: NativeQuery;
+/** Common methods supported by all query types */
+export class QueryBase<
+  NativeQueryType extends NativeQuery | NativeVectorQuery,
+  QueryType,
+> implements AsyncIterable<RecordBatch>
+{
+  protected constructor(protected inner: NativeQueryType) {}
 
-  constructor(tbl: NativeTable) {
-    this.inner = tbl.query();
+  /**
+   * A filter statement to be applied to this query.
+   *
+   * The filter should be supplied as an SQL query string.  For example:
+   * @example
+   * x > 10
+   * y > 0 AND y < 100
+   * x > 5 OR y = 'test'
+   *
+   * Filtering performance can often be improved by creating a scalar index
+   * on the filter column(s).
+   */
+  where(predicate: string): QueryType {
+    this.inner.onlyIf(predicate);
+    return this as unknown as QueryType;
   }
 
-  /** Set the column to run query. */
-  column(column: string): Query {
-    this.inner.column(column);
-    return this;
+  /**
+   * Return only the specified columns.
+   *
+   * By default a query will return all columns from the table.  However, this can have
+   * a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+   * means we can finely tune our I/O to select exactly the columns we need.
+   *
+   * As a best practice you should always limit queries to the columns that you need.  If you
+   * pass in an array of column names then only those columns will be returned.
+   *
+   * You can also use this method to create new "dynamic" columns based on your existing columns.
+   * For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+   * seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+   *
+   * To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+   * for each entry in the map.  The key provides the name of the column.  The value is
+   * an SQL string used to specify how the column is calculated.
+   *
+   * For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+   * input to this method would be:
+   * @example
+   * new Map([["combined", "a + b"], ["c", "c"]])
+   *
+   * Columns will always be returned in the order given, even if that order is different than
+   * the order used when adding the data.
+   *
+   * Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+   * uses `Object.entries` which should preserve the insertion order of the object.  However,
+   * object insertion order is easy to get wrong and `Map` is more foolproof.
+   */
+  select(
+    columns: string[] | Map<string, string> | Record<string, string>,
+  ): QueryType {
+    let columnTuples: [string, string][];
+    if (Array.isArray(columns)) {
+      columnTuples = columns.map((c) => [c, c]);
+    } else if (columns instanceof Map) {
+      columnTuples = Array.from(columns.entries());
+    } else {
+      columnTuples = Object.entries(columns);
+    }
+    this.inner.select(columnTuples);
+    return this as unknown as QueryType;
   }
 
-  /** Set the filter predicate, only returns the results that satisfy the filter.
+  /**
+   * Set the maximum number of results to return.
+   *
+   * By default, a plain search has no limit.  If this method is not
+   * called then every valid row from the table will be returned.
+   */
+  limit(limit: number): QueryType {
+    this.inner.limit(limit);
+    return this as unknown as QueryType;
+  }
+
+  protected nativeExecute(): Promise<NativeBatchIterator> {
+    return this.inner.execute();
+  }
+
+  /**
+   * Execute the query and return the results as an @see {@link AsyncIterator}
+   * of @see {@link RecordBatch}.
+   *
+   * By default, LanceDb will use many threads to calculate results and, when
+   * the result set is large, multiple batches will be processed at one time.
+   * This readahead is limited however and backpressure will be applied if this
+   * stream is consumed slowly (this constrains the maximum memory used by a
+   * single query)
    *
    */
-  filter(predicate: string): Query {
-    this.inner.filter(predicate);
-    return this;
+  protected execute(): RecordBatchIterator {
+    return new RecordBatchIterator(this.nativeExecute());
   }
 
-  /**
-   * Select the columns to return. If not set, all columns are returned.
-   */
-  select(columns: string[]): Query {
-    this.inner.select(columns);
-    return this;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
+    const promise = this.nativeExecute();
+    return new RecordBatchIterator(promise);
   }
 
-  /**
-   * Set the limit of rows to return.
-   */
-  limit(limit: number): Query {
-    this.inner.limit(limit);
-    return this;
-  }
-
-  prefilter(prefilter: boolean): Query {
-    this.inner.prefilter(prefilter);
-    return this;
-  }
-
-  /**
-   * Set the query vector.
-   */
-  nearestTo(vector: number[]): Query {
-    this.inner.nearestTo(Float32Array.from(vector));
-    return this;
-  }
-
-  /**
-   * Set the number of IVF partitions to use for the query.
-   */
-  nprobes(nprobes: number): Query {
-    this.inner.nprobes(nprobes);
-    return this;
-  }
-
-  /**
-   * Set the refine factor for the query.
-   */
-  refineFactor(refineFactor: number): Query {
-    this.inner.refineFactor(refineFactor);
-    return this;
-  }
-
-  /**
-   * Execute the query and return the results as an AsyncIterator.
-   */
-  async executeStream(): Promise<RecordBatchIterator> {
-    const inner = await this.inner.executeStream();
-    return new RecordBatchIterator(inner);
-  }
-
-  /** Collect the results as an Arrow Table. */
+  /** Collect the results as an Arrow @see {@link ArrowTable}. */
   async toArrow(): Promise<ArrowTable> {
     const batches = [];
     for await (const batch of this) {
@@ -137,18 +165,211 @@ export class Query implements AsyncIterable<RecordBatch> {
     return new ArrowTable(batches);
   }
 
-  /** Returns a JSON Array of All results.
-   *
-   */
+  /** Collect the results as an array of objects. */
   async toArray(): Promise<unknown[]> {
     const tbl = await this.toArrow();
     // eslint-disable-next-line @typescript-eslint/no-unsafe-return
     return tbl.toArray();
   }
+}
 
+/**
+ * An interface for a query that can be executed
+ *
+ * Supported by all query types
+ */
+export interface ExecutableQuery {}
+
+/**
+ * A builder used to construct a vector search
+ *
+ * This builder can be reused to execute the query many times.
+ */
+export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
+  constructor(inner: NativeVectorQuery) {
+    super(inner);
+  }
+
+  /**
+   * Set the number of partitions to search (probe)
+   *
+   * This argument is only used when the vector column has an IVF PQ index.
+   * If there is no index then this value is ignored.
+   *
+   * The IVF stage of IVF PQ divides the input into partitions (clusters) of
+   * related values.
+   *
+   * The partition whose centroids are closest to the query vector will be
+   * exhaustiely searched to find matches.  This parameter controls how many
+   * partitions should be searched.
+   *
+   * Increasing this value will increase the recall of your query but will
+   * also increase the latency of your query.  The default value is 20.  This
+   * default is good for many cases but the best value to use will depend on
+   * your data and the recall that you need to achieve.
+   *
+   * For best results we recommend tuning this parameter with a benchmark against
+   * your actual data to find the smallest possible value that will still give
+   * you the desired recall.
+   */
+  nprobes(nprobes: number): VectorQuery {
+    this.inner.nprobes(nprobes);
+    return this;
+  }
+
+  /**
+   * Set the vector column to query
+   *
+   * This controls which column is compared to the query vector supplied in
+   * the call to @see {@link Query#nearestTo}
+   *
+   * This parameter must be specified if the table has more than one column
+   * whose data type is a fixed-size-list of floats.
+   */
+  column(column: string): VectorQuery {
+    this.inner.column(column);
+    return this;
+  }
+
+  /**
+   * Set the distance metric to use
+   *
+   * When performing a vector search we try and find the "nearest" vectors according
+   * to some kind of distance metric.  This parameter controls which distance metric to
+   * use.  See @see {@link IvfPqOptions.distanceType} for more details on the different
+   * distance metrics available.
+   *
+   * Note: if there is a vector index then the distance type used MUST match the distance
+   * type used to train the vector index.  If this is not done then the results will be
+   * invalid.
+   *
+   * By default "l2" is used.
+   */
+  distanceType(distanceType: string): VectorQuery {
+    this.inner.distanceType(distanceType);
+    return this;
+  }
+
+  /**
+   * A multiplier to control how many additional rows are taken during the refine step
+   *
+   * This argument is only used when the vector column has an IVF PQ index.
+   * If there is no index then this value is ignored.
+   *
+   * An IVF PQ index stores compressed (quantized) values.  They query vector is compared
+   * against these values and, since they are compressed, the comparison is inaccurate.
+   *
+   * This parameter can be used to refine the results.  It can improve both improve recall
+   * and correct the ordering of the nearest results.
+   *
+   * To refine results LanceDb will first perform an ANN search to find the nearest
+   * `limit` * `refine_factor` results.  In other words, if `refine_factor` is 3 and
+   * `limit` is the default (10) then the first 30 results will be selected.  LanceDb
+   * then fetches the full, uncompressed, values for these 30 results.  The results are
+   * then reordered by the true distance and only the nearest 10 are kept.
+   *
+   * Note: there is a difference between calling this method with a value of 1 and never
+   * calling this method at all.  Calling this method with any value will have an impact
+   * on your search latency.  When you call this method with a `refine_factor` of 1 then
+   * LanceDb still needs to fetch the full, uncompressed, values so that it can potentially
+   * reorder the results.
+   *
+   * Note: if this method is NOT called then the distances returned in the _distance column
+   * will be approximate distances based on the comparison of the quantized query vector
+   * and the quantized result vectors.  This can be considerably different than the true
+   * distance between the query vector and the actual uncompressed vector.
+   */
+  refineFactor(refineFactor: number): VectorQuery {
+    this.inner.refineFactor(refineFactor);
+    return this;
+  }
+
+  /**
+   * If this is called then filtering will happen after the vector search instead of
+   * before.
+   *
+   * By default filtering will be performed before the vector search.  This is how
+   * filtering is typically understood to work.  This prefilter step does add some
+   * additional latency.  Creating a scalar index on the filter column(s) can
+   * often improve this latency.  However, sometimes a filter is too complex or scalar
+   * indices cannot be applied to the column.  In these cases postfiltering can be
+   * used instead of prefiltering to improve latency.
+   *
+   * Post filtering applies the filter to the results of the vector search.  This means
+   * we only run the filter on a much smaller set of data.  However, it can cause the
+   * query to return fewer than `limit` results (or even no results) if none of the nearest
+   * results match the filter.
+   *
+   * Post filtering happens during the "refine stage" (described in more detail in
+   * @see {@link VectorQuery#refineFactor}).  This means that setting a higher refine
+   * factor can often help restore some of the results lost by post filtering.
+   */
+  postfilter(): VectorQuery {
+    this.inner.postfilter();
+    return this;
+  }
+
+  /**
+   * If this is called then any vector index is skipped
+   *
+   * An exhaustive (flat) search will be performed.  The query vector will
+   * be compared to every vector in the table.  At high scales this can be
+   * expensive.  However, this is often still useful.  For example, skipping
+   * the vector index can give you ground truth results which you can use to
+   * calculate your recall to select an appropriate value for nprobes.
+   */
+  bypassVectorIndex(): VectorQuery {
+    this.inner.bypassVectorIndex();
+    return this;
+  }
+}
+
+/** A builder for LanceDB queries. */
+export class Query extends QueryBase<NativeQuery, Query> {
+  constructor(tbl: NativeTable) {
+    super(tbl.query());
+  }
+
+  /**
+   * Find the nearest vectors to the given query vector.
+   *
+   * This converts the query from a plain query to a vector query.
+   *
+   * This method will attempt to convert the input to the query vector
+   * expected by the embedding model.  If the input cannot be converted
+   * then an error will be thrown.
+   *
+   * By default, there is no embedding model, and the input should be
+   * an array-like object of numbers (something that can be used as input
+   * to Float32Array.from)
+   *
+   * If there is only one vector column (a column whose data type is a
+   * fixed size list of floats) then the column does not need to be specified.
+   * If there is more than one vector column you must use
+   * @see {@link VectorQuery#column}  to specify which column you would like
+   * to compare with.
+   *
+   * If no index has been created on the vector column then a vector query
+   * will perform a distance comparison between the query vector and every
+   * vector in the database and then sort the results.  This is sometimes
+   * called a "flat search"
+   *
+   * For small databases, with a few hundred thousand vectors or less, this can
+   * be reasonably fast.  In larger databases you should create a vector index
+   * on the column.  If there is a vector index then an "approximate" nearest
+   * neighbor search (frequently called an ANN search) will be performed.  This
+   * search is much faster, but the results will be approximate.
+   *
+   * The query can be further parameterized using the returned builder.  There
+   * are various ANN search parameters that will let you fine tune your recall
+   * accuracy vs search latency.
+   *
+   * Vector searches always have a `limit`.  If `limit` has not been called then
+   * a default `limit` of 10 will be used.  @see {@link Query#limit}
+   */
+  nearestTo(vector: unknown): VectorQuery {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
-    const promise = this.inner.executeStream();
-    return new RecordBatchIterator(undefined, promise);
+    const vectorQuery = this.inner.nearestTo(Float32Array.from(vector as any));
+    return new VectorQuery(vectorQuery);
   }
 }

nodejs/lancedb/sanitize.ts

@@ -481,6 +481,13 @@ function sanitizeField(fieldLike: unknown): Field {
   return new Field(name, type, nullable, metadata);
 }
 
+/**
+ * Convert something schemaLike into a Schema instance
+ *
+ * This method is often needed even when the caller is using a Schema
+ * instance because they might be using a different instance of apache-arrow
+ * than lancedb is using.
+ */
 export function sanitizeSchema(schemaLike: unknown): Schema {
   if (schemaLike instanceof Schema) {
     return schemaLike;

nodejs/lancedb/table.ts

@@ -16,23 +16,40 @@ import { Schema, tableFromIPC } from "apache-arrow";
 import {
   AddColumnsSql,
   ColumnAlteration,
+  IndexConfig,
   Table as _NativeTable,
 } from "./native";
-import { Query } from "./query";
-import { IndexBuilder } from "./indexer";
+import { Query, VectorQuery } from "./query";
+import { IndexOptions } from "./indices";
 import { Data, fromDataToBuffer } from "./arrow";
 
+export { IndexConfig } from "./native";
 /**
  * Options for adding data to a table.
  */
 export interface AddDataOptions {
-  /** If "append" (the default) then the new data will be added to the table
+  /**
+   * If "append" (the default) then the new data will be added to the table
    *
    * If "overwrite" then the new data will replace the existing data in the table.
    */
   mode: "append" | "overwrite";
 }
 
+export interface UpdateOptions {
+  /**
+   * A filter that limits the scope of the update.
+   *
+   * This should be an SQL filter expression.
+   *
+   * Only rows that satisfy the expression will be updated.
+   *
+   * For example, this could be 'my_col == 0' to replace all instances
+   * of 0 in a column with some other default value.
+   */
+  where: string;
+}
+
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -58,7 +75,8 @@ export class Table {
     return this.inner.isOpen();
   }
 
-  /** Close the table, releasing any underlying resources.
+  /**
+   * Close the table, releasing any underlying resources.
    *
    * It is safe to call this method multiple times.
    *
@@ -82,9 +100,7 @@ export class Table {
 
   /**
    * Insert records into this Table.
-   *
    * @param {Data} data Records to be inserted into the Table
-   * @return The number of rows added to the table
    */
   async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
     const mode = options?.mode ?? "append";
@@ -93,6 +109,45 @@ export class Table {
     await this.inner.add(buffer, mode);
   }
 
+  /**
+   * Update existing records in the Table
+   *
+   * An update operation can be used to adjust existing values.  Use the
+   * returned builder to specify which columns to update.  The new value
+   * can be a literal value (e.g. replacing nulls with some default value)
+   * or an expression applied to the old value (e.g. incrementing a value)
+   *
+   * An optional condition can be specified (e.g. "only update if the old
+   * value is 0")
+   *
+   * Note: if your condition is something like "some_id_column == 7" and
+   * you are updating many rows (with different ids) then you will get
+   * better performance with a single [`merge_insert`] call instead of
+   * repeatedly calilng this method.
+   * @param {Map<string, string> | Record<string, string>} updates - the
+   * columns to update
+   *
+   * Keys in the map should specify the name of the column to update.
+   * Values in the map provide the new value of the column.  These can
+   * be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+   * based on the row being updated (e.g. "my_col + 1")
+   * @param {Partial<UpdateOptions>} options - additional options to control
+   * the update behavior
+   */
+  async update(
+    updates: Map<string, string> | Record<string, string>,
+    options?: Partial<UpdateOptions>,
+  ) {
+    const onlyIf = options?.where;
+    let columns: [string, string][];
+    if (updates instanceof Map) {
+      columns = Array.from(updates.entries());
+    } else {
+      columns = Object.entries(updates);
+    }
+    await this.inner.update(onlyIf, columns);
+  }
+
   /** Count the total number of rows in the dataset. */
   async countRows(filter?: string): Promise<number> {
     return await this.inner.countRows(filter);
@@ -103,106 +158,105 @@ export class Table {
     await this.inner.delete(predicate);
   }
 
-  /** Create an index over the columns.
-   *
-   * @param {string} column The column to create the index on. If not specified,
-   *                        it will create an index on vector field.
+  /**
+   * Create an index to speed up queries.
    *
+   * Indices can be created on vector columns or scalar columns.
+   * Indices on vector columns will speed up vector searches.
+   * Indices on scalar columns will speed up filtering (in both
+   * vector and non-vector searches)
    * @example
-   *
-   * By default, it creates vector idnex on one vector column.
-   *
-   * ```typescript
+   * // If the column has a vector (fixed size list) data type then
+   * // an IvfPq vector index will be created.
    * const table = await conn.openTable("my_table");
-   * await table.createIndex().build();
-   * ```
-   *
-   * You can specify `IVF_PQ` parameters via `ivf_pq({})` call.
-   * ```typescript
+   * await table.createIndex(["vector"]);
+   * @example
+   * // For advanced control over vector index creation you can specify
+   * // the index type and options.
    * const table = await conn.openTable("my_table");
-   * await table.createIndex("my_vec_col")
+   * await table.createIndex(["vector"], I)
    *   .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
    *   .build();
-   * ```
-   *
-   * Or create a Scalar index
-   *
-   * ```typescript
+   * @example
+   * // Or create a Scalar index
    * await table.createIndex("my_float_col").build();
-   * ```
    */
-  createIndex(column?: string): IndexBuilder {
-    let builder = new IndexBuilder(this.inner);
-    if (column !== undefined) {
-      builder = builder.column(column);
-    }
-    return builder;
+  async createIndex(column: string, options?: Partial<IndexOptions>) {
+    // Bit of a hack to get around the fact that TS has no package-scope.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const nativeIndex = (options?.config as any)?.inner;
+    await this.inner.createIndex(nativeIndex, column, options?.replace);
   }
 
   /**
-   * Create a generic {@link Query} Builder.
+   * Create a {@link Query} Builder.
+   *
+   * Queries allow you to search your existing data.  By default the query will
+   * return all the data in the table in no particular order.  The builder
+   * returned by this method can be used to control the query using filtering,
+   * vector similarity, sorting, and more.
+   *
+   * Note: By default, all columns are returned.  For best performance, you should
+   * only fetch the columns you need.  See [`Query::select_with_projection`] for
+   * more details.
    *
    * When appropriate, various indices and statistics based pruning will be used to
    * accelerate the query.
-   *
    * @example
-   *
-   * ### Run a SQL-style query
-   * ```typescript
+   * // SQL-style filtering
+   * //
+   * // This query will return up to 1000 rows whose value in the `id` column
+   * // is greater than 5.  LanceDb supports a broad set of filtering functions.
    * for await (const batch of table.query()
    *                          .filter("id > 1").select(["id"]).limit(20)) {
    *  console.log(batch);
    * }
-   * ```
-   *
-   * ### Run Top-10 vector similarity search
-   * ```typescript
+   * @example
+   * // Vector Similarity Search
+   * //
+   * // This example will find the 10 rows whose value in the "vector" column are
+   * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
+   * // on the "vector" column then this will perform an ANN search.
+   * //
+   * // The `refine_factor` and `nprobes` methods are used to control the recall /
+   * // latency tradeoff of the search.
    * for await (const batch of table.query()
    *                    .nearestTo([1, 2, 3])
    *                    .refineFactor(5).nprobe(10)
    *                    .limit(10)) {
    *  console.log(batch);
    * }
-   *```
-   *
-   * ### Scan the full dataset
-   * ```typescript
+   * @example
+   * // Scan the full dataset
+   * //
+   * // This query will return everything in the table in no particular order.
    * for await (const batch of table.query()) {
    *   console.log(batch);
    * }
-   *
-   * ### Return the full dataset as Arrow Table
-   * ```typescript
-   * let arrowTbl = await table.query().nearestTo([1.0, 2.0, 0.5, 6.7]).toArrow();
-   * ```
-   *
-   * @returns {@link Query}
+   * @returns {Query} A builder that can be used to parameterize the query
    */
   query(): Query {
     return new Query(this.inner);
   }
 
-  /** Search the table with a given query vector.
+  /**
+   * Search the table with a given query vector.
    *
-   * This is a convenience method for preparing an ANN {@link Query}.
+   * This is a convenience method for preparing a vector query and
+   * is the same thing as calling `nearestTo` on the builder returned
+   * by `query`.  @see {@link Query#nearestTo} for more details.
    */
-  search(vector: number[], column?: string): Query {
-    const q = this.query();
-    q.nearestTo(vector);
-    if (column !== undefined) {
-      q.column(column);
-    }
-    return q;
+  vectorSearch(vector: unknown): VectorQuery {
+    return this.query().nearestTo(vector);
   }
 
   // TODO: Support BatchUDF
   /**
    * Add new columns with defined values.
-   *
-   * @param newColumnTransforms pairs of column names and the SQL expression to use
-   *                            to calculate the value of the new column. These
-   *                            expressions will be evaluated for each row in the
-   *                            table, and can reference existing columns in the table.
+   * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
+   * the SQL expression to use to calculate the value of the new column. These
+   * expressions will be evaluated for each row in the table, and can
+   * reference existing columns in the table.
    */
   async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
     await this.inner.addColumns(newColumnTransforms);
@@ -210,8 +264,8 @@ export class Table {
 
   /**
    * Alter the name or nullability of columns.
-   *
-   * @param columnAlterations One or more alterations to apply to columns.
+   * @param {ColumnAlteration[]} columnAlterations One or more alterations to
+   * apply to columns.
    */
   async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
     await this.inner.alterColumns(columnAlterations);
@@ -224,12 +278,76 @@ export class Table {
    * underlying storage. In order to remove the data, you must subsequently
    * call ``compact_files`` to rewrite the data without the removed columns and
    * then call ``cleanup_files`` to remove the old files.
-   *
-   * @param columnNames The names of the columns to drop. These can be nested
-   *                    column references (e.g. "a.b.c") or top-level column
-   *                    names (e.g. "a").
+   * @param {string[]} columnNames The names of the columns to drop. These can
+   * be nested column references (e.g. "a.b.c") or top-level column names
+   * (e.g. "a").
    */
   async dropColumns(columnNames: string[]): Promise<void> {
     await this.inner.dropColumns(columnNames);
   }
+
+  /**
+   * Retrieve the version of the table
+   *
+   * LanceDb supports versioning.  Every operation that modifies the table increases
+   * version.  As long as a version hasn't been deleted you can `[Self::checkout]` that
+   * version to view the data at that point.  In addition, you can `[Self::restore]` the
+   * version to replace the current table with a previous version.
+   */
+  async version(): Promise<number> {
+    return await this.inner.version();
+  }
+
+  /**
+   * Checks out a specific version of the Table
+   *
+   * Any read operation on the table will now access the data at the checked out version.
+   * As a consequence, calling this method will disable any read consistency interval
+   * that was previously set.
+   *
+   * This is a read-only operation that turns the table into a sort of "view"
+   * or "detached head".  Other table instances will not be affected.  To make the change
+   * permanent you can use the `[Self::restore]` method.
+   *
+   * Any operation that modifies the table will fail while the table is in a checked
+   * out state.
+   *
+   * To return the table to a normal state use `[Self::checkout_latest]`
+   */
+  async checkout(version: number): Promise<void> {
+    await this.inner.checkout(version);
+  }
+
+  /**
+   * Ensures the table is pointing at the latest version
+   *
+   * This can be used to manually update a table when the read_consistency_interval is None
+   * It can also be used to undo a `[Self::checkout]` operation
+   */
+  async checkoutLatest(): Promise<void> {
+    await this.inner.checkoutLatest();
+  }
+
+  /**
+   * Restore the table to the currently checked out version
+   *
+   * This operation will fail if checkout has not been called previously
+   *
+   * This operation will overwrite the latest version of the table with a
+   * previous version.  Any changes made since the checked out version will
+   * no longer be visible.
+   *
+   * Once the operation concludes the table will no longer be in a checked
+   * out state and the read_consistency_interval, if any, will apply.
+   */
+  async restore(): Promise<void> {
+    await this.inner.restore();
+  }
+
+  /**
+   * List all indices that have been created with Self::create_index
+   */
+  async listIndices(): Promise<IndexConfig[]> {
+    return await this.inner.listIndices();
+  }
 }

nodejs/npm/darwin-arm64/README.md

@@ -1,3 +1,3 @@
-# `lancedb-darwin-arm64`
+# `@lancedb/lancedb-darwin-arm64`
 
-This is the **aarch64-apple-darwin** binary for `lancedb`
+This is the **aarch64-apple-darwin** binary for `@lancedb/lancedb`

nodejs/npm/darwin-arm64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-darwin-arm64",
+  "name": "@lancedb/lancedb-darwin-arm64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -11,7 +11,7 @@
   "files": [
     "lancedb.darwin-arm64.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
     "node": ">= 18"
   }

nodejs/npm/darwin-x64/README.md

@@ -1,3 +1,3 @@
-# `lancedb-darwin-x64`
+# `@lancedb/lancedb-darwin-x64`
 
-This is the **x86_64-apple-darwin** binary for `lancedb`
+This is the **x86_64-apple-darwin** binary for `@lancedb/lancedb`

nodejs/npm/darwin-x64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-darwin-x64",
+  "name": "@lancedb/lancedb-darwin-x64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -11,7 +11,7 @@
   "files": [
     "lancedb.darwin-x64.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
     "node": ">= 18"
   }

nodejs/npm/linux-arm64-gnu/README.md

@@ -1,3 +1,3 @@
-# `lancedb-linux-arm64-gnu`
+# `@lancedb/lancedb-linux-arm64-gnu`
 
-This is the **aarch64-unknown-linux-gnu** binary for `lancedb`
+This is the **aarch64-unknown-linux-gnu** binary for `@lancedb/lancedb`

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-linux-arm64-gnu",
+  "name": "@lancedb/lancedb-linux-arm64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -11,9 +11,9 @@
   "files": [
     "lancedb.linux-arm64-gnu.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
-    "node": ">= 10"
+    "node": ">= 18"
   },
   "libc": [
     "glibc"

nodejs/npm/linux-x64-gnu/README.md

@@ -1,3 +1,3 @@
-# `lancedb-linux-x64-gnu`
+# `@lancedb/lancedb-linux-x64-gnu`
 
-This is the **x86_64-unknown-linux-gnu** binary for `lancedb`
+This is the **x86_64-unknown-linux-gnu** binary for `@lancedb/lancedb`

nodejs/npm/linux-x64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "lancedb-linux-x64-gnu",
+  "name": "@lancedb/lancedb-linux-x64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -11,9 +11,9 @@
   "files": [
     "lancedb.linux-x64-gnu.node"
   ],
-  "license": "MIT",
+  "license": "Apache 2.0",
   "engines": {
-    "node": ">= 10"
+    "node": ">= 18"
   },
   "libc": [
     "glibc"

nodejs/npm/win32-x64-msvc/README.md

@@ -0,0 +1,3 @@
+# `@lancedb/lancedb-win32-x64-msvc`
+
+This is the **x86_64-pc-windows-msvc** binary for `@lancedb/lancedb`

nodejs/npm/win32-x64-msvc/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@lancedb/lancedb-win32-x64-msvc",
+  "version": "0.4.3",
+  "os": [
+    "win32"
+  ],
+  "cpu": [
+    "x64"
+  ],
+  "main": "lancedb.win32-x64-msvc.node",
+  "files": [
+    "lancedb.win32-x64-msvc.node"
+  ],
+  "license": "Apache 2.0",
+  "engines": {
+    "node": ">= 18"
+  }
+}

nodejs/package-lock.json

@@ -1,11 +1,11 @@
 {
-  "name": "lancedb",
+  "name": "@lancedb/lancedb",
   "version": "0.4.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "lancedb",
+      "name": "@lancedb/lancedb",
       "version": "0.4.3",
       "cpu": [
         "x64",
@@ -15,8 +15,12 @@
       "os": [
         "darwin",
         "linux",
-        "windows"
+        "win32"
       ],
+      "dependencies": {
+        "apache-arrow": "^15.0.0",
+        "openai": "^4.29.2"
+      },
       "devDependencies": {
         "@napi-rs/cli": "^2.18.0",
         "@types/jest": "^29.1.2",
@@ -26,8 +30,10 @@
         "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "eslint": "^8.57.0",
         "eslint-config-prettier": "^9.1.0",
+        "eslint-plugin-jsdoc": "^48.2.1",
         "jest": "^29.7.0",
         "prettier": "^3.1.0",
+        "shx": "^0.3.4",
         "tmp": "^0.2.3",
         "ts-jest": "^29.1.2",
         "typedoc": "^0.25.7",
@@ -39,14 +45,11 @@
         "node": ">= 18"
       },
       "optionalDependencies": {
-        "lancedb-darwin-arm64": "0.4.3",
-        "lancedb-darwin-x64": "0.4.3",
-        "lancedb-linux-arm64-gnu": "0.4.3",
-        "lancedb-linux-x64-gnu": "0.4.3",
-        "openai": "^4.28.4"
-      },
-      "peerDependencies": {
-        "apache-arrow": "^15.0.0"
+        "@lancedb/lancedb-darwin-arm64": "0.4.3",
+        "@lancedb/lancedb-darwin-x64": "0.4.3",
+        "@lancedb/lancedb-linux-arm64-gnu": "0.4.3",
+        "@lancedb/lancedb-linux-x64-gnu": "0.4.3",
+        "@lancedb/lancedb-win32-x64-msvc": "0.4.3"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -755,6 +758,20 @@
       "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==",
       "dev": true
     },
+    "node_modules/@es-joy/jsdoccomment": {
+      "version": "0.42.0",
+      "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.42.0.tgz",
+      "integrity": "sha512-R1w57YlVA6+YE01wch3GPYn6bCsrOV3YW/5oGGE2tmX6JcL9Nr+b5IikrjMPF+v9CV3ay+obImEdsDhovhJrzw==",
+      "dev": true,
+      "dependencies": {
+        "comment-parser": "1.4.1",
+        "esquery": "^1.5.0",
+        "jsdoc-type-pratt-parser": "~4.0.0"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "node_modules/@eslint-community/eslint-utils": {
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
@@ -1302,6 +1319,66 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@lancedb/lancedb-darwin-arm64": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.4.3.tgz",
+      "integrity": "sha512-+kxuWUK9vtLBbjFMkIKeQ32kxK2tgvZRCQaU1I3RJ3+dLmDIVeIj+KJSlMelkKa2QC4JoyHQi9Ty1PdS2DojmQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-darwin-x64": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-x64/-/lancedb-darwin-x64-0.4.3.tgz",
+      "integrity": "sha512-JYvsSYxTOa/7OMojulz9h0gN2FwvypG/6l6dpLkViZ5LDvRcfVyDTzOLcOJkFn+db4TKeBOVyMWnnpDKaB+jLA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-linux-x64-gnu": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.4.3.tgz",
+      "integrity": "sha512-jDANHchWNGmu1wfAyBk0apoFlLxtJ7FRc31pAQ3tKE4fwlgG7bUcaTX6s5C3vMNWXnyQLQtVuWZNXi2nVj879g==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-win32-x64-msvc": {
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.4.3.tgz",
+      "integrity": "sha512-qADveXyv4YzllIbOOq8soqFfL7p7I35uhrD3PcTvj4Qxuo6q7pgQWQz2Mt3kGBpyPkH2yE4wWAGJhayShLRbiQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/@napi-rs/cli": {
       "version": "2.18.0",
       "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
@@ -1381,7 +1458,6 @@
       "version": "0.5.6",
       "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.6.tgz",
       "integrity": "sha512-aYX01Ke9hunpoCexYAgQucEpARGQ5w/cqHFrIR+e9gdKb1QWTsVJuTJ2ozQzIAxLyRQe/m+2RqzkyOOGiMKRQA==",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.4.0"
       }
@@ -1430,8 +1506,7 @@
     "node_modules/@types/command-line-args": {
       "version": "5.2.3",
       "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz",
-      "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==",
-      "peer": true
+      "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw=="
     },
     "node_modules/@types/command-line-usage": {
       "version": "5.0.2",
@@ -1499,7 +1574,6 @@
       "version": "2.6.11",
       "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz",
       "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==",
-      "optional": true,
       "dependencies": {
         "@types/node": "*",
         "form-data": "^4.0.0"
@@ -1768,7 +1842,6 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
       "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
-      "optional": true,
       "dependencies": {
         "event-target-shim": "^5.0.0"
       },
@@ -1801,7 +1874,6 @@
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
       "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
-      "optional": true,
       "dependencies": {
         "humanize-ms": "^1.2.1"
       },
@@ -1898,7 +1970,6 @@
       "version": "15.0.0",
       "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-15.0.0.tgz",
       "integrity": "sha512-e6aunxNKM+woQf137ny3tp/xbLjFJS2oGQxQhYGqW6dGeIwNV1jOeEAeR6sS2jwAI2qLO83gYIP2MBz02Gw5Xw==",
-      "peer": true,
       "dependencies": {
         "@swc/helpers": "^0.5.2",
         "@types/command-line-args": "^5.2.1",
@@ -1948,6 +2019,15 @@
       "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
       "dev": true
     },
+    "node_modules/are-docs-informative": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/are-docs-informative/-/are-docs-informative-0.0.2.tgz",
+      "integrity": "sha512-ixiS0nLNNG5jNQzgZJNoUpBKdo9yTYZMGJ+QgT2jmjR7G7+QHRCc4v6LQ3NgE7EBJq+o0ams3waJwkrlBom8Ig==",
+      "dev": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -1977,8 +2057,7 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "optional": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
     "node_modules/babel-jest": {
       "version": "29.7.0",
@@ -2105,8 +2184,7 @@
     "node_modules/base-64": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
-      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==",
-      "optional": true
+      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
     },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
@@ -2189,6 +2267,18 @@
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
       "dev": true
     },
+    "node_modules/builtin-modules": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz",
+      "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/camelcase": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
@@ -2260,7 +2350,6 @@
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
       "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
-      "optional": true,
       "engines": {
         "node": "*"
       }
@@ -2321,7 +2410,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "optional": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -2373,6 +2461,15 @@
         "node": ">=12.17"
       }
     },
+    "node_modules/comment-parser": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.4.1.tgz",
+      "integrity": "sha512-buhp5kePrmda3vhc5B9t7pUQXAb2Tnd0qgpkIhPhkHXxJpiPJ11H0ZEU0oBpJ2QztSbzG/ZxMj/CHsYJqRHmyg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 12.0.0"
+      }
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -2424,7 +2521,6 @@
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
       "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
-      "optional": true,
       "engines": {
         "node": "*"
       }
@@ -2485,7 +2581,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "optional": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -2512,7 +2607,6 @@
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
       "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
-      "optional": true,
       "dependencies": {
         "base-64": "^0.1.0",
         "md5": "^2.3.0"
@@ -2660,6 +2754,29 @@
         "eslint": ">=7.0.0"
       }
     },
+    "node_modules/eslint-plugin-jsdoc": {
+      "version": "48.2.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-48.2.1.tgz",
+      "integrity": "sha512-iUvbcyDZSO/9xSuRv2HQBw++8VkV/pt3UWtX9cpPH0l7GKPq78QC/6+PmyQHHvNZaTjAce6QVciEbnc6J/zH5g==",
+      "dev": true,
+      "dependencies": {
+        "@es-joy/jsdoccomment": "~0.42.0",
+        "are-docs-informative": "^0.0.2",
+        "comment-parser": "1.4.1",
+        "debug": "^4.3.4",
+        "escape-string-regexp": "^4.0.0",
+        "esquery": "^1.5.0",
+        "is-builtin-module": "^3.2.1",
+        "semver": "^7.6.0",
+        "spdx-expression-parse": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "peerDependencies": {
+        "eslint": "^7.0.0 || ^8.0.0 || ^9.0.0"
+      }
+    },
     "node_modules/eslint-scope": {
       "version": "7.2.2",
       "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz",
@@ -2794,7 +2911,6 @@
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
       "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
-      "optional": true,
       "engines": {
         "node": ">=6"
       }
@@ -2956,7 +3072,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "optional": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -2969,14 +3084,12 @@
     "node_modules/form-data-encoder": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
-      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
-      "optional": true
+      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
     },
     "node_modules/formdata-node": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
       "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
-      "optional": true,
       "dependencies": {
         "node-domexception": "1.0.0",
         "web-streams-polyfill": "4.0.0-beta.3"
@@ -2989,7 +3102,6 @@
       "version": "4.0.0-beta.3",
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
       "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
-      "optional": true,
       "engines": {
         "node": ">= 14"
       }
@@ -3204,7 +3316,6 @@
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
       "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
-      "optional": true,
       "dependencies": {
         "ms": "^2.0.0"
       }
@@ -3287,6 +3398,15 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "dev": true
     },
+    "node_modules/interpret": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
+      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/is-arrayish": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
@@ -3296,8 +3416,22 @@
     "node_modules/is-buffer": {
       "version": "1.1.6",
       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
-      "optional": true
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
+    "node_modules/is-builtin-module": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz",
+      "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==",
+      "dev": true,
+      "dependencies": {
+        "builtin-modules": "^3.3.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
     },
     "node_modules/is-core-module": {
       "version": "2.13.1",
@@ -4172,6 +4306,15 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/jsdoc-type-pratt-parser": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-4.0.0.tgz",
+      "integrity": "sha512-YtOli5Cmzy3q4dP26GraSOeAhqecewG04hoO8DY56CH4KJ9Fvv5qKWUCCo3HZob7esJQHCv6/+bnTy72xZZaVQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/jsesc": {
       "version": "2.5.2",
       "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
@@ -4366,7 +4509,6 @@
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
       "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
-      "optional": true,
       "dependencies": {
         "charenc": "0.0.2",
         "crypt": "0.0.2",
@@ -4405,7 +4547,6 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "optional": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -4414,7 +4555,6 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "optional": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -4446,8 +4586,7 @@
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
-      "optional": true
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/natural-compare": {
       "version": "1.4.0",
@@ -4475,7 +4614,6 @@
           "url": "https://paypal.me/jimmywarting"
         }
       ],
-      "optional": true,
       "engines": {
         "node": ">=10.5.0"
       }
@@ -4484,7 +4622,6 @@
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
       "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
-      "optional": true,
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
@@ -4531,10 +4668,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.28.4",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.28.4.tgz",
-      "integrity": "sha512-RNIwx4MT/F0zyizGcwS+bXKLzJ8QE9IOyigDG/ttnwB220d58bYjYFp0qjvGwEFBO6+pvFVIDABZPGDl46RFsg==",
-      "optional": true,
+      "version": "4.29.2",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz",
+      "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -4551,10 +4687,9 @@
       }
     },
     "node_modules/openai/node_modules/@types/node": {
-      "version": "18.19.20",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.20.tgz",
-      "integrity": "sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==",
-      "optional": true,
+      "version": "18.19.26",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.26.tgz",
+      "integrity": "sha512-+wiMJsIwLOYCvUqSdKTrfkS8mpTp+MPINe6+Np4TAGFWWRWiBQ5kSq9nZGCSPkzx9mvT+uEukzpX4MOSCydcvw==",
       "dependencies": {
         "undici-types": "~5.26.4"
       }
@@ -4904,6 +5039,18 @@
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
       "dev": true
     },
+    "node_modules/rechoir": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
+      "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
+      "dev": true,
+      "dependencies": {
+        "resolve": "^1.1.6"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/repeat-string": {
       "version": "1.6.1",
       "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
@@ -5018,9 +5165,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.5.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
-      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "version": "7.6.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz",
+      "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -5053,6 +5200,23 @@
         "node": ">=8"
       }
     },
+    "node_modules/shelljs": {
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
+      "dev": true,
+      "dependencies": {
+        "glob": "^7.0.0",
+        "interpret": "^1.0.0",
+        "rechoir": "^0.6.2"
+      },
+      "bin": {
+        "shjs": "bin/shjs"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/shiki": {
       "version": "0.14.7",
       "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.7.tgz",
@@ -5065,6 +5229,22 @@
         "vscode-textmate": "^8.0.0"
       }
     },
+    "node_modules/shx": {
+      "version": "0.3.4",
+      "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz",
+      "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.3",
+        "shelljs": "^0.8.5"
+      },
+      "bin": {
+        "shx": "lib/cli.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -5105,6 +5285,28 @@
         "source-map": "^0.6.0"
       }
     },
+    "node_modules/spdx-exceptions": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz",
+      "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==",
+      "dev": true
+    },
+    "node_modules/spdx-expression-parse": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz",
+      "integrity": "sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==",
+      "dev": true,
+      "dependencies": {
+        "spdx-exceptions": "^2.1.0",
+        "spdx-license-ids": "^3.0.0"
+      }
+    },
+    "node_modules/spdx-license-ids": {
+      "version": "3.0.17",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.17.tgz",
+      "integrity": "sha512-sh8PWc/ftMqAAdFiBu6Fy6JUOYjqDJBJvIhpfDMyHrr0Rbp5liZqd4TjtQ/RgfLjKFZb+LMx5hpml5qOWy0qvg==",
+      "dev": true
+    },
     "node_modules/sprintf-js": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@@ -5318,8 +5520,7 @@
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
-      "optional": true
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
     "node_modules/ts-api-utils": {
       "version": "1.0.3",
@@ -5815,7 +6016,6 @@
       "version": "3.3.3",
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
       "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
-      "optional": true,
       "engines": {
         "node": ">= 8"
       }
@@ -5823,14 +6023,12 @@
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
-      "optional": true
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
     },
     "node_modules/whatwg-url": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
       "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-      "optional": true,
       "dependencies": {
         "tr46": "~0.0.3",
         "webidl-conversions": "^3.0.0"

nodejs/package.json

@@ -1,17 +1,18 @@
 {
-  "name": "lancedb",
+  "name": "@lancedb/lancedb",
   "version": "0.4.3",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "napi": {
-    "name": "lancedb-nodejs",
+    "name": "lancedb",
     "triples": {
       "defaults": false,
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-unknown-linux-gnu",
         "x86_64-apple-darwin",
-        "x86_64-unknown-linux-gnu"
+        "x86_64-unknown-linux-gnu",
+        "x86_64-pc-windows-msvc"
       ]
     }
   },
@@ -25,8 +26,10 @@
     "apache-arrow-old": "npm:apache-arrow@13.0.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-jsdoc": "^48.2.1",
     "jest": "^29.7.0",
     "prettier": "^3.1.0",
+    "shx": "^0.3.4",
     "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
     "typedoc": "^0.25.7",
@@ -47,13 +50,14 @@
   "os": [
     "darwin",
     "linux",
-    "windows"
+    "win32"
   ],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:native": "napi build --platform --release --js lancedb/native.js --dts lancedb/native.d.ts dist/",
     "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
-    "build": "npm run build:debug && tsc -b",
+    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
+    "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
+    "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
     "chkformat": "prettier . --check",
     "docs": "typedoc --plugin typedoc-plugin-markdown lancedb/index.ts",
     "lint": "eslint lancedb && eslint __test__",
@@ -63,13 +67,14 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "lancedb-darwin-arm64": "0.4.3",
-    "lancedb-darwin-x64": "0.4.3",
-    "lancedb-linux-arm64-gnu": "0.4.3",
-    "lancedb-linux-x64-gnu": "0.4.3",
-    "openai": "^4.28.4"
+    "@lancedb/lancedb-darwin-arm64": "0.4.3",
+    "@lancedb/lancedb-darwin-x64": "0.4.3",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.4.3",
+    "@lancedb/lancedb-linux-x64-gnu": "0.4.3",
+    "@lancedb/lancedb-win32-x64-msvc": "0.4.3"
   },
-  "peerDependencies": {
+  "dependencies": {
+    "openai": "^4.29.2",
     "apache-arrow": "^15.0.0"
   }
 }

nodejs/src/connection.rs

@@ -124,7 +124,7 @@ impl Connection {
         let mode = Self::parse_create_mode_str(&mode)?;
         let tbl = self
             .get_inner()?
-            .create_table(&name, Box::new(batches))
+            .create_table(&name, batches)
             .mode(mode)
             .execute()
             .await

nodejs/src/error.rs

@@ -0,0 +1,12 @@
+pub type Result<T> = napi::Result<T>;
+
+pub trait NapiErrorExt<T> {
+    /// Convert to a napi error using from_reason(err.to_string())
+    fn default_error(self) -> Result<T>;
+}
+
+impl<T> NapiErrorExt<T> for std::result::Result<T, lancedb::Error> {
+    fn default_error(self) -> Result<T> {
+        self.map_err(|err| napi::Error::from_reason(err.to_string()))
+    }
+}

nodejs/src/index.rs

@@ -14,126 +14,66 @@
 
 use std::sync::Mutex;
 
-use lance_linalg::distance::MetricType as LanceMetricType;
-use lancedb::index::IndexBuilder as LanceDbIndexBuilder;
-use lancedb::Table as LanceDbTable;
+use lancedb::index::scalar::BTreeIndexBuilder;
+use lancedb::index::vector::IvfPqIndexBuilder;
+use lancedb::index::Index as LanceDbIndex;
 use napi_derive::napi;
 
-#[napi]
-pub enum IndexType {
-    Scalar,
-    IvfPq,
-}
+use crate::util::parse_distance_type;
 
 #[napi]
-pub enum MetricType {
-    L2,
-    Cosine,
-    Dot,
+pub struct Index {
+    inner: Mutex<Option<LanceDbIndex>>,
 }
 
-impl From<MetricType> for LanceMetricType {
-    fn from(metric: MetricType) -> Self {
-        match metric {
-            MetricType::L2 => Self::L2,
-            MetricType::Cosine => Self::Cosine,
-            MetricType::Dot => Self::Dot,
-        }
+impl Index {
+    pub fn consume(&self) -> napi::Result<LanceDbIndex> {
+        self.inner
+            .lock()
+            .unwrap()
+            .take()
+            .ok_or(napi::Error::from_reason(
+                "attempt to use an index more than once",
+            ))
     }
 }
 
 #[napi]
-pub struct IndexBuilder {
-    inner: Mutex<Option<LanceDbIndexBuilder>>,
-}
-
-impl IndexBuilder {
-    fn modify(
-        &self,
-        mod_fn: impl Fn(LanceDbIndexBuilder) -> LanceDbIndexBuilder,
-    ) -> napi::Result<()> {
-        let mut inner = self.inner.lock().unwrap();
-        let inner_builder = inner.take().ok_or_else(|| {
-            napi::Error::from_reason("IndexBuilder has already been consumed".to_string())
-        })?;
-        let inner_builder = mod_fn(inner_builder);
-        inner.replace(inner_builder);
-        Ok(())
-    }
-}
-
-#[napi]
-impl IndexBuilder {
-    pub fn new(tbl: &LanceDbTable) -> Self {
-        let inner = tbl.create_index(&[]);
-        Self {
-            inner: Mutex::new(Some(inner)),
-        }
-    }
-
-    #[napi]
-    pub fn replace(&self, v: bool) -> napi::Result<()> {
-        self.modify(|b| b.replace(v))
-    }
-
-    #[napi]
-    pub fn column(&self, c: String) -> napi::Result<()> {
-        self.modify(|b| b.columns(&[c.as_str()]))
-    }
-
-    #[napi]
-    pub fn name(&self, name: String) -> napi::Result<()> {
-        self.modify(|b| b.name(name.as_str()))
-    }
-
-    #[napi]
+impl Index {
+    #[napi(factory)]
     pub fn ivf_pq(
-        &self,
-        metric_type: Option<MetricType>,
+        distance_type: Option<String>,
         num_partitions: Option<u32>,
         num_sub_vectors: Option<u32>,
-        num_bits: Option<u32>,
         max_iterations: Option<u32>,
         sample_rate: Option<u32>,
-    ) -> napi::Result<()> {
-        self.modify(|b| {
-            let mut b = b.ivf_pq();
-            if let Some(metric_type) = metric_type {
-                b = b.metric_type(metric_type.into());
+    ) -> napi::Result<Self> {
+        let mut ivf_pq_builder = IvfPqIndexBuilder::default();
+        if let Some(distance_type) = distance_type {
+            let distance_type = parse_distance_type(distance_type)?;
+            ivf_pq_builder = ivf_pq_builder.distance_type(distance_type);
         }
         if let Some(num_partitions) = num_partitions {
-                b = b.num_partitions(num_partitions);
+            ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
         }
         if let Some(num_sub_vectors) = num_sub_vectors {
-                b = b.num_sub_vectors(num_sub_vectors);
-            }
-            if let Some(num_bits) = num_bits {
-                b = b.num_bits(num_bits);
+            ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
         }
         if let Some(max_iterations) = max_iterations {
-                b = b.max_iterations(max_iterations);
+            ivf_pq_builder = ivf_pq_builder.max_iterations(max_iterations);
         }
         if let Some(sample_rate) = sample_rate {
-                b = b.sample_rate(sample_rate);
+            ivf_pq_builder = ivf_pq_builder.sample_rate(sample_rate);
         }
-            b
+        Ok(Self {
+            inner: Mutex::new(Some(LanceDbIndex::IvfPq(ivf_pq_builder))),
         })
     }
 
-    #[napi]
-    pub fn scalar(&self) -> napi::Result<()> {
-        self.modify(|b| b.scalar())
-    }
-
-    #[napi]
-    pub async fn build(&self) -> napi::Result<()> {
-        let inner = self.inner.lock().unwrap().take().ok_or_else(|| {
-            napi::Error::from_reason("IndexBuilder has already been consumed".to_string())
-        })?;
-        inner
-            .build()
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("Failed to build index: {}", e)))?;
-        Ok(())
+    #[napi(factory)]
+    pub fn btree() -> Self {
+        Self {
+            inner: Mutex::new(Some(LanceDbIndex::BTree(BTreeIndexBuilder::default()))),
+        }
     }
 }

nodejs/src/iterator.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use futures::StreamExt;
-use lance::io::RecordBatchStream;
+use lancedb::arrow::SendableRecordBatchStream;
 use lancedb::ipc::batches_to_ipc_file;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
@@ -21,12 +21,12 @@ use napi_derive::napi;
 /** Typescript-style Async Iterator over RecordBatches */
 #[napi]
 pub struct RecordBatchIterator {
-    inner: Box<dyn RecordBatchStream + Unpin>,
+    inner: SendableRecordBatchStream,
 }
 
 #[napi]
 impl RecordBatchIterator {
-    pub(crate) fn new(inner: Box<dyn RecordBatchStream + Unpin>) -> Self {
+    pub(crate) fn new(inner: SendableRecordBatchStream) -> Self {
         Self { inner }
     }
 

nodejs/src/lib.rs

@@ -16,10 +16,12 @@ use connection::Connection;
 use napi_derive::*;
 
 mod connection;
+mod error;
 mod index;
 mod iterator;
 mod query;
 mod table;
+mod util;
 
 #[napi(object)]
 #[derive(Debug)]

nodejs/src/query.rs

@@ -12,36 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use lancedb::query::Query as LanceDBQuery;
+use lancedb::query::ExecutableQuery;
+use lancedb::query::Query as LanceDbQuery;
+use lancedb::query::QueryBase;
+use lancedb::query::Select;
+use lancedb::query::VectorQuery as LanceDbVectorQuery;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
 
+use crate::error::NapiErrorExt;
 use crate::iterator::RecordBatchIterator;
+use crate::util::parse_distance_type;
 
 #[napi]
 pub struct Query {
-    inner: LanceDBQuery,
+    inner: LanceDbQuery,
 }
 
 #[napi]
 impl Query {
-    pub fn new(query: LanceDBQuery) -> Self {
+    pub fn new(query: LanceDbQuery) -> Self {
         Self { inner: query }
     }
 
+    // We cannot call this r#where because NAPI gets confused by the r#
     #[napi]
-    pub fn column(&mut self, column: String) {
-        self.inner = self.inner.clone().column(&column);
+    pub fn only_if(&mut self, predicate: String) {
+        self.inner = self.inner.clone().only_if(predicate);
     }
 
     #[napi]
-    pub fn filter(&mut self, filter: String) {
-        self.inner = self.inner.clone().filter(filter);
-    }
-
-    #[napi]
-    pub fn select(&mut self, columns: Vec<String>) {
-        self.inner = self.inner.clone().select(&columns);
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner = self.inner.clone().select(Select::dynamic(&columns));
     }
 
     #[napi]
@@ -50,13 +52,46 @@ impl Query {
     }
 
     #[napi]
-    pub fn prefilter(&mut self, prefilter: bool) {
-        self.inner = self.inner.clone().prefilter(prefilter);
+    pub fn nearest_to(&mut self, vector: Float32Array) -> Result<VectorQuery> {
+        let inner = self
+            .inner
+            .clone()
+            .nearest_to(vector.as_ref())
+            .default_error()?;
+        Ok(VectorQuery { inner })
     }
 
     #[napi]
-    pub fn nearest_to(&mut self, vector: Float32Array) {
-        self.inner = self.inner.clone().nearest_to(&vector);
+    pub async fn execute(&self) -> napi::Result<RecordBatchIterator> {
+        let inner_stream = self.inner.execute().await.map_err(|e| {
+            napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
+        })?;
+        Ok(RecordBatchIterator::new(inner_stream))
+    }
+}
+
+#[napi]
+pub struct VectorQuery {
+    inner: LanceDbVectorQuery,
+}
+
+#[napi]
+impl VectorQuery {
+    #[napi]
+    pub fn column(&mut self, column: String) {
+        self.inner = self.inner.clone().column(&column);
+    }
+
+    #[napi]
+    pub fn distance_type(&mut self, distance_type: String) -> napi::Result<()> {
+        let distance_type = parse_distance_type(distance_type)?;
+        self.inner = self.inner.clone().distance_type(distance_type);
+        Ok(())
+    }
+
+    #[napi]
+    pub fn postfilter(&mut self) {
+        self.inner = self.inner.clone().postfilter();
     }
 
     #[napi]
@@ -70,10 +105,30 @@ impl Query {
     }
 
     #[napi]
-    pub async fn execute_stream(&self) -> napi::Result<RecordBatchIterator> {
-        let inner_stream = self.inner.execute_stream().await.map_err(|e| {
+    pub fn bypass_vector_index(&mut self) {
+        self.inner = self.inner.clone().bypass_vector_index()
+    }
+
+    #[napi]
+    pub fn only_if(&mut self, predicate: String) {
+        self.inner = self.inner.clone().only_if(predicate);
+    }
+
+    #[napi]
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner = self.inner.clone().select(Select::dynamic(&columns));
+    }
+
+    #[napi]
+    pub fn limit(&mut self, limit: u32) {
+        self.inner = self.inner.clone().limit(limit as usize);
+    }
+
+    #[napi]
+    pub async fn execute(&self) -> napi::Result<RecordBatchIterator> {
+        let inner_stream = self.inner.execute().await.map_err(|e| {
             napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
         })?;
-        Ok(RecordBatchIterator::new(Box::new(inner_stream)))
+        Ok(RecordBatchIterator::new(inner_stream))
     }
 }

nodejs/src/table.rs

@@ -13,14 +13,17 @@
 // limitations under the License.
 
 use arrow_ipc::writer::FileWriter;
-use lance::dataset::ColumnAlteration as LanceColumnAlteration;
 use lancedb::ipc::ipc_file_to_batches;
-use lancedb::table::{AddDataMode, Table as LanceDbTable};
+use lancedb::table::{
+    AddDataMode, ColumnAlteration as LanceColumnAlteration, NewColumnTransform,
+    Table as LanceDbTable,
+};
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
 
-use crate::index::IndexBuilder;
-use crate::query::Query;
+use crate::error::NapiErrorExt;
+use crate::index::Index;
+use crate::query::{Query, VectorQuery};
 
 #[napi]
 pub struct Table {
@@ -86,7 +89,7 @@ impl Table {
     pub async fn add(&self, buf: Buffer, mode: String) -> napi::Result<()> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
-        let mut op = self.inner_ref()?.add(Box::new(batches));
+        let mut op = self.inner_ref()?.add(batches);
 
         op = if mode == "append" {
             op.mode(AddDataMode::Append)
@@ -129,8 +132,38 @@ impl Table {
     }
 
     #[napi]
-    pub fn create_index(&self) -> napi::Result<IndexBuilder> {
-        Ok(IndexBuilder::new(self.inner_ref()?))
+    pub async fn create_index(
+        &self,
+        index: Option<&Index>,
+        column: String,
+        replace: Option<bool>,
+    ) -> napi::Result<()> {
+        let lancedb_index = if let Some(index) = index {
+            index.consume()?
+        } else {
+            lancedb::index::Index::Auto
+        };
+        let mut builder = self.inner_ref()?.create_index(&[column], lancedb_index);
+        if let Some(replace) = replace {
+            builder = builder.replace(replace);
+        }
+        builder.execute().await.default_error()
+    }
+
+    #[napi]
+    pub async fn update(
+        &self,
+        only_if: Option<String>,
+        columns: Vec<(String, String)>,
+    ) -> napi::Result<()> {
+        let mut op = self.inner_ref()?.update();
+        if let Some(only_if) = only_if {
+            op = op.only_if(only_if);
+        }
+        for (column_name, value) in columns {
+            op = op.column(column_name, value);
+        }
+        op.execute().await.default_error()
     }
 
     #[napi]
@@ -138,13 +171,18 @@ impl Table {
         Ok(Query::new(self.inner_ref()?.query()))
     }
 
+    #[napi]
+    pub fn vector_search(&self, vector: Float32Array) -> napi::Result<VectorQuery> {
+        self.query()?.nearest_to(vector)
+    }
+
     #[napi]
     pub async fn add_columns(&self, transforms: Vec<AddColumnsSql>) -> napi::Result<()> {
         let transforms = transforms
             .into_iter()
             .map(|sql| (sql.name, sql.value_sql))
             .collect::<Vec<_>>();
-        let transforms = lance::dataset::NewColumnTransform::SqlExpressions(transforms);
+        let transforms = NewColumnTransform::SqlExpressions(transforms);
         self.inner_ref()?
             .add_columns(transforms, None)
             .await
@@ -197,6 +235,67 @@ impl Table {
             })?;
         Ok(())
     }
+
+    #[napi]
+    pub async fn version(&self) -> napi::Result<i64> {
+        self.inner_ref()?
+            .version()
+            .await
+            .map(|val| val as i64)
+            .default_error()
+    }
+
+    #[napi]
+    pub async fn checkout(&self, version: i64) -> napi::Result<()> {
+        self.inner_ref()?
+            .checkout(version as u64)
+            .await
+            .default_error()
+    }
+
+    #[napi]
+    pub async fn checkout_latest(&self) -> napi::Result<()> {
+        self.inner_ref()?.checkout_latest().await.default_error()
+    }
+
+    #[napi]
+    pub async fn restore(&self) -> napi::Result<()> {
+        self.inner_ref()?.restore().await.default_error()
+    }
+
+    #[napi]
+    pub async fn list_indices(&self) -> napi::Result<Vec<IndexConfig>> {
+        Ok(self
+            .inner_ref()?
+            .list_indices()
+            .await
+            .default_error()?
+            .into_iter()
+            .map(IndexConfig::from)
+            .collect::<Vec<_>>())
+    }
+}
+
+#[napi(object)]
+/// A description of an index currently configured on a column
+pub struct IndexConfig {
+    /// The type of the index
+    pub index_type: String,
+    /// The columns in the index
+    ///
+    /// Currently this is always an array of size 1.  In the future there may
+    /// be more columns to represent composite indices.
+    pub columns: Vec<String>,
+}
+
+impl From<lancedb::index::IndexConfig> for IndexConfig {
+    fn from(value: lancedb::index::IndexConfig) -> Self {
+        let index_type = format!("{:?}", value.index_type);
+        Self {
+            index_type,
+            columns: value.columns,
+        }
+    }
 }
 
 ///  A definition of a column alteration. The alteration changes the column at

nodejs/src/util.rs

@@ -0,0 +1,13 @@
+use lancedb::DistanceType;
+
+pub fn parse_distance_type(distance_type: impl AsRef<str>) -> napi::Result<DistanceType> {
+    match distance_type.as_ref().to_lowercase().as_str() {
+        "l2" => Ok(DistanceType::L2),
+        "cosine" => Ok(DistanceType::Cosine),
+        "dot" => Ok(DistanceType::Dot),
+        _ => Err(napi::Error::from_reason(format!(
+            "Invalid distance type '{}'.  Must be one of l2, cosine, or dot",
+            distance_type.as_ref()
+        ))),
+    }
+}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.6.2
+current_version = 0.6.5
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/Cargo.toml

@@ -22,6 +22,9 @@ pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
 
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "*", features = ["static"] }
+pin-project = "1.1.5"
+futures.workspace = true
+tokio = { version = "1.36.0", features = ["sync"] }
 
 [build-dependencies]
 pyo3-build-config = { version = "0.20.3", features = [

python/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "lancedb"
-version = "0.6.2"
+version = "0.6.5"
 dependencies = [
     "deprecation",
-    "pylance==0.10.2",
+    "pylance==0.10.5",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
     "tqdm>=4.27.0",
@@ -81,6 +81,7 @@ embeddings = [
     "awscli>=1.29.57",
     "botocore>=1.31.57",
 ]
+azure = ["adlfs>=2024.2.0"]
 
 [tool.maturin]
 python-source = "python"
@@ -93,13 +94,11 @@ lancedb = "lancedb.cli.cli:cli"
 requires = ["maturin>=1.4"]
 build-backend = "maturin"
 
-
 [tool.ruff.lint]
 select = ["F", "E", "W", "I", "G", "TCH", "PERF"]
 
 [tool.pytest.ini_options]
 addopts = "--strict-markers --ignore-glob=lancedb/embeddings/*.py"
-
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "asyncio",

python/python/lancedb/__init__.py

@@ -23,8 +23,9 @@ from ._lancedb import connect as lancedb_connect
 from .common import URI, sanitize_uri
 from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
-from .schema import vector  # noqa: F401
-from .utils import sentry_log  # noqa: F401
+from .schema import vector
+from .table import AsyncTable
+from .utils import sentry_log
 
 
 def connect(
@@ -35,6 +36,7 @@ def connect(
     host_override: Optional[str] = None,
     read_consistency_interval: Optional[timedelta] = None,
     request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
+    **kwargs,
 ) -> DBConnection:
     """Connect to a LanceDB database.
 
@@ -99,7 +101,12 @@ def connect(
         if isinstance(request_thread_pool, int):
             request_thread_pool = ThreadPoolExecutor(request_thread_pool)
         return RemoteDBConnection(
-            uri, api_key, region, host_override, request_thread_pool=request_thread_pool
+            uri,
+            api_key,
+            region,
+            host_override,
+            request_thread_pool=request_thread_pool,
+            **kwargs,
         )
     return LanceDBConnection(uri, read_consistency_interval=read_consistency_interval)
 
@@ -138,34 +145,20 @@ async def connect_async(
         the last check, then the table will be checked for updates. Note: this
         consistency only applies to read operations. Write operations are
         always consistent.
-    request_thread_pool: int or ThreadPoolExecutor, optional
-        The thread pool to use for making batch requests to the LanceDB Cloud API.
-        If an integer, then a ThreadPoolExecutor will be created with that
-        number of threads. If None, then a ThreadPoolExecutor will be created
-        with the default number of threads. If a ThreadPoolExecutor, then that
-        executor will be used for making requests. This is for LanceDB Cloud
-        only and is only used when making batch requests (i.e., passing in
-        multiple queries to the search method at once).
 
     Examples
     --------
 
-    For a local directory, provide a path for the database:
-
     >>> import lancedb
-    >>> db = lancedb.connect("~/.lancedb")
-
-    For object storage, use a URI prefix:
-
-    >>> db = lancedb.connect("s3://my-bucket/lancedb")
-
-    Connect to LancdDB cloud:
-
-    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
+    >>> async def doctest_example():
+    ...     # For a local directory, provide a path to the database
+    ...     db = await lancedb.connect_async("~/.lancedb")
+    ...     # For object storage, use a URI prefix
+    ...     db = await lancedb.connect_async("s3://my-bucket/lancedb")
 
     Returns
     -------
-    conn : DBConnection
+    conn : AsyncConnection
         A connection to a LanceDB database.
     """
     if read_consistency_interval is not None:
@@ -182,3 +175,19 @@ async def connect_async(
             read_consistency_interval_secs,
         )
     )
+
+
+__all__ = [
+    "connect",
+    "connect_async",
+    "AsyncConnection",
+    "AsyncTable",
+    "URI",
+    "sanitize_uri",
+    "sentry_log",
+    "vector",
+    "DBConnection",
+    "LanceDBConnection",
+    "RemoteDBConnection",
+    "__version__",
+]

python/python/lancedb/_lancedb.pyi

@@ -1,7 +1,19 @@
-from typing import Optional
+from typing import Dict, List, Optional, Tuple
 
 import pyarrow as pa
 
+class Index:
+    @staticmethod
+    def ivf_pq(
+        distance_type: Optional[str],
+        num_partitions: Optional[int],
+        num_sub_vectors: Optional[int],
+        max_iterations: Optional[int],
+        sample_rate: Optional[int],
+    ) -> Index: ...
+    @staticmethod
+    def btree() -> Index: ...
+
 class Connection(object):
     async def table_names(
         self, start_after: Optional[str], limit: Optional[int]
@@ -13,10 +25,27 @@ class Connection(object):
         self, name: str, mode: str, schema: pa.Schema
     ) -> Table: ...
 
-class Table(object):
+class Table:
     def name(self) -> str: ...
     def __repr__(self) -> str: ...
     async def schema(self) -> pa.Schema: ...
+    async def add(self, data: pa.RecordBatchReader, mode: str) -> None: ...
+    async def update(self, updates: Dict[str, str], where: Optional[str]) -> None: ...
+    async def count_rows(self, filter: Optional[str]) -> int: ...
+    async def create_index(
+        self, column: str, config: Optional[Index], replace: Optional[bool]
+    ): ...
+    async def version(self) -> int: ...
+    async def checkout(self, version): ...
+    async def checkout_latest(self): ...
+    async def restore(self): ...
+    async def list_indices(self) -> List[IndexConfig]: ...
+    def query(self) -> Query: ...
+    def vector_search(self) -> VectorQuery: ...
+
+class IndexConfig:
+    index_type: str
+    columns: List[str]
 
 async def connect(
     uri: str,
@@ -25,3 +54,27 @@ async def connect(
     host_override: Optional[str],
     read_consistency_interval: Optional[float],
 ) -> Connection: ...
+
+class RecordBatchStream:
+    def schema(self) -> pa.Schema: ...
+    async def next(self) -> Optional[pa.RecordBatch]: ...
+
+class Query:
+    def where(self, filter: str): ...
+    def select(self, columns: Tuple[str, str]): ...
+    def limit(self, limit: int): ...
+    def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
+    async def execute(self) -> RecordBatchStream: ...
+
+class VectorQuery:
+    async def execute(self) -> RecordBatchStream: ...
+    def where(self, filter: str): ...
+    def select(self, columns: List[str]): ...
+    def select_with_projection(self, columns: Tuple[str, str]): ...
+    def limit(self, limit: int): ...
+    def column(self, column: str): ...
+    def distance_type(self, distance_type: str): ...
+    def postfilter(self): ...
+    def refine_factor(self, refine_factor: int): ...
+    def nprobes(self, nprobes: int): ...
+    def bypass_vector_index(self): ...

python/python/lancedb/arrow.py

@@ -0,0 +1,44 @@
+from typing import List
+
+import pyarrow as pa
+
+from ._lancedb import RecordBatchStream
+
+
+class AsyncRecordBatchReader:
+    """
+    An async iterator over a stream of RecordBatches.
+
+    Also allows access to the schema of the stream
+    """
+
+    def __init__(self, inner: RecordBatchStream):
+        self.inner_ = inner
+
+    @property
+    def schema(self) -> pa.Schema:
+        """
+        Get the schema of the batches produced by the stream
+
+        Accessing the schema does not consume any data from the stream
+        """
+        return self.inner_.schema()
+
+    async def read_all(self) -> List[pa.RecordBatch]:
+        """
+        Read all the record batches from the stream
+
+        This consumes the entire stream and returns a list of record batches
+
+        If there are a lot of results this may consume a lot of memory
+        """
+        return [batch async for batch in self]
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> pa.RecordBatch:
+        next = await self.inner_.next()
+        if next is None:
+            raise StopAsyncIteration
+        return next

python/python/lancedb/db.py

@@ -25,13 +25,18 @@ from overrides import EnforceOverrides, override
 from pyarrow import fs
 
 from lancedb.common import data_to_reader, validate_schema
-from lancedb.embeddings.registry import EmbeddingFunctionRegistry
 from lancedb.utils.events import register_event
 
 from ._lancedb import connect as lancedb_connect
 from .pydantic import LanceModel
 from .table import AsyncTable, LanceTable, Table, _sanitize_data
-from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
+from .util import (
+    fs_from_uri,
+    get_uri_location,
+    get_uri_scheme,
+    join_uri,
+    validate_table_name,
+)
 
 if TYPE_CHECKING:
     from datetime import timedelta
@@ -387,6 +392,7 @@ class LanceDBConnection(DBConnection):
         """
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
+        validate_table_name(name)
 
         tbl = LanceTable.create(
             self,
@@ -444,16 +450,17 @@ class LanceDBConnection(DBConnection):
 class AsyncConnection(object):
     """An active LanceDB connection
 
-    To obtain a connection you can use the [connect] function.
+    To obtain a connection you can use the [connect_async][lancedb.connect_async]
+    function.
 
     This could be a native connection (using lance) or a remote connection (e.g. for
     connecting to LanceDb Cloud)
 
     Local connections do not currently hold any open resources but they may do so in the
     future (for example, for shared cache or connections to catalog services) Remote
-    connections represent an open connection to the remote server.  The [close] method
-    can be used to release any underlying resources eagerly.  The connection can also
-    be used as a context manager:
+    connections represent an open connection to the remote server.  The
+    [close][lancedb.db.AsyncConnection.close] method can be used to release any
+    underlying resources eagerly.  The connection can also be used as a context manager.
 
     Connections can be shared on multiple threads and are expected to be long lived.
     Connections can also be used as a context manager, however, in many cases a single
@@ -464,10 +471,9 @@ class AsyncConnection(object):
     Examples
     --------
 
-    >>> import asyncio
     >>> import lancedb
-    >>> async def my_connect():
-    ...   with await lancedb.connect("/tmp/my_dataset") as conn:
+    >>> async def doctest_example():
+    ...   with await lancedb.connect_async("/tmp/my_dataset") as conn:
     ...     # do something with the connection
     ...     pass
     ...   # conn is closed here
@@ -528,9 +534,8 @@ class AsyncConnection(object):
         exist_ok: Optional[bool] = None,
         on_bad_vectors: Optional[str] = None,
         fill_value: Optional[float] = None,
-        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
-    ) -> Table:
-        """Create a [Table][lancedb.table.Table] in the database.
+    ) -> AsyncTable:
+        """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
 
         Parameters
         ----------
@@ -569,7 +574,7 @@ class AsyncConnection(object):
 
         Returns
         -------
-        LanceTable
+        AsyncTable
             A reference to the newly created table.
 
         !!! note
@@ -583,12 +588,14 @@ class AsyncConnection(object):
         Can create with list of tuples or dictionaries:
 
         >>> import lancedb
-        >>> db = lancedb.connect("./.lancedb")
-        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        >>> async def doctest_example():
+        ...     db = await lancedb.connect_async("./.lancedb")
+        ...     data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
         ...             {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
-        >>> db.create_table("my_table", data)
-        LanceTable(connection=..., name="my_table")
-        >>> db["my_table"].head()
+        ...     my_table = await db.create_table("my_table", data)
+        ...     print(await my_table.query().limit(5).to_arrow())
+        >>> import asyncio
+        >>> asyncio.run(doctest_example())
         pyarrow.Table
         vector: fixed_size_list<item: float>[2]
           child 0, item: float
@@ -607,9 +614,11 @@ class AsyncConnection(object):
         ...    "lat": [45.5, 40.1],
         ...    "long": [-122.7, -74.1]
         ... })
-        >>> db.create_table("table2", data)
-        LanceTable(connection=..., name="table2")
-        >>> db["table2"].head()
+        >>> async def pandas_example():
+        ...     db = await lancedb.connect_async("./.lancedb")
+        ...     my_table = await db.create_table("table2", data)
+        ...     print(await my_table.query().limit(5).to_arrow())
+        >>> asyncio.run(pandas_example())
         pyarrow.Table
         vector: fixed_size_list<item: float>[2]
           child 0, item: float
@@ -629,9 +638,11 @@ class AsyncConnection(object):
         ...   pa.field("lat", pa.float32()),
         ...   pa.field("long", pa.float32())
         ... ])
-        >>> db.create_table("table3", data, schema = custom_schema)
-        LanceTable(connection=..., name="table3")
-        >>> db["table3"].head()
+        >>> async def with_schema():
+        ...     db = await lancedb.connect_async("./.lancedb")
+        ...     my_table = await db.create_table("table3", data, schema = custom_schema)
+        ...     print(await my_table.query().limit(5).to_arrow())
+        >>> asyncio.run(with_schema())
         pyarrow.Table
         vector: fixed_size_list<item: float>[2]
           child 0, item: float
@@ -663,9 +674,10 @@ class AsyncConnection(object):
         ...     pa.field("item", pa.utf8()),
         ...     pa.field("price", pa.float32()),
         ... ])
-        >>> db.create_table("table4", make_batches(), schema=schema)
-        LanceTable(connection=..., name="table4")
-
+        >>> async def iterable_example():
+        ...     db = await lancedb.connect_async("./.lancedb")
+        ...     await db.create_table("table4", make_batches(), schema=schema)
+        >>> asyncio.run(iterable_example())
         """
         if inspect.isclass(schema) and issubclass(schema, LanceModel):
             # convert LanceModel to pyarrow schema
@@ -674,12 +686,6 @@ class AsyncConnection(object):
             schema = schema.to_arrow_schema()
 
         metadata = None
-        if embedding_functions is not None:
-            # If we passed in embedding functions explicitly
-            # then we'll override any schema metadata that
-            # may was implicitly specified by the LanceModel schema
-            registry = EmbeddingFunctionRegistry.get_instance()
-            metadata = registry.get_table_metadata(embedding_functions)
 
         # Defining defaults here and not in function prototype.  In the future
         # these defaults will move into rust so better to keep them as None.
@@ -760,11 +766,11 @@ class AsyncConnection(object):
         name: str
             The name of the table.
         """
-        raise NotImplementedError
+        await self._inner.drop_table(name)
 
     async def drop_database(self):
         """
         Drop database
         This is the same thing as dropping all the tables
         """
-        raise NotImplementedError
+        await self._inner.drop_db()

python/python/lancedb/embeddings/imagebind.py

@@ -31,7 +31,7 @@ class ImageBindEmbeddings(EmbeddingFunction):
     six different modalities: images, text, audio, depth, thermal, and IMU data
 
     to download package, run :
-        `pip install imagebind@git+https://github.com/raghavdixit99/ImageBind`
+        `pip install imagebind-packaged==0.1.2`
     """
 
     name: str = "imagebind_huge"

python/python/lancedb/embeddings/openai.py

@@ -113,5 +113,5 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
         if self.organization:
             kwargs["organization"] = self.organization
         if self.api_key:
-            kwargs["api_key"] = self
+            kwargs["api_key"] = self.api_key
         return openai.OpenAI(**kwargs)