Bump version: 0.22.1-beta.6 → 0.22.1

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Chores**
- Updated workspace dependencies to use a stable release version for
improved consistency and reliability. No changes to application features
or functionality.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.18.0-beta.0"
+current_version = "0.19.1-beta.5"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -87,26 +87,11 @@ glob = "node/package.json"
 replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\""
 search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\""
 
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-linux-arm64-musl\": \"{new_version}\""
-search = "\"@lancedb/vectordb-linux-arm64-musl\": \"{current_version}\""
-
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-linux-x64-musl\": \"{new_version}\""
-search = "\"@lancedb/vectordb-linux-x64-musl\": \"{current_version}\""
-
 [[tool.bumpversion.files]]
 glob = "node/package.json"
 replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""
 search = "\"@lancedb/vectordb-win32-x64-msvc\": \"{current_version}\""
 
-[[tool.bumpversion.files]]
-glob = "node/package.json"
-replace = "\"@lancedb/vectordb-win32-arm64-msvc\": \"{new_version}\""
-search = "\"@lancedb/vectordb-win32-arm64-msvc\": \"{current_version}\""
-
 # Cargo files
 # ------------
 [[tool.bumpversion.files]]

.cargo/config.toml

@@ -34,6 +34,10 @@ rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"
 [target.x86_64-unknown-linux-musl]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
 
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"
+rustflags = ["-C", "target-feature=-crt-static"]
+
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
 
@@ -44,4 +48,4 @@ rustflags = ["-Ctarget-feature=+crt-static"]
 
 # Experimental target for Arm64 Windows
 [target.aarch64-pc-windows-msvc]
-rustflags = ["-Ctarget-feature=+crt-static"]
+rustflags = ["-Ctarget-feature=+crt-static"]

.github/workflows/build_linux_wheel/action.yml

@@ -36,8 +36,7 @@ runs:
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          yum install -y openssl-devel \
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip
     - name: Build Arm Manylinux Wheel
@@ -52,7 +51,7 @@ runs:
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          yum install -y openssl-devel clang \
+          yum install -y clang \
             && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip

.github/workflows/docs.yml

@@ -18,17 +18,24 @@ concurrency:
   group: "pages"
   cancel-in-progress: true
 
+env:
+  # This reduces the disk space needed for the build
+  RUSTFLAGS: "-C debuginfo=0"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+
 jobs:
   # Single deploy job since we're just deploying
   build:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install dependecies needed for ubuntu
+      - name: Install dependencies needed for ubuntu
         run: |
           sudo apt install -y protobuf-compiler libssl-dev
           rustup update && rustup default
@@ -38,6 +45,7 @@ jobs:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: "docs/requirements.txt"
+      - uses: Swatinem/rust-cache@v2
       - name: Build Python
         working-directory: python
         run: |
@@ -49,7 +57,6 @@ jobs:
           node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
       - name: Install node dependencies
         working-directory: node
         run: |

.github/workflows/java-publish.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: "1.79.0"
+          toolchain: "1.81.0"
           cache-workspaces: "./java/core/lancedb-jni"
           # Disable full debug symbol generation to speed up CI build and keep memory down
           # "1" means line tables only, which is useful for panic tracebacks.
@@ -97,7 +97,7 @@ jobs:
       - name: Dry run
         if: github.event_name == 'pull_request'
         run: |
-          mvn --batch-mode -DskipTests package
+          mvn --batch-mode -DskipTests -Drust.release.build=true package
       - name: Set github
         run: |
           git config --global user.email "LanceDB Github Runner"
@@ -108,7 +108,7 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
+          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}

.github/workflows/pypi-publish.yml

@@ -4,6 +4,11 @@ on:
   push:
     tags:
       - 'python-v*'
+  pull_request:
+    # This should trigger a dry run (we skip the final publish step)
+    paths:
+      - .github/workflows/pypi-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
 
 jobs:
   linux:
@@ -46,6 +51,7 @@ jobs:
           arm-build: ${{ matrix.config.platform == 'aarch64' }}
           manylinux: ${{ matrix.config.manylinux }}
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
@@ -75,6 +81,7 @@ jobs:
           python-minor-version: 8
           args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
@@ -96,10 +103,12 @@ jobs:
           args: "--release --strip"
           vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
       - uses: ./.github/workflows/upload_wheel
+        if: startsWith(github.ref, 'refs/tags/python-v')
         with:
           pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           fury_token: ${{ secrets.FURY_TOKEN }}
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/python-v')
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/python.yml

@@ -13,6 +13,11 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  # Color output for pytest is off by default.
+  PYTEST_ADDOPTS: "--color=yes"
+  FORCE_COLOR: "1"
+
 jobs:
   lint:
     name: "Lint"
@@ -131,6 +136,10 @@ jobs:
       - uses: ./.github/workflows/run_tests
         with:
           integration: true
+      - name: Test without pylance or pandas
+        run: |
+          pip uninstall -y pylance pandas
+          pytest -vv python/tests/test_table.py
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
@@ -219,6 +228,7 @@ jobs:
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
+          pip install pyarrow==16
           pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests

.github/workflows/rust.yml

@@ -40,6 +40,9 @@ jobs:
         with:
           fetch-depth: 0
           lfs: true
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt, clippy
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -157,153 +160,33 @@ jobs:
 
   windows:
     runs-on: windows-2022
+    strategy:
+      matrix:
+        target:
+          - x86_64-pc-windows-msvc
+          - aarch64-pc-windows-msvc
+    defaults:
+      run:
+        working-directory: rust/lancedb
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
       - name: Install Protoc v21.12
-        working-directory: C:\
+        run: choco install --no-progress protoc
+      - name: Build
         run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
+          rustup target add ${{ matrix.target }}
-          Set-Location C:\protoc
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          cargo build --features remote --tests --locked --target ${{ matrix.target }}
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
       - name: Run tests
+        # Can only run tests when target matches host
+        if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo test --features remote --locked
 
-  windows-arm64-cross:
-    # We cross compile in Node releases, so we want to make sure
-    # this can run successfully.
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies (part 1)
-        run: |
-          set -e
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-      - name: Install rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          target: aarch64-pc-windows-msvc
-      - name: Install dependencies (part 2)
-        run: |
-          set -e
-          mkdir -p sysroot
-          cd sysroot
-          sh ../ci/sysroot-aarch64-pc-windows-msvc.sh
-      - name: Check
-        env:
-          CC: clang
-          AR: llvm-ar
-          C_INCLUDE_PATH: /usr/aarch64-pc-windows-msvc/usr/include
-          CARGO_BUILD_TARGET: aarch64-pc-windows-msvc
-          RUSTFLAGS: -Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib
-        run: |
-          source $HOME/.cargo/env
-          cargo check --features remote --locked
-
-  windows-arm64:
-    runs-on: windows-4x-arm
-    steps:
-      - name: Install Git
-        run: |
-          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-        shell: powershell
-      - name: Add Git to PATH
-        run: |
-          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-        shell: powershell
-      - name: Configure Git symlinks
-        run: git config --global core.symlinks true
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-      - name: Install Visual Studio Build Tools
-        run: |
-          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-            "--installPath", "C:\BuildTools", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-        shell: powershell
-      - name: Add Visual Studio Build Tools to PATH
-        run: |
-          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
-
-          # Add MSVC runtime libraries to LIB
-          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
-                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
-                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
-
-          # Add INCLUDE paths
-          $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
-          Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
-        shell: powershell
-      - name: Install Rust
-        run: |
-          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc --default-toolchain 1.83.0
-        shell: powershell
-      - name: Add Rust to PATH
-        run: |
-          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-        shell: powershell
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install 7-Zip ARM
-        run: |
-          New-Item -Path 'C:\7zip' -ItemType Directory
-          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-        shell: powershell
-      - name: Add 7-Zip to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-        shell: powershell
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          if (Test-Path 'C:\protoc') {
-            Write-Host "Protoc directory exists, skipping installation"
-            return
-          }
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-        shell: powershell
-      - name: Add Protoc to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Run tests
-        run: |
-          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --target aarch64-pc-windows-msvc --features remote --locked
-
   msrv:
     # Check the minimum supported Rust version
     name: MSRV Check - Rust v${{ matrix.msrv }}

Cargo.toml

@@ -21,14 +21,14 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.24.1", "features" = ["dynamodb"] }
+lance = { "version" = "=0.27.2", "features" = ["dynamodb"] }
-lance-io = { version = "=0.24.1" }
+lance-io = { version = "=0.27.2" }
-lance-index = { version = "=0.24.1" }
+lance-index = { version = "=0.27.2" }
-lance-linalg = { version = "=0.24.1" }
+lance-linalg = { version = "=0.27.2" }
-lance-table = { version = "=0.24.1" }
+lance-table = { version = "=0.27.2" }
-lance-testing = { version = "=0.24.1" }
+lance-testing = { version = "=0.27.2" }
-lance-datafusion = { version = "=0.24.1" }
+lance-datafusion = { version = "=0.27.2" }
-lance-encoding = { version = "=0.24.1" }
+lance-encoding = { version = "=0.27.2" }
 # Note that this one does not include pyarrow
 arrow = { version = "54.1", optional = false }
 arrow-array = "54.1"
@@ -39,12 +39,12 @@ arrow-schema = "54.1"
 arrow-arith = "54.1"
 arrow-cast = "54.1"
 async-trait = "0"
-datafusion = { version = "45.0", default-features = false }
+datafusion = { version = "46.0", default-features = false }
-datafusion-catalog = "45.0"
+datafusion-catalog = "46.0"
-datafusion-common = { version = "45.0", default-features = false }
+datafusion-common = { version = "46.0", default-features = false }
-datafusion-execution = "45.0"
+datafusion-execution = "46.0"
-datafusion-expr = "45.0"
+datafusion-expr = "46.0"
-datafusion-physical-plan = "45.0"
+datafusion-physical-plan = "46.0"
 env_logger = "0.11"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",
@@ -61,12 +61,12 @@ rand = "0.8"
 regex = "1.10"
 lazy_static = "1"
 semver = "1.0.25"
-
 # Temporary pins to work around downstream issues
 # https://github.com/apache/arrow-rs/commit/2fddf85afcd20110ce783ed5b4cdeb82293da30b
 chrono = "=0.4.39"
 # https://github.com/RustCrypto/formats/issues/1684
 base64ct = "=1.6.0"
-
 # Workaround for: https://github.com/eira-fransham/crunchy/issues/13
 crunchy = "=0.2.2"
+# Workaround for: https://github.com/Lokathor/bytemuck/issues/306
+bytemuck_derive = ">=1.8.1, <1.9.0"

README.md

@@ -1,9 +1,17 @@
+<a href="https://cloud.lancedb.com" target="_blank">
+  <img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
+</a>
+
 <div align="center">
 <p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/user-attachments/assets/ac270358-333e-4bea-a132-acefaa94040e">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/user-attachments/assets/b864d814-0d29-4784-8fd9-807297c758c0">
+  <img alt="LanceDB Logo" src="https://github.com/user-attachments/assets/b864d814-0d29-4784-8fd9-807297c758c0" width=300>
+</picture>
 
-**Developer-friendly, database for multimodal AI**
+**Search More, Manage Less**
 
 <a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
 <a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>

ci/build_linux_artifacts_nodejs.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-ARCH=${1:-x86_64}
-
-# We pass down the current user so that when we later mount the local files
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_node
-docker build \
-    -t lancedb-node-manylinux-$ARCH \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
-
-# We turn on memory swap to avoid OOM killer
-docker run \
-    -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
-    lancedb-node-manylinux-$ARCH \
-    bash ci/manylinux_node/build_lancedb.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -1,34 +0,0 @@
-# Builds the macOS artifacts (nodejs binaries).
-# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-set -e
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/lancedb
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd nodejs
-    echo "Building nodejs library for $1"
-    export RUST_TARGET=$1
-    npm run build-release
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/manylinux_node/Dockerfile

@@ -1,5 +1,5 @@
 # Many linux dockerfile with Rust, Node, and Lance dependencies installed.
-# This container allows building the node modules native libraries in an 
+# This container allows building the node modules native libraries in an
 # environment with a very old glibc, so that we are compatible with a wide
 # range of linux distributions.
 ARG ARCH=x86_64
@@ -9,10 +9,6 @@ FROM quay.io/pypa/manylinux_2_28_${ARCH}
 ARG ARCH=x86_64
 ARG DOCKER_USER=default_user
 
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
 # Protobuf is also installed as root.
 COPY install_protobuf.sh install_protobuf.sh
 RUN ./install_protobuf.sh ${ARCH}
@@ -21,7 +17,7 @@ ENV DOCKER_USER=${DOCKER_USER}
 # Create a group and user, but only if it doesn't exist
 RUN echo ${ARCH} && id -u ${DOCKER_USER} >/dev/null 2>&1 || adduser --user-group --create-home --uid ${DOCKER_USER} build_user
 
-# We switch to the user to install Rust and Node, since those like to be 
+# We switch to the user to install Rust and Node, since those like to be
 # installed at the user level.
 USER ${DOCKER_USER}
 

ci/manylinux_node/build_lancedb.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-#Alpine doesn't have .bashrc
-FILE=$HOME/.bashrc && test -f $FILE && source $FILE
-
-cd nodejs
-npm ci
-npm run build-release

ci/manylinux_node/build_vectordb.sh

@@ -4,14 +4,6 @@ set -e
 ARCH=${1:-x86_64}
 TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
 #Alpine doesn't have .bashrc
 FILE=$HOME/.bashrc && test -f $FILE && source $FILE
 

ci/manylinux_node/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1v \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

docs/README.md

@@ -2,7 +2,7 @@
 
 LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
 
-Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+Docs is built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
 unreleased features.
 

docs/mkdocs.yml

@@ -124,6 +124,9 @@ nav:
               - Overview: hybrid_search/hybrid_search.md
               - Comparing Rerankers: hybrid_search/eval.md
               - Airbnb financial data example: notebooks/hybrid_search.ipynb
+          - Late interaction with MultiVector search:
+              - Overview: guides/multi-vector.md
+              - Example: notebooks/Multivector_on_LanceDB.ipynb
           - RAG:
               - Vanilla RAG: rag/vanilla_rag.md
               - Multi-head RAG: rag/multi_head_rag.md
@@ -202,6 +205,7 @@ nav:
           - PromptTools: integrations/prompttools.md
           - dlt: integrations/dlt.md
           - phidata: integrations/phidata.md
+          - Genkit: integrations/genkit.md
       - 🎯 Examples:
           - Overview: examples/index.md
           - 🐍 Python:
@@ -233,13 +237,6 @@ nav:
           - 👾 JavaScript (vectordb): javascript/modules.md
           - 👾 JavaScript (lancedb): js/globals.md
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
-      - ☁️ LanceDB Cloud:
-          - Overview: cloud/index.md
-          - API reference:
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
-          - FAQs: cloud/cloud_faq.md
 
   - Quick start: basic.md
   - Concepts:
@@ -260,6 +257,9 @@ nav:
           - Overview: hybrid_search/hybrid_search.md
           - Comparing Rerankers: hybrid_search/eval.md
           - Airbnb financial data example: notebooks/hybrid_search.ipynb
+      - Late interaction with MultiVector search:
+          - Overview: guides/multi-vector.md
+          - Document search Example: notebooks/Multivector_on_LanceDB.ipynb
       - RAG:
           - Vanilla RAG: rag/vanilla_rag.md
           - Multi-head RAG: rag/multi_head_rag.md
@@ -332,6 +332,7 @@ nav:
       - PromptTools: integrations/prompttools.md
       - dlt: integrations/dlt.md
       - phidata: integrations/phidata.md
+      - Genkit: integrations/genkit.md
   - Examples:
       - examples/index.md
       - 🐍 Python:
@@ -363,13 +364,6 @@ nav:
       - Javascript (vectordb): javascript/modules.md
       - Javascript (lancedb): js/globals.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
-  - LanceDB Cloud:
-      - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
-      - FAQs: cloud/cloud_faq.md
 
 extra_css:
   - styles/global.css

docs/openapi.yml

@@ -171,7 +171,7 @@ paths:
                 distance_type:
                   type: string
                   description: |
-                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
+                    The distance metric to use for search. l2, Cosine, Dot and Hamming are supported. Default is l2.
                 bypass_vector_index:
                   type: boolean
                   description: |
@@ -450,7 +450,7 @@ paths:
                   type: string
                   nullable: false
                   description: |
-                    The metric type to use for the index. L2, Cosine, Dot are supported.
+                    The metric type to use for the index. l2, Cosine, Dot are supported.
                 index_type:
                   type: string
       responses:

docs/src/ann_indexes.md

@@ -69,7 +69,7 @@ Lance supports `IVF_PQ` index type by default.
 
 The following IVF_PQ paramters can be specified:
 
-- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
+- **distance_type**: The distance metric to use. By default it uses euclidean distance "`l2`".
   We also support "cosine" and "dot" distance as well.
 - **num_partitions**: The number of partitions in the index. The default is the square root
   of the number of rows.

docs/src/cloud/index.md

@@ -2,7 +2,7 @@
 
 LanceDB Cloud is a SaaS (software-as-a-service) solution that runs serverless in the cloud, clearly separating storage from compute. It's designed to be highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
 
-[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
+[Try out LanceDB Cloud (Public Beta)](https://cloud.lancedb.com){ .md-button .md-button--primary }
 
 ## Architecture
 

docs/src/concepts/index_hnsw.md

@@ -59,7 +59,7 @@ Then the greedy search routine operates as follows:
 
 There are three key parameters to set when constructing an HNSW index:
 
-* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `metric`: Use an `l2` euclidean distance metric. We also support `dot` and `cosine` distance.
 * `m`: The number of neighbors to select for each vector in the HNSW graph.
 * `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph.
 

docs/src/concepts/index_ivfpq.md

@@ -47,7 +47,7 @@ We can combine the above concepts to understand how to build and query an IVF-PQ
 
 There are three key parameters to set when constructing an IVF-PQ index:
 
-* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `metric`: Use an `l2` euclidean distance metric. We also support `dot` and `cosine` distance.
 * `num_partitions`: The number of partitions in the IVF portion of the index.
 * `num_sub_vectors`: The number of sub-vectors that will be created during Product Quantization (PQ).
 
@@ -56,7 +56,7 @@ In Python, the index can be created as follows:
 ```python
 # Create and train the index for a 1536-dimensional vector
 # Make sure you have enough data in the table for an effective training step
-tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
+tbl.create_index(metric="l2", num_partitions=256, num_sub_vectors=96)
 ```
 !!! note
     `num_partitions`=256 and `num_sub_vectors`=96 does not work for every dataset. Those values needs to be adjusted for your particular dataset.

docs/src/embeddings/understanding_embeddings.md

@@ -54,7 +54,7 @@ As mentioned, after creating embedding, each data point is represented as a vect
 
 Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar. To quantify this closeness, we use distance as a metric which can be measured in the  following way - 
 
-1. **Euclidean Distance (L2)**: It calculates the straight-line distance between two points (vectors) in a multidimensional space.
+1. **Euclidean Distance (l2)**: It calculates the straight-line distance between two points (vectors) in a multidimensional space.
 2. **Cosine Similarity**: It measures the cosine of the angle between two vectors, providing a normalized measure of similarity based on their direction.
 3. **Dot product**: It is calculated as the sum of the products of their corresponding components. To measure relatedness it considers both the magnitude and direction of the vectors.
 

docs/src/examples/index.md

@@ -8,15 +8,5 @@ LanceDB provides language APIs, allowing you to embed a database in your languag
 * 👾 [JavaScript](examples_js.md) examples
 * 🦀 Rust examples (coming soon)
 
-## Python Applications powered by LanceDB
+!!! tip "Hosted LanceDB"
-
+    If you want S3 cost-efficiency and local performance via a simple serverless API, checkout **LanceDB Cloud**. For private deployments, high performance at extreme scale, or if you have strict security requirements, talk to us about **LanceDB Enterprise**. [Learn more](https://docs.lancedb.com/)
-| Project Name | Description |
-| --- | --- |
-| **Ultralytics Explorer 🚀**<br>[![Ultralytics](https://img.shields.io/badge/Ultralytics-Docs-green?labelColor=0f3bc4&style=flat-square&logo=https://cdn.prod.website-files.com/646dd1f1a3703e451ba81ecc/64994922cf2a6385a4bf4489_UltralyticsYOLO_mark_blue.svg&link=https://docs.ultralytics.com/datasets/explorer/)](https://docs.ultralytics.com/datasets/explorer/)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ultralytics/ultralytics/blob/main/docs/en/datasets/explorer/explorer.ipynb) |  - 🔍 **Explore CV Datasets**: Semantic search, SQL queries, vector similarity, natural language.<br>- 🖥️ **GUI & Python API**: Seamless dataset interaction.<br>- ⚡ **Efficient & Scalable**: Leverages LanceDB for large datasets.<br>- 📊 **Detailed Analysis**: Easily analyze data patterns.<br>- 🌐 **Browser GUI Demo**: Create embeddings, search images, run queries. |
-| **Website Chatbot🤖**<br>[![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/lancedb-vercel-chatbot)<br>[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&amp;env=OPENAI_API_KEY&amp;envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&amp;project-name=lancedb-vercel-chatbot&amp;repository-name=lancedb-vercel-chatbot&amp;demo-title=LanceDB%20Chatbot%20Demo&amp;demo-description=Demo%20website%20chatbot%20with%20LanceDB.&amp;demo-url=https%3A%2F%2Flancedb.vercel.app&amp;demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png) | - 🌐 **Chatbot from Sitemap/Docs**: Create a chatbot using site or document context.<br>- 🚀 **Embed LanceDB in Next.js**: Lightweight, on-prem storage.<br>- 🧠 **AI-Powered Context Retrieval**: Efficiently access relevant data.<br>- 🔧 **Serverless & Native JS**: Seamless integration with Next.js.<br>- ⚡ **One-Click Deploy on Vercel**: Quick and easy setup.. |
-
-## Nodejs Applications powered by LanceDB
-
-| Project Name | Description |
-| --- | --- |
-| **Langchain Writing Assistant✍️ **<br>[![Github](../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/applications/node/lanchain_writing_assistant) |  - **📂 Data Source Integration**:  Use your own data by specifying data source file, and the app instantly processes it to provide insights. <br>- **🧠 Intelligent Suggestions**:  Powered by LangChain.js and LanceDB, it improves writing productivity and accuracy.  <br>- **💡 Enhanced Writing Experience**: It delivers real-time contextual insights and factual suggestions while the user writes. |

docs/src/guides/multi-vector.md

@@ -0,0 +1,85 @@
+# Late interaction & MultiVector embedding type
+Late interaction is a technique used in retrieval that calculates the relevance of a query to a document by comparing their multi-vector representations. The key difference between late interaction and other popular methods:
+
+![late interaction vs other methods](https://raw.githubusercontent.com/lancedb/assets/b035a0ceb2c237734e0d393054c146d289792339/docs/assets/integration/colbert-blog-interaction.svg)
+
+
+[ Illustration from https://jina.ai/news/what-is-colbert-and-late-interaction-and-why-they-matter-in-search/]
+
+<b>No interaction:</b> Refers to independently embedding the query and document, that are compared to calcualte similarity without any interaction between them. This is typically used in vector search operations.
+
+<b>Partial interaction</b> Refers to a specific approach where the similarity computation happens primarily between query vectors and document vectors, without extensive interaction between individual components of each. An example of this is dual-encoder models like BERT.
+
+<b>Early full interaction</b> Refers to techniques like cross-encoders that process query and docs in pairs with full interaction across various stages of encoding. This is a powerful, but relatively slower technique. Because it requires processing query and docs in pairs, doc embeddings can't be pre-computed for fast retrieval. This is why cross encoders are typically used as reranking models combined with vector search. Learn more about [LanceDB Reranking support](https://lancedb.github.io/lancedb/reranking/).
+
+<b>Late interaction</b> Late interaction is a technique that calculates the doc and query similarity independently and then the interaction or evaluation happens during the retrieval process. This is typically used in retrieval models like ColBERT. Unlike early interaction, It allows speeding up the retrieval process without compromising the depth of semantic analysis.
+
+## Internals of ColBERT 
+Let's take a look at the steps involved in performing late interaction based retrieval using ColBERT:
+
+• ColBERT employs BERT-based encoders for both queries `(fQ)` and documents `(fD)`
+• A single BERT model is shared between query and document encoders and special tokens distinguish input types: `[Q]` for queries and `[D]` for documents
+
+**Query Encoder (fQ):**
+• Query q is tokenized into WordPiece tokens: `q1, q2, ..., ql`. `[Q]` token is prepended right after BERT's `[CLS]` token
+• If query length < Nq, it's padded with [MASK] tokens up to Nq.
+• The padded sequence goes through BERT's transformer architecture
+• Final embeddings are L2-normalized.
+
+**Document Encoder (fD):**
+• Document d is tokenized into tokens `d1, d2, ..., dm`. `[D]` token is prepended after `[CLS]` token
+• Unlike queries, documents are NOT padded with `[MASK]` tokens
+• Document tokens are processed through BERT and the same linear layer
+
+**Late Interaction:**
+• Late interaction estimates relevance score `S(q,d)` using embedding `Eq` and `Ed`. Late interaction happens after independent encoding
+• For each query embedding, maximum similarity is computed against all document embeddings
+• The similarity measure can be cosine similarity or squared L2 distance
+
+**MaxSim Calculation:**
+```
+S(q,d) := Σ max(Eqi⋅EdjT)
+          i∈|Eq| j∈|Ed|
+```
+• This finds the best matching document embedding for each query embedding
+• Captures relevance based on strongest local matches between contextual embeddings
+
+## LanceDB MultiVector type
+LanceDB supports multivector type, this is useful when you have multiple vectors for a single item (e.g. with ColBert and ColPali).
+
+You can index on a column with multivector type and search on it, the query can be single vector or multiple vectors. For now, only cosine metric is supported for multivector search. The vector value type can be float16, float32 or float64. LanceDB integrateds [ConteXtualized Token Retriever(XTR)](https://arxiv.org/abs/2304.01982), which introduces a simple, yet novel, objective function that encourages the model to retrieve the most important document tokens first. 
+
+```python
+import lancedb
+import numpy as np
+import pyarrow as pa
+
+db = lancedb.connect("data/multivector_demo")
+schema = pa.schema(
+    [
+        pa.field("id", pa.int64()),
+        # float16, float32, and float64 are supported
+        pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
+    ]
+)
+data = [
+    {
+        "id": i,
+        "vector": np.random.random(size=(2, 256)).tolist(),
+    }
+    for i in range(1024)
+]
+tbl = db.create_table("my_table", data=data, schema=schema)
+
+# only cosine similarity is supported for multi-vectors
+tbl.create_index(metric="cosine")
+
+# query with single vector
+query = np.random.random(256).astype(np.float16)
+tbl.search(query).to_arrow()
+
+# query with multiple vectors
+query = np.random.random(size=(2, 256))
+tbl.search(query).to_arrow()
+```
+Find more about vector search in LanceDB [here](https://lancedb.github.io/lancedb/search/#multivector-type).

docs/src/guides/storage.md

@@ -342,7 +342,7 @@ For **read and write access**, LanceDB will need a policy such as:
             "Action": [
               "s3:PutObject",
               "s3:GetObject",
-              "s3:DeleteObject",
+              "s3:DeleteObject"
             ],
             "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
         },
@@ -374,7 +374,7 @@ For **read-only access**, LanceDB will need a policy such as:
         {
             "Effect": "Allow",
             "Action": [
-              "s3:GetObject",
+              "s3:GetObject"
             ],
             "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
         },

docs/src/guides/tables.md

@@ -765,7 +765,10 @@ This can be used to update zero to all rows depending on how many rows match the
         ];
         const tbl = await db.createTable("my_table", data)
 
-        await tbl.update({vector: [10, 10]}, { where: "x = 2"})
+        await tbl.update({ 
+            values: { vector: [10, 10] },
+            where: "x = 2"
+        });
         ```
 
     === "vectordb (deprecated)"
@@ -784,7 +787,10 @@ This can be used to update zero to all rows depending on how many rows match the
         ];
         const tbl = await db.createTable("my_table", data)
 
-        await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        await tbl.update({ 
+            where: "x = 2", 
+            values: { vector: [10, 10] } 
+        });
         ```
 
 #### Updating using a sql query

docs/src/index.md

@@ -4,6 +4,9 @@ LanceDB is an open-source vector database for AI that's designed to store, manag
 
 Both the database and the underlying data format are designed from the ground up to be **easy-to-use**, **scalable** and **cost-effective**.
 
+!!! tip "Hosted LanceDB"
+    If you want S3 cost-efficiency and local performance via a simple serverless API, checkout **LanceDB Cloud**. For private deployments, high performance at extreme scale, or if you have strict security requirements, talk to us about **LanceDB Enterprise**. [Learn more](https://docs.lancedb.com/)
+
 ![](assets/lancedb_and_lance.png)
 
 ## Truly multi-modal
@@ -20,7 +23,7 @@ LanceDB **OSS** is an **open-source**, batteries-included embedded vector databa
 
 LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverless in the cloud, making the storage clearly separated from compute. It's designed to be cost-effective and highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
 
-[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
+[Try out LanceDB Cloud (Public Beta) Now](https://cloud.lancedb.com){ .md-button .md-button--primary }
 
 ## Why use LanceDB?
 

docs/src/integrations/genkit.md

@@ -0,0 +1,183 @@
+### genkitx-lancedb
+This is a lancedb plugin for genkit framework. It allows you to use LanceDB for ingesting and rereiving data using genkit framework.
+
+![integration-banner-genkit](https://github.com/user-attachments/assets/a6cc28af-98e9-4425-b87c-7ab139bd7893)
+
+### Installation
+```bash
+pnpm install genkitx-lancedb
+```
+
+### Usage
+
+Adding LanceDB plugin to your genkit instance.
+
+```ts
+import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from 'genkitx-lancedb';
+import { textEmbedding004, vertexAI } from '@genkit-ai/vertexai';
+import { gemini } from '@genkit-ai/vertexai';
+import { z, genkit } from 'genkit';
+import { Document } from 'genkit/retriever';
+import { chunk } from 'llm-chunk';
+import { readFile } from 'fs/promises';
+import path from 'path';
+import pdf from 'pdf-parse/lib/pdf-parse';
+
+const ai = genkit({
+  plugins: [
+    // vertexAI provides the textEmbedding004 embedder
+    vertexAI(),
+
+    // the local vector store requires an embedder to translate from text to vector
+    lancedb([
+      {
+        dbUri: '.db', // optional lancedb uri, default to .db
+        tableName: 'table', // optional table name, default to table
+        embedder: textEmbedding004,
+      },
+    ]),
+  ],
+});
+```
+
+You can run this app with the following command:
+```bash
+genkit start -- tsx --watch src/index.ts
+```
+
+This'll add LanceDB as a retriever and indexer to the genkit instance. You can see it in the GUI view
+<img width="1710" alt="Screenshot 2025-05-11 at 7 21 05 PM" src="https://github.com/user-attachments/assets/e752f7f4-785b-4797-a11e-72ab06a531b7" />
+
+**Testing retrieval on a sample table**
+Let's see the raw retrieval results
+
+<img width="1710" alt="Screenshot 2025-05-11 at 7 21 05 PM" src="https://github.com/user-attachments/assets/b8d356ed-8421-4790-8fc0-d6af563b9657" />
+On running this query, you'll 5 results fetched from the lancedb table, where each result looks something like this:
+<img width="1417" alt="Screenshot 2025-05-11 at 7 21 18 PM" src="https://github.com/user-attachments/assets/77429525-36e2-4da6-a694-e58c1cf9eb83" />
+
+
+
+## Creating a custom RAG flow
+
+Now that we've seen how you can use LanceDB for in a genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retreiver with its outputs postprocessed an fed into an LLM for final response
+
+### Creating custom indexer flows
+You can also create custom indexer flows, utilizing more options and features provided by LanceDB.
+
+```ts
+export const menuPdfIndexer = lancedbIndexerRef({
+   // Using all defaults, for dbUri, tableName, and embedder, etc
+});
+
+const chunkingConfig = {
+  minLength: 1000,
+  maxLength: 2000,
+  splitter: 'sentence',
+  overlap: 100,
+  delimiters: '',
+} as any;
+
+
+async function extractTextFromPdf(filePath: string) {
+  const pdfFile = path.resolve(filePath);
+  const dataBuffer = await readFile(pdfFile);
+  const data = await pdf(dataBuffer);
+  return data.text;
+}
+
+export const indexMenu = ai.defineFlow(
+  {
+    name: 'indexMenu',
+    inputSchema: z.string().describe('PDF file path'),
+    outputSchema: z.void(),
+  },
+  async (filePath: string) => {
+    filePath = path.resolve(filePath);
+
+    // Read the pdf.
+    const pdfTxt = await ai.run('extract-text', () =>
+      extractTextFromPdf(filePath)
+    );
+
+    // Divide the pdf text into segments.
+    const chunks = await ai.run('chunk-it', async () =>
+      chunk(pdfTxt, chunkingConfig)
+    );
+
+    // Convert chunks of text into documents to store in the index.
+    const documents = chunks.map((text) => {
+      return Document.fromText(text, { filePath });
+    });
+
+    // Add documents to the index.
+    await ai.index({
+      indexer: menuPdfIndexer,
+      documents,
+      options: {
+        writeMode: WriteMode.Overwrite,
+      } as any
+    });
+  }
+);
+```
+
+<img width="1316" alt="Screenshot 2025-05-11 at 8 35 56 PM" src="https://github.com/user-attachments/assets/e2a20ce4-d1d0-4fa2-9a84-f2cc26e3a29f" />
+
+In your console, you can see the logs
+
+<img width="511" alt="Screenshot 2025-05-11 at 7 19 14 PM" src="https://github.com/user-attachments/assets/243f26c5-ed38-40b6-b661-002f40f0423a" />
+
+### Creating custom retriever flows
+You can also create custom retriever flows, utilizing more options and features provided by LanceDB.
+```ts
+export const menuRetriever = lancedbRetrieverRef({
+  tableName: "table", // Use the same table name as the indexer.
+  displayName: "Menu", // Use a custom display name.
+
+export const menuQAFlow = ai.defineFlow(
+  { name: "Menu", inputSchema: z.string(), outputSchema: z.string() },
+  async (input: string) => {
+    // retrieve relevant documents
+    const docs = await ai.retrieve({
+      retriever: menuRetriever,
+      query: input,
+      options: { 
+        k: 3,
+      },
+    });
+
+    const extractedContent = docs.map(doc => {
+      if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) {
+        if (doc.content[0].media && doc.content[0].media.url) {
+          return doc.content[0].media.url;
+        }
+      }
+      return "No content found";
+    });
+
+    console.log("Extracted content:", extractedContent);
+
+    const { text } = await ai.generate({
+      model: gemini('gemini-2.0-flash'),
+      prompt: `
+You are acting as a helpful AI assistant that can answer 
+questions about the food available on the menu at Genkit Grub Pub.
+
+Use only the context provided to answer the question.
+If you don't know, do not make up an answer.
+Do not add or change items on the menu.
+
+Context:
+${extractedContent.join('\n\n')}
+
+Question: ${input}`,
+      docs,
+    });
+    
+    return text;
+  }
+);
+```
+Now using our retrieval flow, we can ask question about the ingsted PDF
+<img width="1306" alt="Screenshot 2025-05-11 at 7 18 45 PM" src="https://github.com/user-attachments/assets/86c66b13-7c12-4d5f-9d81-ae36bfb1c346" />
+

docs/src/integrations/langchain.md

@@ -108,7 +108,7 @@ This method creates a scalar(for non-vector cols) or a vector index on a table.
 |:---|:---|:---|:---|
 |`vector_col`|`Optional[str]`| Provide if you want to create index on a vector column. |`None`|
 |`col_name`|`Optional[str]`| Provide if you want to create index on a non-vector column. |`None`|
-|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'L2', 'dot', 'cosine'. |`L2`|
+|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'l2', 'dot', 'cosine'. |`l2`|
 |`num_partitions`|`Optional[int]`|Number of partitions to use for the index.|`256`|
 |`num_sub_vectors`|`Optional[int]` |Number of sub-vectors to use for the index.|`96`|
 |`index_cache_size`|`Optional[int]` |Size of the index cache.|`None`|

docs/src/integrations/llamaIndex.md

@@ -125,7 +125,7 @@ The exhaustive list of parameters for `LanceDBVectorStore` vector store are :
         ```               
 - **_table_exists(self, tbl_name: `Optional[str]` = `None`) -> `bool`** : Returns `True` if `tbl_name` exists in database.
 - __create_index(  
-  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "L2",  
+  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "l2",  
 ) -> `None`__ : Creates a scalar(for non-vector cols) or a vector index on a table.
         Make sure your vector column has enough data before creating an index on it.
 

docs/src/javascript/enums/MetricType.md

@@ -10,7 +10,7 @@ Distance metrics type.
 
 - [Cosine](MetricType.md#cosine)
 - [Dot](MetricType.md#dot)
-- [L2](MetricType.md#l2)
+- [l2](MetricType.md#l2)
 
 ## Enumeration Members
 

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -85,7 +85,7 @@ ___
 
 • `Optional` **metric\_type**: [`MetricType`](../enums/MetricType.md)
 
-Metric type, L2 or Cosine
+Metric type, l2 or Cosine
 
 #### Defined in
 

docs/src/js/README.md

@@ -15,11 +15,9 @@ npm install @lancedb/lancedb
 This will download the appropriate native library for your platform. We currently
 support:
 
-- Linux (x86_64 and aarch64)
+- Linux (x86_64 and aarch64 on glibc and musl)
 - MacOS (Intel and ARM/M1/M2)
-- Windows (x86_64 only)
+- Windows (x86_64 and aarch64)
-
-We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
 
 ## Usage
 

docs/src/js/classes/BoostQuery.md

@@ -0,0 +1,67 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BoostQuery
+
+# Class: BoostQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new BoostQuery()
+
+```ts
+new BoostQuery(
+   positive,
+   negative,
+   options?): BoostQuery
+```
+
+Creates an instance of BoostQuery.
+The boost returns documents that match the positive query,
+but penalizes those that match the negative query.
+the penalty is controlled by the `negativeBoost` parameter.
+
+#### Parameters
+
+* **positive**: [`FullTextQuery`](../interfaces/FullTextQuery.md)
+    The positive query that boosts the relevance score.
+
+* **negative**: [`FullTextQuery`](../interfaces/FullTextQuery.md)
+    The negative query that reduces the relevance score.
+
+* **options?**
+    Optional parameters for the boost query.
+    - `negativeBoost`: The boost factor for the negative query (default is 0.0).
+
+* **options.negativeBoost?**: `number`
+
+#### Returns
+
+[`BoostQuery`](BoostQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/Index.md

@@ -126,6 +126,37 @@ the vectors.
 
 ***
 
+### ivfFlat()
+
+```ts
+static ivfFlat(options?): Index
+```
+
+Create an IvfFlat index
+
+This index groups vectors into partitions of similar vectors.  Each partition keeps track of
+a centroid which is the average value of all vectors in the group.
+
+During a query the centroids are compared with the query vector to find the closest
+partitions.  The vectors in these partitions are then searched to find
+the closest vectors.
+
+The partitioning process is called IVF and the `num_partitions` parameter controls how
+many groups to create.
+
+Note that training an IVF FLAT index on a large dataset is a slow operation and
+currently is also a memory intensive operation.
+
+#### Parameters
+
+* **options?**: `Partial`<[`IvfFlatOptions`](../interfaces/IvfFlatOptions.md)>
+
+#### Returns
+
+[`Index`](Index.md)
+
+***
+
 ### ivfPq()
 
 ```ts

docs/src/js/classes/MatchQuery.md

@@ -0,0 +1,70 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MatchQuery
+
+# Class: MatchQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new MatchQuery()
+
+```ts
+new MatchQuery(
+   query,
+   column,
+   options?): MatchQuery
+```
+
+Creates an instance of MatchQuery.
+
+#### Parameters
+
+* **query**: `string`
+    The text query to search for.
+
+* **column**: `string`
+    The name of the column to search within.
+
+* **options?**
+    Optional parameters for the match query.
+    - `boost`: The boost factor for the query (default is 1.0).
+    - `fuzziness`: The fuzziness level for the query (default is 0).
+    - `maxExpansions`: The maximum number of terms to consider for fuzzy matching (default is 50).
+
+* **options.boost?**: `number`
+
+* **options.fuzziness?**: `number`
+
+* **options.maxExpansions?**: `number`
+
+#### Returns
+
+[`MatchQuery`](MatchQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/MergeInsertBuilder.md

@@ -33,20 +33,22 @@ Construct a MergeInsertBuilder. __Internal use only.__
 ### execute()
 
 ```ts
-execute(data): Promise<void>
+execute(data, execOptions?): Promise<MergeResult>
 ```
 
 Executes the merge insert operation
 
-Nothing is returned but the `Table` is updated
-
 #### Parameters
 
 * **data**: [`Data`](../type-aliases/Data.md)
 
+* **execOptions?**: `Partial`&lt;[`WriteExecutionOptions`](../interfaces/WriteExecutionOptions.md)&gt;
+
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`MergeResult`](../interfaces/MergeResult.md)&gt;
+
+the merge result
 
 ***
 

docs/src/js/classes/MultiMatchQuery.md

@@ -0,0 +1,64 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MultiMatchQuery
+
+# Class: MultiMatchQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new MultiMatchQuery()
+
+```ts
+new MultiMatchQuery(
+   query,
+   columns,
+   options?): MultiMatchQuery
+```
+
+Creates an instance of MultiMatchQuery.
+
+#### Parameters
+
+* **query**: `string`
+    The text query to search for across multiple columns.
+
+* **columns**: `string`[]
+    An array of column names to search within.
+
+* **options?**
+    Optional parameters for the multi-match query.
+    - `boosts`: An array of boost factors for each column (default is 1.0 for all).
+
+* **options.boosts?**: `number`[]
+
+#### Returns
+
+[`MultiMatchQuery`](MultiMatchQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/PhraseQuery.md

@@ -0,0 +1,55 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / PhraseQuery
+
+# Class: PhraseQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new PhraseQuery()
+
+```ts
+new PhraseQuery(query, column): PhraseQuery
+```
+
+Creates an instance of `PhraseQuery`.
+
+#### Parameters
+
+* **query**: `string`
+    The phrase to search for in the specified column.
+
+* **column**: `string`
+    The name of the column to search within.
+
+#### Returns
+
+[`PhraseQuery`](PhraseQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/Query.md

@@ -30,6 +30,53 @@ protected inner: Query | Promise<Query>;
 
 ## Methods
 
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+
+***
+
 ### execute()
 
 ```ts
@@ -159,7 +206,7 @@ fullTextSearch(query, options?): this
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 
@@ -262,7 +309,7 @@ nearestToText(query, columns?): Query
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **columns?**: `string`[]
 

docs/src/js/classes/QueryBase.md

@@ -36,6 +36,49 @@ protected inner: NativeQueryType | Promise<NativeQueryType>;
 
 ## Methods
 
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+***
+
 ### execute()
 
 ```ts
@@ -149,7 +192,7 @@ fullTextSearch(query, options?): this
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 

docs/src/js/classes/Table.md

@@ -40,7 +40,7 @@ Returns the name of the table
 ### add()
 
 ```ts
-abstract add(data, options?): Promise<void>
+abstract add(data, options?): Promise<AddResult>
 ```
 
 Insert records into this Table.
@@ -54,14 +54,17 @@ Insert records into this Table.
 
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`AddResult`](../interfaces/AddResult.md)&gt;
+
+A promise that resolves to an object
+containing the new version number of the table
 
 ***
 
 ### addColumns()
 
 ```ts
-abstract addColumns(newColumnTransforms): Promise<void>
+abstract addColumns(newColumnTransforms): Promise<AddColumnsResult>
 ```
 
 Add new columns with defined values.
@@ -76,14 +79,17 @@ Add new columns with defined values.
 
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`AddColumnsResult`](../interfaces/AddColumnsResult.md)&gt;
+
+A promise that resolves to an object
+containing the new version number of the table after adding the columns.
 
 ***
 
 ### alterColumns()
 
 ```ts
-abstract alterColumns(columnAlterations): Promise<void>
+abstract alterColumns(columnAlterations): Promise<AlterColumnsResult>
 ```
 
 Alter the name or nullability of columns.
@@ -96,7 +102,10 @@ Alter the name or nullability of columns.
 
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`AlterColumnsResult`](../interfaces/AlterColumnsResult.md)&gt;
+
+A promise that resolves to an object
+containing the new version number of the table after altering the columns.
 
 ***
 
@@ -117,8 +126,8 @@ wish to return to standard mode, call `checkoutLatest`.
 
 #### Parameters
 
-* **version**: `number`
+* **version**: `string` \| `number`
-    The version to checkout
+    The version to checkout, could be version number or tag
 
 #### Returns
 
@@ -252,7 +261,7 @@ await table.createIndex("my_float_col");
 ### delete()
 
 ```ts
-abstract delete(predicate): Promise<void>
+abstract delete(predicate): Promise<DeleteResult>
 ```
 
 Delete the rows that satisfy the predicate.
@@ -263,7 +272,10 @@ Delete the rows that satisfy the predicate.
 
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`DeleteResult`](../interfaces/DeleteResult.md)&gt;
+
+A promise that resolves to an object
+containing the new version number of the table
 
 ***
 
@@ -284,7 +296,7 @@ Return a brief description of the table
 ### dropColumns()
 
 ```ts
-abstract dropColumns(columnNames): Promise<void>
+abstract dropColumns(columnNames): Promise<DropColumnsResult>
 ```
 
 Drop one or more columns from the dataset
@@ -303,7 +315,10 @@ then call ``cleanup_files`` to remove the old files.
 
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`DropColumnsResult`](../interfaces/DropColumnsResult.md)&gt;
+
+A promise that resolves to an object
+containing the new version number of the table after dropping the columns.
 
 ***
 
@@ -454,6 +469,28 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 ***
 
+### prewarmIndex()
+
+```ts
+abstract prewarmIndex(name): Promise<void>
+```
+
+Prewarm an index in the table.
+
+#### Parameters
+
+* **name**: `string`
+    The name of the index.
+    This will load the index into memory.  This may reduce the cold-start time for
+    future queries.  If the index does not fit in the cache then this call may be
+    wasteful.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### query()
 
 ```ts
@@ -575,7 +612,7 @@ of the given query
 
 #### Parameters
 
-* **query**: `string` \| [`IntoVector`](../type-aliases/IntoVector.md)
+* **query**: `string` \| [`IntoVector`](../type-aliases/IntoVector.md) \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
     the query, a vector or string
 
 * **queryType?**: `string`
@@ -593,6 +630,50 @@ of the given query
 
 ***
 
+### stats()
+
+```ts
+abstract stats(): Promise<TableStatistics>
+```
+
+Returns table and fragment statistics
+
+#### Returns
+
+`Promise`&lt;[`TableStatistics`](../interfaces/TableStatistics.md)&gt;
+
+The table and fragment statistics
+
+***
+
+### tags()
+
+```ts
+abstract tags(): Promise<Tags>
+```
+
+Get a tags manager for this table.
+
+Tags allow you to label specific versions of a table with a human-readable name.
+The returned tags manager can be used to list, create, update, or delete tags.
+
+#### Returns
+
+`Promise`&lt;[`Tags`](Tags.md)&gt;
+
+A tags manager for this table
+
+#### Example
+
+```typescript
+const tagsManager = await table.tags();
+await tagsManager.create("v1", 1);
+const tags = await tagsManager.list();
+console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
+```
+
+***
+
 ### toArrow()
 
 ```ts
@@ -612,7 +693,7 @@ Return the table as an arrow table
 #### update(opts)
 
 ```ts
-abstract update(opts): Promise<void>
+abstract update(opts): Promise<UpdateResult>
 ```
 
 Update existing records in the Table
@@ -623,7 +704,10 @@ Update existing records in the Table
 
 ##### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`UpdateResult`](../interfaces/UpdateResult.md)&gt;
+
+A promise that resolves to an object containing
+the number of rows updated and the new version number
 
 ##### Example
 
@@ -634,7 +718,7 @@ table.update({where:"x = 2", values:{"vector": [10, 10]}})
 #### update(opts)
 
 ```ts
-abstract update(opts): Promise<void>
+abstract update(opts): Promise<UpdateResult>
 ```
 
 Update existing records in the Table
@@ -645,7 +729,10 @@ Update existing records in the Table
 
 ##### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`UpdateResult`](../interfaces/UpdateResult.md)&gt;
+
+A promise that resolves to an object containing
+the number of rows updated and the new version number
 
 ##### Example
 
@@ -656,7 +743,7 @@ table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
 #### update(updates, options)
 
 ```ts
-abstract update(updates, options?): Promise<void>
+abstract update(updates, options?): Promise<UpdateResult>
 ```
 
 Update existing records in the Table
@@ -679,10 +766,6 @@ repeatedly calilng this method.
 * **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
     the
     columns to update
-    Keys in the map should specify the name of the column to update.
-    Values in the map provide the new value of the column.  These can
-    be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
-    based on the row being updated (e.g. "my_col + 1")
 
 * **options?**: `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
     additional options to control
@@ -690,7 +773,15 @@ repeatedly calilng this method.
 
 ##### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`UpdateResult`](../interfaces/UpdateResult.md)&gt;
+
+A promise that resolves to an object
+containing the number of rows updated and the new version number
+
+Keys in the map should specify the name of the column to update.
+Values in the map provide the new value of the column.  These can
+be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+based on the row being updated (e.g. "my_col + 1")
 
 ***
 
@@ -731,3 +822,26 @@ Retrieve the version of the table
 #### Returns
 
 `Promise`&lt;`number`&gt;
+
+***
+
+### waitForIndex()
+
+```ts
+abstract waitForIndex(indexNames, timeoutSeconds): Promise<void>
+```
+
+Waits for asynchronous indexing to complete on the table.
+
+#### Parameters
+
+* **indexNames**: `string`[]
+    The name of the indices to wait for
+
+* **timeoutSeconds**: `number`
+    The number of seconds to wait before timing out
+    This will raise an error if the indices are not created and fully indexed within the timeout.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;

docs/src/js/classes/TagContents.md

@@ -0,0 +1,35 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TagContents
+
+# Class: TagContents
+
+## Constructors
+
+### new TagContents()
+
+```ts
+new TagContents(): TagContents
+```
+
+#### Returns
+
+[`TagContents`](TagContents.md)
+
+## Properties
+
+### manifestSize
+
+```ts
+manifestSize: number;
+```
+
+***
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/classes/Tags.md

@@ -0,0 +1,99 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Tags
+
+# Class: Tags
+
+## Constructors
+
+### new Tags()
+
+```ts
+new Tags(): Tags
+```
+
+#### Returns
+
+[`Tags`](Tags.md)
+
+## Methods
+
+### create()
+
+```ts
+create(tag, version): Promise<void>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+* **version**: `number`
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
+### delete()
+
+```ts
+delete(tag): Promise<void>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
+### getVersion()
+
+```ts
+getVersion(tag): Promise<number>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+#### Returns
+
+`Promise`&lt;`number`&gt;
+
+***
+
+### list()
+
+```ts
+list(): Promise<Record<string, TagContents>>
+```
+
+#### Returns
+
+`Promise`&lt;`Record`&lt;`string`, [`TagContents`](TagContents.md)&gt;&gt;
+
+***
+
+### update()
+
+```ts
+update(tag, version): Promise<void>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+* **version**: `number`
+
+#### Returns
+
+`Promise`&lt;`void`&gt;

docs/src/js/classes/VectorQuery.md

@@ -48,6 +48,53 @@ addQueryVector(vector): VectorQuery
 
 ***
 
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+
+***
+
 ### bypassVectorIndex()
 
 ```ts
@@ -300,7 +347,7 @@ fullTextSearch(query, options?): this
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 

docs/src/js/enumerations/FullTextQueryType.md

@@ -0,0 +1,46 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FullTextQueryType
+
+# Enumeration: FullTextQueryType
+
+Enum representing the types of full-text queries supported.
+
+- `Match`: Performs a full-text search for terms in the query string.
+- `MatchPhrase`: Searches for an exact phrase match in the text.
+- `Boost`: Boosts the relevance score of specific terms in the query.
+- `MultiMatch`: Searches across multiple fields for the query terms.
+
+## Enumeration Members
+
+### Boost
+
+```ts
+Boost: "boost";
+```
+
+***
+
+### Match
+
+```ts
+Match: "match";
+```
+
+***
+
+### MatchPhrase
+
+```ts
+MatchPhrase: "match_phrase";
+```
+
+***
+
+### MultiMatch
+
+```ts
+MultiMatch: "multi_match";
+```

docs/src/js/functions/packBits.md

@@ -0,0 +1,19 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / packBits
+
+# Function: packBits()
+
+```ts
+function packBits(data): number[]
+```
+
+## Parameters
+
+* **data**: `number`[]
+
+## Returns
+
+`number`[]

docs/src/js/globals.md

@@ -9,37 +9,57 @@
 - [embedding](namespaces/embedding/README.md)
 - [rerankers](namespaces/rerankers/README.md)
 
+## Enumerations
+
+- [FullTextQueryType](enumerations/FullTextQueryType.md)
+
 ## Classes
 
+- [BoostQuery](classes/BoostQuery.md)
 - [Connection](classes/Connection.md)
 - [Index](classes/Index.md)
 - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
+- [MatchQuery](classes/MatchQuery.md)
 - [MergeInsertBuilder](classes/MergeInsertBuilder.md)
+- [MultiMatchQuery](classes/MultiMatchQuery.md)
+- [PhraseQuery](classes/PhraseQuery.md)
 - [Query](classes/Query.md)
 - [QueryBase](classes/QueryBase.md)
 - [RecordBatchIterator](classes/RecordBatchIterator.md)
 - [Table](classes/Table.md)
+- [TagContents](classes/TagContents.md)
+- [Tags](classes/Tags.md)
 - [VectorColumnOptions](classes/VectorColumnOptions.md)
 - [VectorQuery](classes/VectorQuery.md)
 
 ## Interfaces
 
+- [AddColumnsResult](interfaces/AddColumnsResult.md)
 - [AddColumnsSql](interfaces/AddColumnsSql.md)
 - [AddDataOptions](interfaces/AddDataOptions.md)
+- [AddResult](interfaces/AddResult.md)
+- [AlterColumnsResult](interfaces/AlterColumnsResult.md)
 - [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
 - [CompactionStats](interfaces/CompactionStats.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [CreateTableOptions](interfaces/CreateTableOptions.md)
+- [DeleteResult](interfaces/DeleteResult.md)
+- [DropColumnsResult](interfaces/DropColumnsResult.md)
 - [ExecutableQuery](interfaces/ExecutableQuery.md)
+- [FragmentStatistics](interfaces/FragmentStatistics.md)
+- [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
 - [FtsOptions](interfaces/FtsOptions.md)
+- [FullTextQuery](interfaces/FullTextQuery.md)
 - [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
 - [HnswPqOptions](interfaces/HnswPqOptions.md)
 - [HnswSqOptions](interfaces/HnswSqOptions.md)
 - [IndexConfig](interfaces/IndexConfig.md)
 - [IndexOptions](interfaces/IndexOptions.md)
 - [IndexStatistics](interfaces/IndexStatistics.md)
+- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
+- [MergeResult](interfaces/MergeResult.md)
 - [OpenTableOptions](interfaces/OpenTableOptions.md)
 - [OptimizeOptions](interfaces/OptimizeOptions.md)
 - [OptimizeStats](interfaces/OptimizeStats.md)
@@ -47,9 +67,12 @@
 - [RemovalStats](interfaces/RemovalStats.md)
 - [RetryConfig](interfaces/RetryConfig.md)
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
+- [TableStatistics](interfaces/TableStatistics.md)
 - [TimeoutConfig](interfaces/TimeoutConfig.md)
 - [UpdateOptions](interfaces/UpdateOptions.md)
+- [UpdateResult](interfaces/UpdateResult.md)
 - [Version](interfaces/Version.md)
+- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
 
 ## Type Aliases
 
@@ -66,3 +89,4 @@
 
 - [connect](functions/connect.md)
 - [makeArrowTable](functions/makeArrowTable.md)
+- [packBits](functions/packBits.md)

docs/src/js/interfaces/AddColumnsResult.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AddColumnsResult
+
+# Interface: AddColumnsResult
+
+## Properties
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/AddResult.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AddResult
+
+# Interface: AddResult
+
+## Properties
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/AlterColumnsResult.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AlterColumnsResult
+
+# Interface: AlterColumnsResult
+
+## Properties
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/ColumnAlteration.md

@@ -16,7 +16,7 @@ must be provided.
 ### dataType?
 
 ```ts
-optional dataType: string;
+optional dataType: string | DataType<Type, any>;
 ```
 
 A new data type for the column. If not provided then the data type will not be changed.

docs/src/js/interfaces/DeleteResult.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DeleteResult
+
+# Interface: DeleteResult
+
+## Properties
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/DropColumnsResult.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DropColumnsResult
+
+# Interface: DropColumnsResult
+
+## Properties
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/FragmentStatistics.md

@@ -0,0 +1,37 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FragmentStatistics
+
+# Interface: FragmentStatistics
+
+## Properties
+
+### lengths
+
+```ts
+lengths: FragmentSummaryStats;
+```
+
+Statistics on the number of rows in the table fragments
+
+***
+
+### numFragments
+
+```ts
+numFragments: number;
+```
+
+The number of fragments in the table
+
+***
+
+### numSmallFragments
+
+```ts
+numSmallFragments: number;
+```
+
+The number of uncompacted fragments in the table

docs/src/js/interfaces/FragmentSummaryStats.md

@@ -0,0 +1,77 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FragmentSummaryStats
+
+# Interface: FragmentSummaryStats
+
+## Properties
+
+### max
+
+```ts
+max: number;
+```
+
+The number of rows in the fragment with the most rows
+
+***
+
+### mean
+
+```ts
+mean: number;
+```
+
+The mean number of rows in the fragments
+
+***
+
+### min
+
+```ts
+min: number;
+```
+
+The number of rows in the fragment with the fewest rows
+
+***
+
+### p25
+
+```ts
+p25: number;
+```
+
+The 25th percentile of number of rows in the fragments
+
+***
+
+### p50
+
+```ts
+p50: number;
+```
+
+The 50th percentile of number of rows in the fragments
+
+***
+
+### p75
+
+```ts
+p75: number;
+```
+
+The 75th percentile of number of rows in the fragments
+
+***
+
+### p99
+
+```ts
+p99: number;
+```
+
+The 99th percentile of number of rows in the fragments

docs/src/js/interfaces/FullTextQuery.md

@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FullTextQuery
+
+# Interface: FullTextQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)

docs/src/js/interfaces/HnswPqOptions.md

@@ -24,18 +24,18 @@ The following distance types are available:
 
 "l2" - Euclidean distance. This is a very common distance metric that
 accounts for both magnitude and direction when determining the distance
-between vectors. L2 distance has a range of [0, ∞).
+between vectors. l2 distance has a range of [0, ∞).
 
 "cosine" - Cosine distance.  Cosine distance is a distance metric
 calculated from the cosine similarity between two vectors. Cosine
 similarity is a measure of similarity between two non-zero vectors of an
 inner product space. It is defined to equal the cosine of the angle
-between them.  Unlike L2, the cosine distance is not affected by the
+between them.  Unlike l2, the cosine distance is not affected by the
 magnitude of the vectors.  Cosine distance has a range of [0, 2].
 
 "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
 distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-L2 norm is 1), then dot distance is equivalent to the cosine distance.
+l2 norm is 1), then dot distance is equivalent to the cosine distance.
 
 ***
 

docs/src/js/interfaces/HnswSqOptions.md

@@ -24,18 +24,18 @@ The following distance types are available:
 
 "l2" - Euclidean distance. This is a very common distance metric that
 accounts for both magnitude and direction when determining the distance
-between vectors. L2 distance has a range of [0, ∞).
+between vectors. l2 distance has a range of [0, ∞).
 
 "cosine" - Cosine distance.  Cosine distance is a distance metric
 calculated from the cosine similarity between two vectors. Cosine
 similarity is a measure of similarity between two non-zero vectors of an
 inner product space. It is defined to equal the cosine of the angle
-between them.  Unlike L2, the cosine distance is not affected by the
+between them.  Unlike l2, the cosine distance is not affected by the
 magnitude of the vectors.  Cosine distance has a range of [0, 2].
 
 "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
 distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-L2 norm is 1), then dot distance is equivalent to the cosine distance.
+l2 norm is 1), then dot distance is equivalent to the cosine distance.
 
 ***
 

docs/src/js/interfaces/IndexOptions.md

@@ -39,3 +39,11 @@ and the same name, then an error will be returned.  This is true even if
 that index is out of date.
 
 The default is true
+
+***
+
+### waitTimeoutSeconds?
+
+```ts
+optional waitTimeoutSeconds: number;
+```

docs/src/js/interfaces/IndexStatistics.md

@@ -30,6 +30,17 @@ The type of the index
 
 ***
 
+### loss?
+
+```ts
+optional loss: number;
+```
+
+The KMeans loss value of the index,
+it is only present for vector indices.
+
+***
+
 ### numIndexedRows
 
 ```ts

docs/src/js/interfaces/IvfFlatOptions.md

@@ -0,0 +1,112 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IvfFlatOptions
+
+# Interface: IvfFlatOptions
+
+Options to create an `IVF_FLAT` index
+
+## Properties
+
+### distanceType?
+
+```ts
+optional distanceType: "l2" | "cosine" | "dot" | "hamming";
+```
+
+Distance type to use to build the index.
+
+Default value is "l2".
+
+This is used when training the index to calculate the IVF partitions
+(vectors are grouped in partitions with similar vectors according to this
+distance type).
+
+The distance type used to train an index MUST match the distance type used
+to search the index.  Failure to do so will yield inaccurate results.
+
+The following distance types are available:
+
+"l2" - Euclidean distance. This is a very common distance metric that
+accounts for both magnitude and direction when determining the distance
+between vectors. l2 distance has a range of [0, ∞).
+
+"cosine" - Cosine distance.  Cosine distance is a distance metric
+calculated from the cosine similarity between two vectors. Cosine
+similarity is a measure of similarity between two non-zero vectors of an
+inner product space. It is defined to equal the cosine of the angle
+between them.  Unlike l2, the cosine distance is not affected by the
+magnitude of the vectors.  Cosine distance has a range of [0, 2].
+
+Note: the cosine distance is undefined when one (or both) of the vectors
+are all zeros (there is no direction).  These vectors are invalid and may
+never be returned from a vector search.
+
+"dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+l2 norm is 1), then dot distance is equivalent to the cosine distance.
+
+"hamming" - Hamming distance. Hamming distance is a distance metric
+calculated from the number of bits that are different between two vectors.
+Hamming distance has a range of [0, dimension]. Note that the hamming distance
+is only valid for binary vectors.
+
+***
+
+### maxIterations?
+
+```ts
+optional maxIterations: number;
+```
+
+Max iteration to train IVF kmeans.
+
+When training an IVF FLAT index we use kmeans to calculate the partitions.  This parameter
+controls how many iterations of kmeans to run.
+
+Increasing this might improve the quality of the index but in most cases these extra
+iterations have diminishing returns.
+
+The default value is 50.
+
+***
+
+### numPartitions?
+
+```ts
+optional numPartitions: number;
+```
+
+The number of IVF partitions to create.
+
+This value should generally scale with the number of rows in the dataset.
+By default the number of partitions is the square root of the number of
+rows.
+
+If this value is too large then the first part of the search (picking the
+right partition) will be slow.  If this value is too small then the second
+part of the search (searching within a partition) will be slow.
+
+***
+
+### sampleRate?
+
+```ts
+optional sampleRate: number;
+```
+
+The number of vectors, per partition, to sample when training IVF kmeans.
+
+When an IVF FLAT index is trained, we need to calculate partitions.  These are groups
+of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+
+Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+random sample of the data.  This parameter controls the size of the sample.  The total
+number of vectors used to train the index is `sample_rate * num_partitions`.
+
+Increasing this value might improve the quality of the index but in most cases the
+default should be sufficient.
+
+The default value is 256.

docs/src/js/interfaces/IvfPqOptions.md

@@ -31,13 +31,13 @@ The following distance types are available:
 
 "l2" - Euclidean distance. This is a very common distance metric that
 accounts for both magnitude and direction when determining the distance
-between vectors. L2 distance has a range of [0, ∞).
+between vectors. l2 distance has a range of [0, ∞).
 
 "cosine" - Cosine distance.  Cosine distance is a distance metric
 calculated from the cosine similarity between two vectors. Cosine
 similarity is a measure of similarity between two non-zero vectors of an
 inner product space. It is defined to equal the cosine of the angle
-between them.  Unlike L2, the cosine distance is not affected by the
+between them.  Unlike l2, the cosine distance is not affected by the
 magnitude of the vectors.  Cosine distance has a range of [0, 2].
 
 Note: the cosine distance is undefined when one (or both) of the vectors
@@ -46,7 +46,7 @@ never be returned from a vector search.
 
 "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
 distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-L2 norm is 1), then dot distance is equivalent to the cosine distance.
+l2 norm is 1), then dot distance is equivalent to the cosine distance.
 
 ***
 

docs/src/js/interfaces/MergeResult.md

@@ -0,0 +1,39 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MergeResult
+
+# Interface: MergeResult
+
+## Properties
+
+### numDeletedRows
+
+```ts
+numDeletedRows: number;
+```
+
+***
+
+### numInsertedRows
+
+```ts
+numInsertedRows: number;
+```
+
+***
+
+### numUpdatedRows
+
+```ts
+numUpdatedRows: number;
+```
+
+***
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/QueryExecutionOptions.md

@@ -20,3 +20,13 @@ The maximum number of rows to return in a single batch
 
 Batches may have fewer rows if the underlying data is stored
 in smaller chunks.
+
+***
+
+### timeoutMs?
+
+```ts
+optional timeoutMs: number;
+```
+
+Timeout for query execution in milliseconds

docs/src/js/interfaces/TableStatistics.md

@@ -0,0 +1,47 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TableStatistics
+
+# Interface: TableStatistics
+
+## Properties
+
+### fragmentStats
+
+```ts
+fragmentStats: FragmentStatistics;
+```
+
+Statistics on table fragments
+
+***
+
+### numIndices
+
+```ts
+numIndices: number;
+```
+
+The number of indices in the table
+
+***
+
+### numRows
+
+```ts
+numRows: number;
+```
+
+The number of rows in the table
+
+***
+
+### totalBytes
+
+```ts
+totalBytes: number;
+```
+
+The total number of bytes in the table

docs/src/js/interfaces/UpdateResult.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / UpdateResult
+
+# Interface: UpdateResult
+
+## Properties
+
+### rowsUpdated
+
+```ts
+rowsUpdated: number;
+```
+
+***
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/interfaces/WriteExecutionOptions.md

@@ -0,0 +1,26 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / WriteExecutionOptions
+
+# Interface: WriteExecutionOptions
+
+## Properties
+
+### timeoutMs?
+
+```ts
+optional timeoutMs: number;
+```
+
+Maximum time to run the operation before cancelling it.
+
+By default, there is a 30-second timeout that is only enforced after the
+first attempt. This is to prevent spending too long retrying to resolve
+conflicts. For example, if a write attempt takes 20 seconds and fails,
+the second attempt will be cancelled after 10 seconds, hitting the
+30-second timeout. However, a write that takes one hour and succeeds on the
+first attempt will not be cancelled.
+
+When this is set, the timeout is enforced on all attempts, including the first.

docs/src/python/python.md

@@ -59,8 +59,6 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ::: lancedb.embeddings.open_clip.OpenClipEmbeddings
 
-::: lancedb.embeddings.utils.with_embeddings
-
 ## Context
 
 ::: lancedb.context.contextualize

docs/src/search.md

@@ -15,7 +15,7 @@ Currently, LanceDB supports the following metrics:
 
 | Metric    | Description                                                                 |
 | --------- | --------------------------------------------------------------------------- |
-| `l2`      | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
+| `l2`      | [Euclidean / l2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
 | `cosine`  | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)        |
 | `dot`     | [Dot Production](https://en.wikipedia.org/wiki/Dot_product)                 |
 | `hamming` | [Hamming Distance](https://en.wikipedia.org/wiki/Hamming_distance)          |
@@ -138,6 +138,19 @@ LanceDB supports binary vectors as a data type, and has the ability to search bi
         --8<-- "python/python/tests/docs/test_binary_vector.py:async_binary_vector"
         ```
 
+    === "TypeScript"
+
+        ```ts
+        --8<-- "nodejs/examples/search.test.ts:import"
+
+        --8<-- "nodejs/examples/search.test.ts:import_bin_util"
+
+        --8<-- "nodejs/examples/search.test.ts:ingest_binary_data"
+
+        --8<-- "nodejs/examples/search.test.ts:search_binary_data"
+        ```
+
+
 ## Multivector type
 
 LanceDB supports multivector type, this is useful when you have multiple vectors for a single item (e.g. with ColBert and ColPali).

docs/src/sql.md

@@ -7,7 +7,7 @@ performed on the top-k results returned by the vector search. However, pre-filte
 option that performs the filter prior to vector search. This can be useful to narrow down
 the search space of a very large dataset to reduce query latency.
 
-Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to set `use_index=false` if you know that the filter is highly selective.
+Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to call `bypass_vector_index()` if you know that the filter is highly selective.
 
 Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain.
 

docs/src/troubleshooting.md

@@ -8,6 +8,10 @@ For trouble shooting, the best place to ask is in our Discord, under the relevan
 language channel. By asking in the language-specific channel, it makes it more
 likely that someone who knows the answer will see your question.
 
+## Common issues
+
+* Multiprocessing with `fork` is not supported. You should use `spawn` instead.
+
 ## Enabling logging
 
 To provide more information, especially for LanceDB Cloud related issues, enable
@@ -31,3 +35,9 @@ print the resolved query plan. You can use the `explain_plan` method to do this:
 * Python Sync: [LanceQueryBuilder.explain_plan][lancedb.query.LanceQueryBuilder.explain_plan]
 * Python Async: [AsyncQueryBase.explain_plan][lancedb.query.AsyncQueryBase.explain_plan]
 * Node @lancedb/lancedb: [LanceQueryBuilder.explainPlan](/lancedb/js/classes/QueryBase/#explainplan)
+
+To understand how a query was actually executed—including metrics like execution time, number of rows processed, I/O stats, and more—use the analyze_plan method. This executes the query and returns a physical execution plan annotated with runtime metrics, making it especially helpful for performance tuning and debugging.
+
+* Python Sync: [LanceQueryBuilder.analyze_plan][lancedb.query.LanceQueryBuilder.analyze_plan]
+* Python Async: [AsyncQueryBase.analyze_plan][lancedb.query.AsyncQueryBase.analyze_plan]
+* Node @lancedb/lancedb: [LanceQueryBuilder.analyzePlan](/lancedb/js/classes/QueryBase/#analyzePlan)

java/.gitignore

@@ -0,0 +1,3 @@
+*.iml
+.java-version
+

java/core/pom.xml

@@ -8,13 +8,16 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.18.0-beta.0</version>
+        <version>0.19.1-beta.5</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
     <artifactId>lancedb-core</artifactId>
     <name>LanceDB Core</name>
     <packaging>jar</packaging>
+    <properties>
+        <rust.release.build>false</rust.release.build>
+    </properties>
 
     <dependencies>
         <dependency>
@@ -68,7 +71,7 @@
                                 </goals>
                                 <configuration>
                                     <path>lancedb-jni</path>
-                                    <release>true</release>
+                                    <release>${rust.release.build}</release>
                                     <!-- Copy native libraries to target/classes for runtime access -->
                                     <copyTo>${project.build.directory}/classes/nativelib</copyTo>
                                     <copyWithPlatformDir>true</copyWithPlatformDir>

java/core/src/main/java/com/lancedb/lancedb/Connection.java

@@ -1,16 +1,25 @@
-// SPDX-License-Identifier: Apache-2.0
+/*
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+ * Licensed under the Apache License, Version 2.0 (the "License");
-
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.lancedb.lancedb;
 
 import io.questdb.jar.jni.JarJniLoader;
+
 import java.io.Closeable;
 import java.util.List;
 import java.util.Optional;
 
-/**
+/** Represents LanceDB database. */
- * Represents LanceDB database.
- */
 public class Connection implements Closeable {
   static {
     JarJniLoader.loadLib(Connection.class, "/nativelib", "lancedb_jni");
@@ -18,14 +27,11 @@ public class Connection implements Closeable {
 
   private long nativeConnectionHandle;
 
-  /**
+  /** Connect to a LanceDB instance. */
-   * Connect to a LanceDB instance.
-   */
   public static native Connection connect(String uri);
 
   /**
-   * Get the names of all tables in the database. The names are sorted in
+   * Get the names of all tables in the database. The names are sorted in ascending order.
-   * ascending order.
    *
    * @return the table names
    */
@@ -34,8 +40,7 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
-   * ascending order.
    *
    * @param limit The number of results to return.
    * @return the table names
@@ -45,12 +50,11 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
-   * ascending order.
    *
    * @param startAfter If present, only return names that come lexicographically after the supplied
-   *                   value. This can be combined with limit to implement pagination
+   *     value. This can be combined with limit to implement pagination by setting this to the last
-   *                   by setting this to the last table name from the previous page.
+   *     table name from the previous page.
    * @return the table names
    */
   public List<String> tableNames(String startAfter) {
@@ -58,12 +62,11 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
-   * ascending order.
    *
    * @param startAfter If present, only return names that come lexicographically after the supplied
-   *                   value. This can be combined with limit to implement pagination
+   *     value. This can be combined with limit to implement pagination by setting this to the last
-   *                   by setting this to the last table name from the previous page.
+   *     table name from the previous page.
    * @param limit The number of results to return.
    * @return the table names
    */
@@ -72,22 +75,19 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
-   * ascending order.
    *
    * @param startAfter If present, only return names that come lexicographically after the supplied
-   *                   value. This can be combined with limit to implement pagination
+   *     value. This can be combined with limit to implement pagination by setting this to the last
-   *                   by setting this to the last table name from the previous page.
+   *     table name from the previous page.
    * @param limit The number of results to return.
    * @return the table names
    */
-  public native List<String> tableNames(
+  public native List<String> tableNames(Optional<String> startAfter, Optional<Integer> limit);
-      Optional<String> startAfter, Optional<Integer> limit);
 
   /**
-   * Closes this connection and releases any system resources associated with it. If
+   * Closes this connection and releases any system resources associated with it. If the connection
-   * the connection is
+   * is already closed, then invoking this method has no effect.
-   * already closed, then invoking this method has no effect.
    */
   @Override
   public void close() {
@@ -98,8 +98,7 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Native method to release the Lance connection resources associated with the
+   * Native method to release the Lance connection resources associated with the given handle.
-   * given handle.
    *
    * @param handle The native handle to the connection resource.
    */

java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java

@@ -1,27 +1,35 @@
-// SPDX-License-Identifier: Apache-2.0
+/*
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.lancedb.lancedb;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.nio.file.Path;
-import java.util.List;
-import java.net.URL;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 public class ConnectionTest {
   private static final String[] TABLE_NAMES = {
-      "dataset_version",
+    "dataset_version", "new_empty_dataset", "test", "write_stream"
-      "new_empty_dataset",
-      "test",
-      "write_stream"
   };
 
-  @TempDir
+  @TempDir static Path tempDir; // Temporary directory for the tests
-  static Path tempDir; // Temporary directory for the tests
   private static URL lanceDbURL;
 
   @BeforeAll
@@ -53,18 +61,21 @@ public class ConnectionTest {
   @Test
   void tableNamesStartAfter() {
     try (Connection conn = Connection.connect(lanceDbURL.toString())) {
-      assertTableNamesStartAfter(conn, TABLE_NAMES[0], 3, TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(
+          conn, TABLE_NAMES[0], 3, TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, TABLE_NAMES[1], 2, TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, TABLE_NAMES[2], 1, TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, TABLE_NAMES[3], 0);
-      assertTableNamesStartAfter(conn, "a_dataset", 4, TABLE_NAMES[0], TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(
+          conn, "a_dataset", 4, TABLE_NAMES[0], TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, "o_dataset", 2, TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, "v_dataset", 1, TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, "z_dataset", 0);
     }
   }
 
-  private void assertTableNamesStartAfter(Connection conn, String startAfter, int expectedSize, String... expectedNames) {
+  private void assertTableNamesStartAfter(
+      Connection conn, String startAfter, int expectedSize, String... expectedNames) {
     List<String> tableNames = conn.tableNames(startAfter);
     assertEquals(expectedSize, tableNames.size());
     for (int i = 0; i < expectedNames.length; i++) {
@@ -74,7 +85,7 @@ public class ConnectionTest {
 
   @Test
   void tableNamesLimit() {
-      try (Connection conn = Connection.connect(lanceDbURL.toString())) {
+    try (Connection conn = Connection.connect(lanceDbURL.toString())) {
       for (int i = 0; i <= TABLE_NAMES.length; i++) {
         List<String> tableNames = conn.tableNames(i);
         assertEquals(i, tableNames.size());

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.18.0-beta.0</version>
+    <version>0.19.1-beta.5</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>
@@ -29,6 +29,25 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
+        <spotless.skip>false</spotless.skip>
+        <spotless.version>2.30.0</spotless.version>
+        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+        <spotless.delimiter>package</spotless.delimiter>
+        <spotless.license.header>
+            /*
+            * Licensed under the Apache License, Version 2.0 (the "License");
+            * you may not use this file except in compliance with the License.
+            * You may obtain a copy of the License at
+            *
+            *     http://www.apache.org/licenses/LICENSE-2.0
+            *
+            * Unless required by applicable law or agreed to in writing, software
+            * distributed under the License is distributed on an "AS IS" BASIS,
+            * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+            * See the License for the specific language governing permissions and
+            * limitations under the License.
+            */
+        </spotless.license.header>
     </properties>
 
     <modules>
@@ -127,7 +146,8 @@
                 <configuration>
                     <configLocation>google_checks.xml</configLocation>
                     <consoleOutput>true</consoleOutput>
-                    <failsOnError>true</failsOnError>
+                    <failsOnError>false</failsOnError>
+                    <failOnViolation>false</failOnViolation>
                     <violationSeverity>warning</violationSeverity>
                     <linkXRef>false</linkXRef>
                 </configuration>
@@ -141,6 +161,10 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>com.diffplug.spotless</groupId>
+                <artifactId>spotless-maven-plugin</artifactId>
+            </plugin>
         </plugins>
         <pluginManagement>
             <plugins>
@@ -166,7 +190,6 @@
                     <artifactId>maven-surefire-plugin</artifactId>
                     <version>3.2.5</version>
                     <configuration>
-                        <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
                         <forkNode
                             implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory" />
                         <useSystemClassLoader>false</useSystemClassLoader>
@@ -180,6 +203,54 @@
                     <artifactId>maven-install-plugin</artifactId>
                     <version>2.5.2</version>
                 </plugin>
+                <plugin>
+                    <groupId>com.diffplug.spotless</groupId>
+                    <artifactId>spotless-maven-plugin</artifactId>
+                    <version>${spotless.version}</version>
+                    <configuration>
+                        <skip>${spotless.skip}</skip>
+                        <upToDateChecking>
+                            <enabled>true</enabled>
+                        </upToDateChecking>
+                        <java>
+                            <includes>
+                                <include>src/main/java/**/*.java</include>
+                                <include>src/test/java/**/*.java</include>
+                            </includes>
+                            <googleJavaFormat>
+                                <version>${spotless.java.googlejavaformat.version}</version>
+                                <style>GOOGLE</style>
+                            </googleJavaFormat>
+
+                            <importOrder>
+                                <order>com.lancedb.lance,,javax,java,\#</order>
+                            </importOrder>
+
+                            <removeUnusedImports />
+                        </java>
+                        <scala>
+                            <includes>
+                                <include>src/main/scala/**/*.scala</include>
+                                <include>src/main/scala-*/**/*.scala</include>
+                                <include>src/test/scala/**/*.scala</include>
+                                <include>src/test/scala-*/**/*.scala</include>
+                            </includes>
+                        </scala>
+                        <licenseHeader>
+                            <content>${spotless.license.header}</content>
+                            <delimiter>${spotless.delimiter}</delimiter>
+                        </licenseHeader>
+                    </configuration>
+                    <executions>
+                        <execution>
+                            <id>spotless-check</id>
+                            <phase>validate</phase>
+                            <goals>
+                                <goal>apply</goal>
+                            </goals>
+                        </execution>
+                    </executions>
+                </plugin>
             </plugins>
         </pluginManagement>
     </build>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.18.0-beta.0",
+  "version": "0.19.1-beta.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.18.0-beta.0",
+      "version": "0.19.1-beta.5",
       "cpu": [
         "x64",
         "arm64"
@@ -52,14 +52,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.18.0-beta.0",
+        "@lancedb/vectordb-darwin-arm64": "0.19.1-beta.5",
-        "@lancedb/vectordb-darwin-x64": "0.18.0-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.19.1-beta.5",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.18.0-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.19.1-beta.5",
-        "@lancedb/vectordb-linux-arm64-musl": "0.18.0-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.19.1-beta.5",
-        "@lancedb/vectordb-linux-x64-gnu": "0.18.0-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.19.1-beta.5"
-        "@lancedb/vectordb-linux-x64-musl": "0.18.0-beta.0",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.18.0-beta.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.18.0-beta.0"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -330,9 +327,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.18.0-beta.0",
+      "version": "0.19.1-beta.5",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.18.0-beta.0.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.19.1-beta.5.tgz",
-      "integrity": "sha512-dLLgMPllYJOiRfPqkqkmoQu48RIa7K4dOF/qFP8Aex3zqeHE/0sFm3DYjtSFc6SR/6yT8u6Y9iFo2cQp5rCFJA==",
+      "integrity": "sha512-9WcTw67We5HYGayDt5jFquGoyAVzFSt/I65ag8+q7H9q4ZYKxeDhgNyQZJ8BmXEvbJtnYtYBSAtTEdFKYMce6w==",
       "cpu": [
         "arm64"
       ],
@@ -343,9 +340,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.18.0-beta.0",
+      "version": "0.19.1-beta.5",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.18.0-beta.0.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.19.1-beta.5.tgz",
-      "integrity": "sha512-la0eauU0rzHO5eeVjBt8o/5UW4VzRYAuRA7nqUFLX5T6SWP5+UWjqusVVbWGz3ski+8uEX6VhlaFZP5uIJKGIg==",
+      "integrity": "sha512-6Pe3PxEMi0VKGsu5R7IhOxTijUM3b5olRAqhxfcu5ti34gXIPNtu7g+T9lS78LKe+0D0v2BjZEY/JQakIFBNRw==",
       "cpu": [
         "x64"
       ],
@@ -356,22 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.18.0-beta.0",
+      "version": "0.19.1-beta.5",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.18.0-beta.0.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.19.1-beta.5.tgz",
-      "integrity": "sha512-AkXI/lB3yu1Di2G1lhilf89V6qPTppb13aAt+/6gU5/PSfA94y9VXD67D4WyvRbuQghJjDvAavMlWMrJc2NuMw==",
+      "integrity": "sha512-VJbBd+Y+6L2SREaOO1OzuUfTPHXyHE4AcsZuM6VMyoeX8k7lPnaA+vNk96o0w4V2KFEAI6o4QPgrRAXmMAzmbg==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-musl": {
-      "version": "0.18.0-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-musl/-/vectordb-linux-arm64-musl-0.18.0-beta.0.tgz",
-      "integrity": "sha512-kTVcJ4LA8w/7egY4m0EXOt8c1DeFUquVtyvexO+VzIFeeHfBkkrMI0DkE0CpHmk+gctkG7EY39jzjgLnPvppnw==",
       "cpu": [
         "arm64"
       ],
@@ -382,9 +366,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.18.0-beta.0",
+      "version": "0.19.1-beta.5",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.18.0-beta.0.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.19.1-beta.5.tgz",
-      "integrity": "sha512-KbtIy5DkaWTsKENm5Q27hjovrR7FRuoHhl0wDJtO/2CUZYlrskjEIfcfkfA2CrEQesBug4s5jgsvNM4Wcp6zoA==",
+      "integrity": "sha512-3wS8Zn5NmHoszXfrY4JzMimHoh5LAmVi3pTX4gD+C9kVGoUJcDBP7/CrAbjnAz7VzzAIPmz8kvBuPz8l9X4hjw==",
       "cpu": [
         "x64"
       ],
@@ -394,36 +378,10 @@
         "linux"
       ]
     },
-    "node_modules/@lancedb/vectordb-linux-x64-musl": {
-      "version": "0.18.0-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-musl/-/vectordb-linux-x64-musl-0.18.0-beta.0.tgz",
-      "integrity": "sha512-SF07gmoGVExcF5v+IE6kBbCbXJSDyTgC7QCt+MDS1NsgoQ9OH7IyH7r6HJu16tKflUOUKlUHnP0hQOPpv1fWpg==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-arm64-msvc": {
-      "version": "0.18.0-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-arm64-msvc/-/vectordb-win32-arm64-msvc-0.18.0-beta.0.tgz",
-      "integrity": "sha512-YYBuSBGDlxJgSI5gHjDmQo9sl05lAXfzil6QiKfgmUMsBtb2sT+GoUCgG6qzsfe99sWiTf+pMeWDsQgfrj9vNw==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.18.0-beta.0",
+      "version": "0.19.1-beta.5",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.18.0-beta.0.tgz",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.19.1-beta.5.tgz",
-      "integrity": "sha512-t9TXeUnMU7YbP+/nUJpStm75aWwUydZj2AK+G2XwDtQrQo4Xg7/NETEbBeogmIOHuidNQYia8jEeQCUon5/+Dw==",
+      "integrity": "sha512-TemM9cvrPa2jFCjvYmKnrL0DTHegi/+LOQ3No9nPDHie2ka2fM9O2q60fAbYsYz+Mo9aV7MvL49ATbNCyl9MLA==",
       "cpu": [
         "x64"
       ],
@@ -1226,9 +1184,10 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.7.7",
+      "version": "1.8.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz",
-      "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==",
+      "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==",
+      "license": "MIT",
       "dependencies": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.18.0-beta.0",
+  "version": "0.19.1-beta.5",
   "description": " Serverless, low-latency vector database for AI applications",
   "private": false,
   "main": "dist/index.js",
@@ -85,20 +85,14 @@
       "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
       "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
       "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
-      "x86_64-unknown-linux-musl": "@lancedb/vectordb-linux-x64-musl",
+      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
-      "aarch64-unknown-linux-musl": "@lancedb/vectordb-linux-arm64-musl",
-      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc",
-      "aarch64-pc-windows-msvc": "@lancedb/vectordb-win32-arm64-msvc"
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.18.0-beta.0",
+    "@lancedb/vectordb-darwin-x64": "0.19.1-beta.5",
-    "@lancedb/vectordb-darwin-arm64": "0.18.0-beta.0",
+    "@lancedb/vectordb-darwin-arm64": "0.19.1-beta.5",
-    "@lancedb/vectordb-linux-x64-gnu": "0.18.0-beta.0",
+    "@lancedb/vectordb-linux-x64-gnu": "0.19.1-beta.5",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.18.0-beta.0",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.19.1-beta.5",
-    "@lancedb/vectordb-linux-x64-musl": "0.18.0-beta.0",
+    "@lancedb/vectordb-win32-x64-msvc": "0.19.1-beta.5"
-    "@lancedb/vectordb-linux-arm64-musl": "0.18.0-beta.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.18.0-beta.0",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.18.0-beta.0"
   }
 }

node/src/index.ts

@@ -1299,7 +1299,7 @@ export interface IvfPQIndexConfig {
   index_name?: string
 
   /**
-   * Metric type, L2 or Cosine
+   * Metric type, l2 or Cosine
    */
   metric_type?: MetricType
 

nodejs/.npmignore

@@ -22,3 +22,4 @@ build.rs
 jest.config.js
 tsconfig.json
 typedoc.json
+typedoc_post_process.js

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.18.0-beta.0"
+version = "0.19.1-beta.5"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -18,7 +18,7 @@ arrow-array.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
-lancedb = { path = "../rust/lancedb", features = ["remote"] }
+lancedb = { path = "../rust/lancedb" }
 napi = { version = "2.16.8", default-features = false, features = [
     "napi9",
     "async"
@@ -28,5 +28,13 @@ napi-derive = "2.16.4"
 lzma-sys = { version = "*", features = ["static"] }
 log.workspace = true
 
+# Workaround for build failure until we can fix it.
+aws-lc-sys = "=0.28.0"
+
 [build-dependencies]
 napi-build = "2.1"
+
+[features]
+default = ["remote"]
+fp16kernels = ["lancedb/fp16kernels"]
+remote = ["lancedb/remote"]

nodejs/README.md

@@ -11,11 +11,9 @@ npm install @lancedb/lancedb
 This will download the appropriate native library for your platform. We currently
 support:
 
-- Linux (x86_64 and aarch64)
+- Linux (x86_64 and aarch64 on glibc and musl)
 - MacOS (Intel and ARM/M1/M2)
-- Windows (x86_64 only)
+- Windows (x86_64 and aarch64)
-
-We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
 
 ## Usage
 

nodejs/__test__/arrow.test.ts

@@ -374,6 +374,71 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         expect(table2.numRows).toBe(4);
         expect(table2.schema).toEqual(schema);
       });
+
+      it("should correctly retain values in nested struct fields", async function () {
+        // Define test data with nested struct
+        const testData = [
+          {
+            id: "doc1",
+            vector: [1, 2, 3],
+            metadata: {
+              filePath: "/path/to/file1.ts",
+              startLine: 10,
+              endLine: 20,
+              text: "function test() { return true; }",
+            },
+          },
+          {
+            id: "doc2",
+            vector: [4, 5, 6],
+            metadata: {
+              filePath: "/path/to/file2.ts",
+              startLine: 30,
+              endLine: 40,
+              text: "function test2() { return false; }",
+            },
+          },
+        ];
+
+        // Create Arrow table from the data
+        const table = makeArrowTable(testData);
+
+        // Verify schema has the nested struct fields
+        const metadataField = table.schema.fields.find(
+          (f) => f.name === "metadata",
+        );
+        expect(metadataField).toBeDefined();
+        // biome-ignore lint/suspicious/noExplicitAny: accessing fields in different Arrow versions
+        const childNames = metadataField?.type.children.map((c: any) => c.name);
+        expect(childNames).toEqual([
+          "filePath",
+          "startLine",
+          "endLine",
+          "text",
+        ]);
+
+        // Convert to buffer and back (simulating storage and retrieval)
+        const buf = await fromTableToBuffer(table);
+        const retrievedTable = tableFromIPC(buf);
+
+        // Verify the retrieved table has the same structure
+        const rows = [];
+        for (let i = 0; i < retrievedTable.numRows; i++) {
+          rows.push(retrievedTable.get(i));
+        }
+
+        // Check values in the first row
+        const firstRow = rows[0];
+        expect(firstRow.id).toBe("doc1");
+        expect(firstRow.vector.toJSON()).toEqual([1, 2, 3]);
+
+        // Verify metadata values are preserved (this is where the bug is)
+        expect(firstRow.metadata).toBeDefined();
+        expect(firstRow.metadata.filePath).toBe("/path/to/file1.ts");
+        expect(firstRow.metadata.startLine).toBe(10);
+        expect(firstRow.metadata.endLine).toBe(20);
+        expect(firstRow.metadata.text).toBe("function test() { return true; }");
+      });
     });
 
     class DummyEmbedding extends EmbeddingFunction<string> {

nodejs/__test__/table.test.ts

@@ -10,7 +10,7 @@ import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
-import { Table, connect } from "../lancedb";
+import { MatchQuery, PhraseQuery, Table, connect } from "../lancedb";
 import {
   Table as ArrowTable,
   Field,
@@ -21,9 +21,11 @@ import {
   Int64,
   List,
   Schema,
+  Uint8,
   Utf8,
   makeArrowTable,
 } from "../lancedb/arrow";
+import * as arrow from "../lancedb/arrow";
 import {
   EmbeddingFunction,
   LanceSchema,
@@ -31,6 +33,8 @@ import {
   register,
 } from "../lancedb/embedding";
 import { Index } from "../lancedb/indices";
+import { instanceOfFullTextQuery } from "../lancedb/query";
+import exp = require("constants");
 
 describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Given a table",
@@ -68,8 +72,33 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       await expect(table.countRows()).resolves.toBe(3);
     });
 
-    it("should overwrite data if asked", async () => {
+    it("should show table stats", async () => {
       await table.add([{ id: 1 }, { id: 2 }]);
+      await table.add([{ id: 1 }]);
+      await expect(table.stats()).resolves.toEqual({
+        fragmentStats: {
+          lengths: {
+            max: 2,
+            mean: 1,
+            min: 1,
+            p25: 1,
+            p50: 2,
+            p75: 2,
+            p99: 2,
+          },
+          numFragments: 2,
+          numSmallFragments: 2,
+        },
+        numIndices: 0,
+        numRows: 3,
+        totalBytes: 24,
+      });
+    });
+
+    it("should overwrite data if asked", async () => {
+      const addRes = await table.add([{ id: 1 }, { id: 2 }]);
+      expect(addRes).toHaveProperty("version");
+      expect(addRes.version).toBe(2);
       await table.add([{ id: 1 }], { mode: "overwrite" });
       await expect(table.countRows()).resolves.toBe(1);
     });
@@ -85,7 +114,11 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       await table.add([{ id: 1 }]);
       expect(await table.countRows("id == 1")).toBe(1);
       expect(await table.countRows("id == 7")).toBe(0);
-      await table.update({ id: "7" });
+      const updateRes = await table.update({ id: "7" });
+      expect(updateRes).toHaveProperty("version");
+      expect(updateRes.version).toBe(3);
+      expect(updateRes).toHaveProperty("rowsUpdated");
+      expect(updateRes.rowsUpdated).toBe(1);
       expect(await table.countRows("id == 1")).toBe(0);
       expect(await table.countRows("id == 7")).toBe(1);
       await table.add([{ id: 2 }]);
@@ -278,6 +311,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(res.getChild("y")?.toJSON()).toEqual([2, null, null, null]);
       expect(res.getChild("z")?.toJSON()).toEqual([null, null, 3n, 5n]);
     });
+
+    it("should handle null vectors at end of data", async () => {
+      // https://github.com/lancedb/lancedb/issues/2240
+      const data = [{ vector: [1, 2, 3] }, { vector: null }];
+      const db = await connect("memory://");
+
+      const table = await db.createTable("my_table", data);
+      expect(await table.countRows()).toEqual(2);
+    });
   },
 );
 
@@ -303,11 +345,17 @@ describe("merge insert", () => {
       { a: 3, b: "y" },
       { a: 4, b: "z" },
     ];
-    await table
+    const mergeInsertRes = await table
       .mergeInsert("a")
       .whenMatchedUpdateAll()
       .whenNotMatchedInsertAll()
-      .execute(newData);
+      .execute(newData, { timeoutMs: 10_000 });
+    expect(mergeInsertRes).toHaveProperty("version");
+    expect(mergeInsertRes.version).toBe(2);
+    expect(mergeInsertRes.numInsertedRows).toBe(1);
+    expect(mergeInsertRes.numUpdatedRows).toBe(2);
+    expect(mergeInsertRes.numDeletedRows).toBe(0);
+
     const expected = [
       { a: 1, b: "a" },
       { a: 2, b: "x" },
@@ -325,10 +373,12 @@ describe("merge insert", () => {
       { a: 3, b: "y" },
       { a: 4, b: "z" },
     ];
-    await table
+    const mergeInsertRes = await table
       .mergeInsert("a")
       .whenMatchedUpdateAll({ where: "target.b = 'b'" })
       .execute(newData);
+    expect(mergeInsertRes).toHaveProperty("version");
+    expect(mergeInsertRes.version).toBe(2);
 
     const expected = [
       { a: 1, b: "a" },
@@ -413,6 +463,20 @@ describe("merge insert", () => {
     res = res.sort((a, b) => a.a - b.a);
     expect(res).toEqual(expected);
   });
+
+  test("timeout", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    await expect(
+      table
+        .mergeInsert("a")
+        .whenMatchedUpdateAll()
+        .whenNotMatchedInsertAll()
+        .execute(newData, { timeoutMs: 0 }),
+    ).rejects.toThrow("merge insert timed out");
+  });
 });
 
 describe("When creating an index", () => {
@@ -460,6 +524,8 @@ describe("When creating an index", () => {
       indexType: "IvfPq",
       columns: ["vec"],
     });
+    const stats = await tbl.indexStats("vec_idx");
+    expect(stats?.loss).toBeDefined();
 
     // Search without specifying the column
     let rst = await tbl
@@ -493,6 +559,15 @@ describe("When creating an index", () => {
     expect(indices2.length).toBe(0);
   });
 
+  it("should wait for index readiness", async () => {
+    // Create an index and then wait for it to be ready
+    await tbl.createIndex("vec");
+    const indices = await tbl.listIndices();
+    expect(indices.length).toBeGreaterThan(0);
+    const idxName = indices[0].name;
+    await expect(tbl.waitForIndex([idxName], 5)).resolves.toBeUndefined();
+  });
+
   it("should search with distance range", async () => {
     await tbl.createIndex("vec");
 
@@ -620,6 +695,23 @@ describe("When creating an index", () => {
     expect(plan2).not.toMatch("LanceScan");
   });
 
+  it("should be able to run analyze plan", async () => {
+    await tbl.createIndex("vec");
+    await tbl.add([
+      {
+        id: 300,
+        vec: Array(32)
+          .fill(1)
+          .map(() => Math.random()),
+        tags: [],
+      },
+    ]);
+
+    const plan = await tbl.query().nearestTo(queryVec).analyzePlan();
+    expect(plan).toMatch("AnalyzeExec");
+    expect(plan).toMatch("metrics=");
+  });
+
   it("should be able to query with row id", async () => {
     const results = await tbl
       .query()
@@ -720,6 +812,7 @@ describe("When creating an index", () => {
     expect(stats?.distanceType).toBeUndefined();
     expect(stats?.indexType).toEqual("BTREE");
     expect(stats?.numIndices).toEqual(1);
+    expect(stats?.loss).toBeUndefined();
   });
 
   test("when getting stats on non-existent index", async () => {
@@ -727,6 +820,38 @@ describe("When creating an index", () => {
     expect(stats).toBeUndefined();
   });
 
+  test("create ivf_flat with binary vectors", async () => {
+    const db = await connect(tmpDir.name);
+    const binarySchema = new Schema([
+      new Field("id", new Int32(), true),
+      new Field("vec", new FixedSizeList(32, new Field("item", new Uint8()))),
+    ]);
+    const tbl = await db.createTable(
+      "binary",
+      makeArrowTable(
+        Array(300)
+          .fill(1)
+          .map((_, i) => ({
+            id: i,
+            vec: Array(32)
+              .fill(1)
+              .map(() => Math.floor(Math.random() * 255)),
+          })),
+        { schema: binarySchema },
+      ),
+    );
+    await tbl.createIndex("vec", {
+      config: Index.ivfFlat({ numPartitions: 10, distanceType: "hamming" }),
+    });
+
+    // query with binary vectors
+    const queryVec = Array(32)
+      .fill(1)
+      .map(() => Math.floor(Math.random() * 255));
+    const rst = await tbl.query().limit(5).nearestTo(queryVec).toArrow();
+    expect(rst.numRows).toBe(5);
+  });
+
   // TODO: Move this test to the query API test (making sure we can reject queries
   // when the dimension is incorrect)
   test("two columns with different dimensions", async () => {
@@ -760,6 +885,7 @@ describe("When creating an index", () => {
     // Only build index over v1
     await tbl.createIndex("vec", {
       config: Index.ivfPq({ numPartitions: 2, numSubVectors: 2 }),
+      waitTimeoutSeconds: 30,
     });
 
     const rst = await tbl
@@ -804,6 +930,44 @@ describe("When creating an index", () => {
   });
 });
 
+describe("When querying a table", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should throw an error when timeout is reached", async () => {
+    const db = await connect(tmpDir.name);
+    const data = makeArrowTable([
+      { text: "a", vector: [0.1, 0.2] },
+      { text: "b", vector: [0.3, 0.4] },
+    ]);
+    const table = await db.createTable("test", data);
+    await table.createIndex("text", { config: Index.fts() });
+
+    await expect(
+      table.query().where("text != 'a'").toArray({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+
+    await expect(
+      table.query().nearestTo([0.0, 0.0]).toArrow({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+
+    await expect(
+      table.search("a", "fts").toArray({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+
+    await expect(
+      table
+        .query()
+        .nearestToText("a")
+        .nearestTo([0.0, 0.0])
+        .toArrow({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+  });
+});
+
 describe("Read consistency interval", () => {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -888,15 +1052,19 @@ describe("schema evolution", function () {
       { id: 1n, vector: [0.1, 0.2] },
     ]);
     // Can create a non-nullable column only through addColumns at the moment.
-    await table.addColumns([
+    const addColumnsRes = await table.addColumns([
       { name: "price", valueSql: "cast(10.0 as double)" },
     ]);
+    expect(addColumnsRes).toHaveProperty("version");
+    expect(addColumnsRes.version).toBe(2);
     expect(await table.schema()).toEqual(schema);
 
-    await table.alterColumns([
+    const alterColumnsRes = await table.alterColumns([
       { path: "id", rename: "new_id" },
       { path: "price", nullable: true },
     ]);
+    expect(alterColumnsRes).toHaveProperty("version");
+    expect(alterColumnsRes.version).toBe(3);
 
     const expectedSchema = new Schema([
       new Field("new_id", new Int64(), true),
@@ -920,6 +1088,93 @@ describe("schema evolution", function () {
       new Field("price", new Float64(), true),
     ]);
     expect(await table.schema()).toEqual(expectedSchema2);
+
+    await table.alterColumns([
+      {
+        path: "vector",
+        dataType: new FixedSizeList(2, new Field("item", new Float64(), true)),
+      },
+    ]);
+    const expectedSchema3 = new Schema([
+      new Field("new_id", new Int32(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float64(), true)),
+        true,
+      ),
+      new Field("price", new Float64(), true),
+    ]);
+    expect(await table.schema()).toEqual(expectedSchema3);
+  });
+
+  it("can cast to various types", async function () {
+    const con = await connect(tmpDir.name);
+
+    // integers
+    const intTypes = [
+      new arrow.Int8(),
+      new arrow.Int16(),
+      new arrow.Int32(),
+      new arrow.Int64(),
+      new arrow.Uint8(),
+      new arrow.Uint16(),
+      new arrow.Uint32(),
+      new arrow.Uint64(),
+    ];
+    const tableInts = await con.createTable("ints", [{ id: 1n }], {
+      schema: new Schema([new Field("id", new Int64(), true)]),
+    });
+    for (const intType of intTypes) {
+      await tableInts.alterColumns([{ path: "id", dataType: intType }]);
+      const schema = new Schema([new Field("id", intType, true)]);
+      expect(await tableInts.schema()).toEqual(schema);
+    }
+
+    // floats
+    const floatTypes = [
+      new arrow.Float16(),
+      new arrow.Float32(),
+      new arrow.Float64(),
+    ];
+    const tableFloats = await con.createTable("floats", [{ val: 2.1 }], {
+      schema: new Schema([new Field("val", new Float32(), true)]),
+    });
+    for (const floatType of floatTypes) {
+      await tableFloats.alterColumns([{ path: "val", dataType: floatType }]);
+      const schema = new Schema([new Field("val", floatType, true)]);
+      expect(await tableFloats.schema()).toEqual(schema);
+    }
+
+    // Lists of floats
+    const listTypes = [
+      new arrow.List(new arrow.Field("item", new arrow.Float32(), true)),
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float64(), true),
+      ),
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float16(), true),
+      ),
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+    ];
+    const tableLists = await con.createTable("lists", [{ val: [2.1, 3.2] }], {
+      schema: new Schema([
+        new Field(
+          "val",
+          new FixedSizeList(2, new arrow.Field("item", new Float32())),
+          true,
+        ),
+      ]),
+    });
+    for (const listType of listTypes) {
+      await tableLists.alterColumns([{ path: "val", dataType: listType }]);
+      const schema = new Schema([new Field("val", listType, true)]);
+      expect(await tableLists.schema()).toEqual(schema);
+    }
   });
 
   it("can drop a column from the schema", async function () {
@@ -927,7 +1182,9 @@ describe("schema evolution", function () {
     const table = await con.createTable("vectors", [
       { id: 1n, vector: [0.1, 0.2] },
     ]);
-    await table.dropColumns(["vector"]);
+    const dropColumnsRes = await table.dropColumns(["vector"]);
+    expect(dropColumnsRes).toHaveProperty("version");
+    expect(dropColumnsRes.version).toBe(2);
 
     const expectedSchema = new Schema([new Field("id", new Int64(), true)]);
     expect(await table.schema()).toEqual(expectedSchema);
@@ -979,6 +1236,99 @@ describe("when dealing with versioning", () => {
   });
 });
 
+describe("when dealing with tags", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("can manage tags", async () => {
+    const conn = await connect(tmpDir.name, {
+      readConsistencyInterval: 0,
+    });
+
+    const table = await conn.createTable("my_table", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    expect(await table.version()).toBe(1);
+
+    await table.add([{ id: 2n, vector: [0.3, 0.4] }]);
+    expect(await table.version()).toBe(2);
+
+    const tagsManager = await table.tags();
+
+    const initialTags = await tagsManager.list();
+    expect(Object.keys(initialTags).length).toBe(0);
+
+    const tag1 = "tag1";
+    await tagsManager.create(tag1, 1);
+    expect(await tagsManager.getVersion(tag1)).toBe(1);
+
+    const tagsAfterFirst = await tagsManager.list();
+    expect(Object.keys(tagsAfterFirst).length).toBe(1);
+    expect(tagsAfterFirst).toHaveProperty(tag1);
+    expect(tagsAfterFirst[tag1].version).toBe(1);
+
+    await tagsManager.create("tag2", 2);
+    expect(await tagsManager.getVersion("tag2")).toBe(2);
+
+    const tagsAfterSecond = await tagsManager.list();
+    expect(Object.keys(tagsAfterSecond).length).toBe(2);
+    expect(tagsAfterSecond).toHaveProperty(tag1);
+    expect(tagsAfterSecond[tag1].version).toBe(1);
+    expect(tagsAfterSecond).toHaveProperty("tag2");
+    expect(tagsAfterSecond["tag2"].version).toBe(2);
+
+    await table.add([{ id: 3n, vector: [0.5, 0.6] }]);
+    await tagsManager.update(tag1, 3);
+    expect(await tagsManager.getVersion(tag1)).toBe(3);
+
+    await tagsManager.delete("tag2");
+    const tagsAfterDelete = await tagsManager.list();
+    expect(Object.keys(tagsAfterDelete).length).toBe(1);
+    expect(tagsAfterDelete).toHaveProperty(tag1);
+    expect(tagsAfterDelete[tag1].version).toBe(3);
+
+    await table.add([{ id: 4n, vector: [0.7, 0.8] }]);
+    expect(await table.version()).toBe(4);
+
+    await table.checkout(tag1);
+    expect(await table.version()).toBe(3);
+
+    await table.checkoutLatest();
+    expect(await table.version()).toBe(4);
+  });
+
+  it("can checkout and restore tags", async () => {
+    const conn = await connect(tmpDir.name, {
+      readConsistencyInterval: 0,
+    });
+
+    const table = await conn.createTable("my_table", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    expect(await table.version()).toBe(1);
+    expect(await table.countRows()).toBe(1);
+    const tagsManager = await table.tags();
+    const tag1 = "tag1";
+    await tagsManager.create(tag1, 1);
+    await table.add([{ id: 2n, vector: [0.3, 0.4] }]);
+    const tag2 = "tag2";
+    await tagsManager.create(tag2, 2);
+    expect(await table.version()).toBe(2);
+    await table.checkout(tag1);
+    expect(await table.version()).toBe(1);
+    await table.restore();
+    expect(await table.version()).toBe(3);
+    expect(await table.countRows()).toBe(1);
+    await table.add([{ id: 3n, vector: [0.5, 0.6] }]);
+    expect(await table.countRows()).toBe(2);
+  });
+});
+
 describe("when optimizing a dataset", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;
@@ -1114,6 +1464,56 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
 
       const results = await table.search("hello").toArray();
       expect(results[0].text).toBe(data[0].text);
+
+      const query = new MatchQuery("goodbye", "text");
+      expect(instanceOfFullTextQuery(query)).toBe(true);
+      const results2 = await table
+        .search(new MatchQuery("goodbye", "text"))
+        .toArray();
+      expect(results2[0].text).toBe(data[1].text);
+    });
+
+    test("prewarm full text search index", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: ["lance database", "the", "search"], vector: [0.1, 0.2, 0.3] },
+        { text: ["lance database"], vector: [0.4, 0.5, 0.6] },
+        { text: ["lance", "search"], vector: [0.7, 0.8, 0.9] },
+        { text: ["database", "search"], vector: [1.0, 1.1, 1.2] },
+        { text: ["unrelated", "doc"], vector: [1.3, 1.4, 1.5] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts(),
+      });
+
+      // For the moment, we just confirm we can call prewarmIndex without error
+      // and still search it afterwards
+      await table.prewarmIndex("text_idx");
+
+      const results = await table.search("lance").toArray();
+      expect(results.length).toBe(3);
+    });
+
+    test("full text index on list", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: ["lance database", "the", "search"], vector: [0.1, 0.2, 0.3] },
+        { text: ["lance database"], vector: [0.4, 0.5, 0.6] },
+        { text: ["lance", "search"], vector: [0.7, 0.8, 0.9] },
+        { text: ["database", "search"], vector: [1.0, 1.1, 1.2] },
+        { text: ["unrelated", "doc"], vector: [1.3, 1.4, 1.5] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts(),
+      });
+
+      const results = await table.search("lance").toArray();
+      expect(results.length).toBe(3);
+
+      const results2 = await table.search('"lance database"').toArray();
+      expect(results2.length).toBe(2);
     });
 
     test("full text search without positions", async () => {
@@ -1166,6 +1566,43 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results.length).toBe(2);
       const phraseResults = await table.search('"hello world"').toArray();
       expect(phraseResults.length).toBe(1);
+      const phraseResults2 = await table
+        .search(new PhraseQuery("hello world", "text"))
+        .toArray();
+      expect(phraseResults2.length).toBe(1);
+    });
+
+    test("full text search fuzzy query", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "fa", vector: [0.1, 0.2, 0.3] },
+        { text: "fo", vector: [0.4, 0.5, 0.6] },
+        { text: "fob", vector: [0.4, 0.5, 0.6] },
+        { text: "focus", vector: [0.4, 0.5, 0.6] },
+        { text: "foo", vector: [0.4, 0.5, 0.6] },
+        { text: "food", vector: [0.4, 0.5, 0.6] },
+        { text: "foul", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts(),
+      });
+
+      const results = await table
+        .search(new MatchQuery("foo", "text"))
+        .toArray();
+      expect(results.length).toBe(1);
+      expect(results[0].text).toBe("foo");
+
+      const fuzzyResults = await table
+        .search(new MatchQuery("foo", "text", { fuzziness: 1 }))
+        .toArray();
+      expect(fuzzyResults.length).toBe(4);
+      const resultSet = new Set(fuzzyResults.map((r) => r.text));
+      expect(resultSet.has("foo")).toBe(true);
+      expect(resultSet.has("fob")).toBe(true);
+      expect(resultSet.has("fo")).toBe(true);
+      expect(resultSet.has("food")).toBe(true);
     });
 
     test.each([
@@ -1213,6 +1650,30 @@ describe("when calling explainPlan", () => {
   });
 });
 
+describe("when calling analyzePlan", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+  let queryVec: number[];
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const con = await connect(tmpDir.name);
+    table = await con.createTable("vectors", [{ id: 1, vector: [1.1, 0.9] }]);
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("retrieves runtime metrics", async () => {
+    queryVec = Array(2)
+      .fill(1)
+      .map(() => Math.random());
+    const plan = await table.query().nearestTo(queryVec).analyzePlan();
+    console.log("Query Plan:\n", plan); // <--- Print the plan
+    expect(plan).toMatch("AnalyzeExec");
+  });
+});
+
 describe("column name options", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;

nodejs/examples/basic.test.ts

@@ -132,6 +132,17 @@ test("basic table examples", async () => {
       },
     ]);
     // --8<-- [end:alter_columns]
+    // --8<-- [start:alter_columns_vector]
+    await tbl.alterColumns([
+      {
+        path: "vector",
+        dataType: new arrow.FixedSizeList(
+          2,
+          new arrow.Field("item", new arrow.Float16(), false),
+        ),
+      },
+    ]);
+    // --8<-- [end:alter_columns_vector]
     // --8<-- [start:drop_columns]
     await tbl.dropColumns(["dbl_price"]);
     // --8<-- [end:drop_columns]

nodejs/examples/search.test.ts

@@ -4,9 +4,12 @@ import { expect, test } from "@jest/globals";
 // --8<-- [start:import]
 import * as lancedb from "@lancedb/lancedb";
 // --8<-- [end:import]
+// --8<-- [start:import_bin_util]
+import { Field, FixedSizeList, Int32, Schema, Uint8 } from "apache-arrow";
+// --8<-- [end:import_bin_util]
 import { withTempDirectory } from "./util.ts";
 
-test("full text search", async () => {
+test("vector search", async () => {
   await withTempDirectory(async (databaseDir) => {
     {
       const db = await lancedb.connect(databaseDir);
@@ -14,8 +17,6 @@ test("full text search", async () => {
       const data = Array.from({ length: 10_000 }, (_, i) => ({
         vector: Array(128).fill(i),
         id: `${i}`,
-        content: "",
-        longId: `${i}`,
       }));
 
       await db.createTable("my_vectors", data);
@@ -52,5 +53,41 @@ test("full text search", async () => {
       expect(r.distance).toBeGreaterThanOrEqual(0.1);
       expect(r.distance).toBeLessThan(0.2);
     }
+
+    {
+      // --8<-- [start:ingest_binary_data]
+      const schema = new Schema([
+        new Field("id", new Int32(), true),
+        new Field("vec", new FixedSizeList(32, new Field("item", new Uint8()))),
+      ]);
+      const data = lancedb.makeArrowTable(
+        Array(1_000)
+          .fill(0)
+          .map((_, i) => ({
+            // the 256 bits would be store in 32 bytes,
+            // if your data is already in this format, you can skip the packBits step
+            id: i,
+            vec: lancedb.packBits(Array(256).fill(i % 2)),
+          })),
+        { schema: schema },
+      );
+
+      const tbl = await db.createTable("binary_table", data);
+      await tbl.createIndex("vec", {
+        config: lancedb.Index.ivfFlat({
+          numPartitions: 10,
+          distanceType: "hamming",
+        }),
+      });
+      // --8<-- [end:ingest_binary_data]
+
+      // --8<-- [start:search_binary_data]
+      const query = Array(32)
+        .fill(1)
+        .map(() => Math.floor(Math.random() * 255));
+      const results = await tbl.query().nearestTo(query).limit(10).toArrow();
+      // --8<-- [end:search_binary_data
+      expect(results.numRows).toBe(10);
+    }
   });
 });

nodejs/lancedb/arrow.ts

@@ -8,7 +8,11 @@ import {
   Bool,
   BufferType,
   DataType,
+  DateUnit,
+  Date_,
+  Decimal,
   Dictionary,
+  Duration,
   Field,
   FixedSizeBinary,
   FixedSizeList,
@@ -21,19 +25,22 @@ import {
   LargeBinary,
   List,
   Null,
+  Precision,
   RecordBatch,
   RecordBatchFileReader,
   RecordBatchFileWriter,
   RecordBatchStreamWriter,
   Schema,
   Struct,
+  Timestamp,
+  Type,
   Utf8,
   Vector,
   makeVector as arrowMakeVector,
+  vectorFromArray as badVectorFromArray,
   makeBuilder,
   makeData,
   makeTable,
-  vectorFromArray,
 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
@@ -179,6 +186,21 @@ export class VectorColumnOptions {
   }
 }
 
+// biome-ignore lint/suspicious/noExplicitAny: skip
+function vectorFromArray(data: any, type?: DataType) {
+  // Workaround for: https://github.com/apache/arrow/issues/45862
+  // If FSL type with float
+  if (DataType.isFixedSizeList(type) && DataType.isFloat(type.valueType)) {
+    const extendedData = [...data, new Array(type.listSize).fill(0.0)];
+    const array = badVectorFromArray(extendedData, type);
+    return array.slice(0, data.length);
+  } else if (type === undefined) {
+    return badVectorFromArray(data);
+  } else {
+    return badVectorFromArray(data, type);
+  }
+}
+
 /** Options to control the makeArrowTable call. */
 export class MakeArrowTableOptions {
   /*
@@ -617,8 +639,9 @@ function transposeData(
 ): Vector {
   if (field.type instanceof Struct) {
     const childFields = field.type.children;
+    const fullPath = [...path, field.name];
     const childVectors = childFields.map((child) => {
-      return transposeData(data, child, [...path, child.name]);
+      return transposeData(data, child, fullPath);
     });
     const structData = makeData({
       type: field.type,
@@ -630,7 +653,14 @@ function transposeData(
     const values = data.map((datum) => {
       let current: unknown = datum;
       for (const key of valuesPath) {
-        if (isObject(current) && Object.hasOwn(current, key)) {
+        if (current == null) {
+          return null;
+        }
+
+        if (
+          isObject(current) &&
+          (Object.hasOwn(current, key) || key in current)
+        ) {
           current = current[key];
         } else {
           return null;
@@ -1170,3 +1200,137 @@ function validateSchemaEmbeddings(
 
   return new Schema(fields, schema.metadata);
 }
+
+interface JsonDataType {
+  type: string;
+  fields?: JsonField[];
+  length?: number;
+}
+
+interface JsonField {
+  name: string;
+  type: JsonDataType;
+  nullable: boolean;
+  metadata: Map<string, string>;
+}
+
+// Matches format of https://github.com/lancedb/lance/blob/main/rust/lance/src/arrow/json.rs
+export function dataTypeToJson(dataType: DataType): JsonDataType {
+  switch (dataType.typeId) {
+    // For primitives, matches https://github.com/lancedb/lance/blob/e12bb9eff2a52f753668d4b62c52e4d72b10d294/rust/lance-core/src/datatypes.rs#L185
+    case Type.Null:
+      return { type: "null" };
+    case Type.Bool:
+      return { type: "bool" };
+    case Type.Int8:
+      return { type: "int8" };
+    case Type.Int16:
+      return { type: "int16" };
+    case Type.Int32:
+      return { type: "int32" };
+    case Type.Int64:
+      return { type: "int64" };
+    case Type.Uint8:
+      return { type: "uint8" };
+    case Type.Uint16:
+      return { type: "uint16" };
+    case Type.Uint32:
+      return { type: "uint32" };
+    case Type.Uint64:
+      return { type: "uint64" };
+    case Type.Int: {
+      const bitWidth = (dataType as Int).bitWidth;
+      const signed = (dataType as Int).isSigned;
+      const prefix = signed ? "" : "u";
+      return { type: `${prefix}int${bitWidth}` };
+    }
+    case Type.Float: {
+      switch ((dataType as Float).precision) {
+        case Precision.HALF:
+          return { type: "halffloat" };
+        case Precision.SINGLE:
+          return { type: "float" };
+        case Precision.DOUBLE:
+          return { type: "double" };
+      }
+      throw Error("Unsupported float precision");
+    }
+    case Type.Float16:
+      return { type: "halffloat" };
+    case Type.Float32:
+      return { type: "float" };
+    case Type.Float64:
+      return { type: "double" };
+    case Type.Utf8:
+      return { type: "string" };
+    case Type.Binary:
+      return { type: "binary" };
+    case Type.LargeUtf8:
+      return { type: "large_string" };
+    case Type.LargeBinary:
+      return { type: "large_binary" };
+    case Type.List:
+      return {
+        type: "list",
+        fields: [fieldToJson((dataType as List).children[0])],
+      };
+    case Type.FixedSizeList: {
+      const fixedSizeList = dataType as FixedSizeList;
+      return {
+        type: "fixed_size_list",
+        fields: [fieldToJson(fixedSizeList.children[0])],
+        length: fixedSizeList.listSize,
+      };
+    }
+    case Type.Struct:
+      return {
+        type: "struct",
+        fields: (dataType as Struct).children.map(fieldToJson),
+      };
+    case Type.Date: {
+      const unit = (dataType as Date_).unit;
+      return {
+        type: unit === DateUnit.DAY ? "date32:day" : "date64:ms",
+      };
+    }
+    case Type.Timestamp: {
+      const timestamp = dataType as Timestamp;
+      const timezone = timestamp.timezone || "-";
+      return {
+        type: `timestamp:${timestamp.unit}:${timezone}`,
+      };
+    }
+    case Type.Decimal: {
+      const decimal = dataType as Decimal;
+      return {
+        type: `decimal:${decimal.bitWidth}:${decimal.precision}:${decimal.scale}`,
+      };
+    }
+    case Type.Duration: {
+      const duration = dataType as Duration;
+      return { type: `duration:${duration.unit}` };
+    }
+    case Type.FixedSizeBinary: {
+      const byteWidth = (dataType as FixedSizeBinary).byteWidth;
+      return { type: `fixed_size_binary:${byteWidth}` };
+    }
+    case Type.Dictionary: {
+      const dict = dataType as Dictionary;
+      const indexType = dataTypeToJson(dict.indices);
+      const valueType = dataTypeToJson(dict.valueType);
+      return {
+        type: `dict:${valueType.type}:${indexType.type}:false`,
+      };
+    }
+  }
+  throw new Error("Unsupported data type");
+}
+
+function fieldToJson(field: Field): JsonField {
+  return {
+    name: field.name,
+    type: dataTypeToJson(field.type),
+    nullable: field.nullable,
+    metadata: field.metadata,
+  };
+}

nodejs/lancedb/index.ts

@@ -14,7 +14,6 @@ import {
 
 export {
   AddColumnsSql,
-  ColumnAlteration,
   ConnectionOptions,
   IndexStatistics,
   IndexConfig,
@@ -24,6 +23,18 @@ export {
   OptimizeStats,
   CompactionStats,
   RemovalStats,
+  TableStatistics,
+  FragmentStatistics,
+  FragmentSummaryStats,
+  Tags,
+  TagContents,
+  MergeResult,
+  AddResult,
+  AddColumnsResult,
+  AlterColumnsResult,
+  DeleteResult,
+  DropColumnsResult,
+  UpdateResult,
 } from "./native.js";
 
 export {
@@ -48,12 +59,19 @@ export {
   QueryExecutionOptions,
   FullTextSearchOptions,
   RecordBatchIterator,
+  FullTextQuery,
+  MatchQuery,
+  PhraseQuery,
+  BoostQuery,
+  MultiMatchQuery,
+  FullTextQueryType,
 } from "./query";
 
 export {
   Index,
   IndexOptions,
   IvfPqOptions,
+  IvfFlatOptions,
   HnswPqOptions,
   HnswSqOptions,
   FtsOptions,
@@ -65,9 +83,10 @@ export {
   UpdateOptions,
   OptimizeOptions,
   Version,
+  ColumnAlteration,
 } from "./table";
 
-export { MergeInsertBuilder } from "./merge";
+export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
 
 export * as embedding from "./embedding";
 export * as rerankers from "./rerankers";
@@ -79,7 +98,7 @@ export {
   DataLike,
   IntoVector,
 } from "./arrow";
-export { IntoSql } from "./util";
+export { IntoSql, packBits } from "./util";
 
 /**
  * Connect to a LanceDB instance at the given URI.

nodejs/lancedb/indices.ts

@@ -62,13 +62,13 @@ export interface IvfPqOptions {
    *
    * "l2" - Euclidean distance. This is a very common distance metric that
    * accounts for both magnitude and direction when determining the distance
-   * between vectors. L2 distance has a range of [0, ∞).
+   * between vectors. l2 distance has a range of [0, ∞).
    *
    * "cosine" - Cosine distance.  Cosine distance is a distance metric
    * calculated from the cosine similarity between two vectors. Cosine
    * similarity is a measure of similarity between two non-zero vectors of an
    * inner product space. It is defined to equal the cosine of the angle
-   * between them.  Unlike L2, the cosine distance is not affected by the
+   * between them.  Unlike l2, the cosine distance is not affected by the
    * magnitude of the vectors.  Cosine distance has a range of [0, 2].
    *
    * Note: the cosine distance is undefined when one (or both) of the vectors
@@ -77,7 +77,7 @@ export interface IvfPqOptions {
    *
    * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
    * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   * l2 norm is 1), then dot distance is equivalent to the cosine distance.
    */
   distanceType?: "l2" | "cosine" | "dot";
 
@@ -125,18 +125,18 @@ export interface HnswPqOptions {
    *
    * "l2" - Euclidean distance. This is a very common distance metric that
    * accounts for both magnitude and direction when determining the distance
-   * between vectors. L2 distance has a range of [0, ∞).
+   * between vectors. l2 distance has a range of [0, ∞).
    *
    * "cosine" - Cosine distance.  Cosine distance is a distance metric
    * calculated from the cosine similarity between two vectors. Cosine
    * similarity is a measure of similarity between two non-zero vectors of an
    * inner product space. It is defined to equal the cosine of the angle
-   * between them.  Unlike L2, the cosine distance is not affected by the
+   * between them.  Unlike l2, the cosine distance is not affected by the
    * magnitude of the vectors.  Cosine distance has a range of [0, 2].
    *
    * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
    * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   * l2 norm is 1), then dot distance is equivalent to the cosine distance.
    */
   distanceType?: "l2" | "cosine" | "dot";
 
@@ -241,18 +241,18 @@ export interface HnswSqOptions {
    *
    * "l2" - Euclidean distance. This is a very common distance metric that
    * accounts for both magnitude and direction when determining the distance
-   * between vectors. L2 distance has a range of [0, ∞).
+   * between vectors. l2 distance has a range of [0, ∞).
    *
    * "cosine" - Cosine distance.  Cosine distance is a distance metric
    * calculated from the cosine similarity between two vectors. Cosine
    * similarity is a measure of similarity between two non-zero vectors of an
    * inner product space. It is defined to equal the cosine of the angle
-   * between them.  Unlike L2, the cosine distance is not affected by the
+   * between them.  Unlike l2, the cosine distance is not affected by the
    * magnitude of the vectors.  Cosine distance has a range of [0, 2].
    *
    * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
    * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   * l2 norm is 1), then dot distance is equivalent to the cosine distance.
    */
   distanceType?: "l2" | "cosine" | "dot";
 
@@ -327,6 +327,94 @@ export interface HnswSqOptions {
   efConstruction?: number;
 }
 
+/**
+ * Options to create an `IVF_FLAT` index
+ */
+export interface IvfFlatOptions {
+  /**
+   * The number of IVF partitions to create.
+   *
+   * This value should generally scale with the number of rows in the dataset.
+   * By default the number of partitions is the square root of the number of
+   * rows.
+   *
+   * If this value is too large then the first part of the search (picking the
+   * right partition) will be slow.  If this value is too small then the second
+   * part of the search (searching within a partition) will be slow.
+   */
+  numPartitions?: number;
+
+  /**
+   * Distance type to use to build the index.
+   *
+   * Default value is "l2".
+   *
+   * This is used when training the index to calculate the IVF partitions
+   * (vectors are grouped in partitions with similar vectors according to this
+   * distance type).
+   *
+   * The distance type used to train an index MUST match the distance type used
+   * to search the index.  Failure to do so will yield inaccurate results.
+   *
+   * The following distance types are available:
+   *
+   * "l2" - Euclidean distance. This is a very common distance metric that
+   * accounts for both magnitude and direction when determining the distance
+   * between vectors. l2 distance has a range of [0, ∞).
+   *
+   * "cosine" - Cosine distance.  Cosine distance is a distance metric
+   * calculated from the cosine similarity between two vectors. Cosine
+   * similarity is a measure of similarity between two non-zero vectors of an
+   * inner product space. It is defined to equal the cosine of the angle
+   * between them.  Unlike l2, the cosine distance is not affected by the
+   * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+   *
+   * Note: the cosine distance is undefined when one (or both) of the vectors
+   * are all zeros (there is no direction).  These vectors are invalid and may
+   * never be returned from a vector search.
+   *
+   * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+   * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+   * l2 norm is 1), then dot distance is equivalent to the cosine distance.
+   *
+   * "hamming" - Hamming distance. Hamming distance is a distance metric
+   * calculated from the number of bits that are different between two vectors.
+   * Hamming distance has a range of [0, dimension]. Note that the hamming distance
+   * is only valid for binary vectors.
+   */
+  distanceType?: "l2" | "cosine" | "dot" | "hamming";
+
+  /**
+   * Max iteration to train IVF kmeans.
+   *
+   * When training an IVF FLAT index we use kmeans to calculate the partitions.  This parameter
+   * controls how many iterations of kmeans to run.
+   *
+   * Increasing this might improve the quality of the index but in most cases these extra
+   * iterations have diminishing returns.
+   *
+   * The default value is 50.
+   */
+  maxIterations?: number;
+
+  /**
+   * The number of vectors, per partition, to sample when training IVF kmeans.
+   *
+   * When an IVF FLAT index is trained, we need to calculate partitions.  These are groups
+   * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+   *
+   * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+   * random sample of the data.  This parameter controls the size of the sample.  The total
+   * number of vectors used to train the index is `sample_rate * num_partitions`.
+   *
+   * Increasing this value might improve the quality of the index but in most cases the
+   * default should be sufficient.
+   *
+   * The default value is 256.
+   */
+  sampleRate?: number;
+}
+
 /**
  * Options to create a full text search index
  */
@@ -426,6 +514,33 @@ export class Index {
     );
   }
 
+  /**
+   * Create an IvfFlat index
+   *
+   * This index groups vectors into partitions of similar vectors.  Each partition keeps track of
+   * a centroid which is the average value of all vectors in the group.
+   *
+   * During a query the centroids are compared with the query vector to find the closest
+   * partitions.  The vectors in these partitions are then searched to find
+   * the closest vectors.
+   *
+   * The partitioning process is called IVF and the `num_partitions` parameter controls how
+   * many groups to create.
+   *
+   * Note that training an IVF FLAT index on a large dataset is a slow operation and
+   * currently is also a memory intensive operation.
+   */
+  static ivfFlat(options?: Partial<IvfFlatOptions>) {
+    return new Index(
+      LanceDbIndex.ivfFlat(
+        options?.distanceType,
+        options?.numPartitions,
+        options?.maxIterations,
+        options?.sampleRate,
+      ),
+    );
+  }
+
   /**
    * Create a btree index
    *
@@ -566,4 +681,6 @@ export interface IndexOptions {
    * The default is true
    */
   replace?: boolean;
+
+  waitTimeoutSeconds?: number;
 }

nodejs/lancedb/merge.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import { Data, Schema, fromDataToBuffer } from "./arrow";
-import { NativeMergeInsertBuilder } from "./native";
+import { MergeResult, NativeMergeInsertBuilder } from "./native";
 
 /** A builder used to create and run a merge insert operation */
 export class MergeInsertBuilder {
@@ -73,9 +73,12 @@ export class MergeInsertBuilder {
   /**
    * Executes the merge insert operation
    *
-   * Nothing is returned but the `Table` is updated
+   * @returns {Promise<MergeResult>} the merge result
    */
-  async execute(data: Data): Promise<void> {
+  async execute(
+    data: Data,
+    execOptions?: Partial<WriteExecutionOptions>,
+  ): Promise<MergeResult> {
     let schema: Schema;
     if (this.#schema instanceof Promise) {
       schema = await this.#schema;
@@ -83,7 +86,28 @@ export class MergeInsertBuilder {
     } else {
       schema = this.#schema;
     }
+
+    if (execOptions?.timeoutMs !== undefined) {
+      this.#native.setTimeout(execOptions.timeoutMs);
+    }
+
     const buffer = await fromDataToBuffer(data, undefined, schema);
-    await this.#native.execute(buffer);
+    return await this.#native.execute(buffer);
   }
 }
+
+export interface WriteExecutionOptions {
+  /**
+   * Maximum time to run the operation before cancelling it.
+   *
+   * By default, there is a 30-second timeout that is only enforced after the
+   * first attempt. This is to prevent spending too long retrying to resolve
+   * conflicts. For example, if a write attempt takes 20 seconds and fails,
+   * the second attempt will be cancelled after 10 seconds, hitting the
+   * 30-second timeout. However, a write that takes one hour and succeeds on the
+   * first attempt will not be cancelled.
+   *
+   * When this is set, the timeout is enforced on all attempts, including the first.
+   */
+  timeoutMs?: number;
+}

nodejs/lancedb/query.ts

@@ -11,12 +11,14 @@ import {
 } from "./arrow";
 import { type IvfPqOptions } from "./indices";
 import {
+  JsFullTextQuery,
   RecordBatchIterator as NativeBatchIterator,
   Query as NativeQuery,
   Table as NativeTable,
   VectorQuery as NativeVectorQuery,
 } from "./native";
 import { Reranker } from "./rerankers";
+
 export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
@@ -62,7 +64,7 @@ class RecordBatchIterable<
   // biome-ignore lint/suspicious/noExplicitAny: skip
   [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>, any, undefined> {
     return new RecordBatchIterator(
-      this.inner.execute(this.options?.maxBatchLength),
+      this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs),
     );
   }
 }
@@ -78,6 +80,11 @@ export interface QueryExecutionOptions {
    * in smaller chunks.
    */
   maxBatchLength?: number;
+
+  /**
+   * Timeout for query execution in milliseconds
+   */
+  timeoutMs?: number;
 }
 
 /**
@@ -152,7 +159,7 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
   }
 
   fullTextSearch(
-    query: string,
+    query: string | FullTextQuery,
     options?: Partial<FullTextSearchOptions>,
   ): this {
     let columns: string[] | null = null;
@@ -164,9 +171,16 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
       }
     }
 
-    this.doCall((inner: NativeQueryType) =>
+    this.doCall((inner: NativeQueryType) => {
-      inner.fullTextSearch(query, columns),
+      if (typeof query === "string") {
-    );
+        inner.fullTextSearch({
+          query: query,
+          columns: columns,
+        });
+      } else {
+        inner.fullTextSearch({ query: query.inner });
+      }
+    });
     return this;
   }
 
@@ -273,9 +287,11 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
     options?: Partial<QueryExecutionOptions>,
   ): Promise<NativeBatchIterator> {
     if (this.inner instanceof Promise) {
-      return this.inner.then((inner) => inner.execute(options?.maxBatchLength));
+      return this.inner.then((inner) =>
+        inner.execute(options?.maxBatchLength, options?.timeoutMs),
+      );
     } else {
-      return this.inner.execute(options?.maxBatchLength);
+      return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
     }
   }
 
@@ -348,6 +364,43 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
       return this.inner.explainPlan(verbose);
     }
   }
+
+  /**
+   * Executes the query and returns the physical query plan annotated with runtime metrics.
+   *
+   * This is useful for debugging and performance analysis, as it shows how the query was executed
+   * and includes metrics such as elapsed time, rows processed, and I/O statistics.
+   *
+   * @example
+   * import * as lancedb from "@lancedb/lancedb"
+   *
+   * const db = await lancedb.connect("./.lancedb");
+   * const table = await db.createTable("my_table", [
+   *   { vector: [1.1, 0.9], id: "1" },
+   * ]);
+   *
+   * const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+   *
+   * Example output (with runtime metrics inlined):
+   * AnalyzeExec verbose=true, metrics=[]
+   *  ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+   *   Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   *    CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+   *     GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+   *      FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+   *       SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+   *        KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+   *         LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+   *
+   * @returns A query execution plan with runtime metrics for each step.
+   */
+  async analyzePlan(): Promise<string> {
+    if (this.inner instanceof Promise) {
+      return this.inner.then((inner) => inner.analyzePlan());
+    } else {
+      return this.inner.analyzePlan();
+    }
+  }
 }
 
 /**
@@ -681,8 +734,177 @@ export class Query extends QueryBase<NativeQuery> {
     }
   }
 
-  nearestToText(query: string, columns?: string[]): Query {
+  nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
-    this.doCall((inner) => inner.fullTextSearch(query, columns));
+    this.doCall((inner) => {
+      if (typeof query === "string") {
+        inner.fullTextSearch({
+          query: query,
+          columns: columns,
+        });
+      } else {
+        inner.fullTextSearch({ query: query.inner });
+      }
+    });
     return this;
   }
 }
+
+/**
+ * Enum representing the types of full-text queries supported.
+ *
+ * - `Match`: Performs a full-text search for terms in the query string.
+ * - `MatchPhrase`: Searches for an exact phrase match in the text.
+ * - `Boost`: Boosts the relevance score of specific terms in the query.
+ * - `MultiMatch`: Searches across multiple fields for the query terms.
+ */
+export enum FullTextQueryType {
+  Match = "match",
+  MatchPhrase = "match_phrase",
+  Boost = "boost",
+  MultiMatch = "multi_match",
+}
+
+/**
+ * Represents a full-text query interface.
+ * This interface defines the structure and behavior for full-text queries,
+ * including methods to retrieve the query type and convert the query to a dictionary format.
+ */
+export interface FullTextQuery {
+  /**
+   * Returns the inner query object.
+   * This is the underlying query object used by the database engine.
+   * @ignore
+   */
+  inner: JsFullTextQuery;
+
+  /**
+   * The type of the full-text query.
+   */
+  queryType(): FullTextQueryType;
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: we want any here
+export function instanceOfFullTextQuery(obj: any): obj is FullTextQuery {
+  return obj != null && obj.inner instanceof JsFullTextQuery;
+}
+
+export class MatchQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of MatchQuery.
+   *
+   * @param query - The text query to search for.
+   * @param column - The name of the column to search within.
+   * @param options - Optional parameters for the match query.
+   *   - `boost`: The boost factor for the query (default is 1.0).
+   *   - `fuzziness`: The fuzziness level for the query (default is 0).
+   *   - `maxExpansions`: The maximum number of terms to consider for fuzzy matching (default is 50).
+   */
+  constructor(
+    query: string,
+    column: string,
+    options?: {
+      boost?: number;
+      fuzziness?: number;
+      maxExpansions?: number;
+    },
+  ) {
+    let fuzziness = options?.fuzziness;
+    if (fuzziness === undefined) {
+      fuzziness = 0;
+    }
+    this.inner = JsFullTextQuery.matchQuery(
+      query,
+      column,
+      options?.boost ?? 1.0,
+      fuzziness,
+      options?.maxExpansions ?? 50,
+    );
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.Match;
+  }
+}
+
+export class PhraseQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of `PhraseQuery`.
+   *
+   * @param query - The phrase to search for in the specified column.
+   * @param column - The name of the column to search within.
+   */
+  constructor(query: string, column: string) {
+    this.inner = JsFullTextQuery.phraseQuery(query, column);
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.MatchPhrase;
+  }
+}
+
+export class BoostQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of BoostQuery.
+   * The boost returns documents that match the positive query,
+   * but penalizes those that match the negative query.
+   * the penalty is controlled by the `negativeBoost` parameter.
+   *
+   * @param positive - The positive query that boosts the relevance score.
+   * @param negative - The negative query that reduces the relevance score.
+   * @param options - Optional parameters for the boost query.
+   *  - `negativeBoost`: The boost factor for the negative query (default is 0.0).
+   */
+  constructor(
+    positive: FullTextQuery,
+    negative: FullTextQuery,
+    options?: {
+      negativeBoost?: number;
+    },
+  ) {
+    this.inner = JsFullTextQuery.boostQuery(
+      positive.inner,
+      negative.inner,
+      options?.negativeBoost,
+    );
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.Boost;
+  }
+}
+
+export class MultiMatchQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of MultiMatchQuery.
+   *
+   * @param query - The text query to search for across multiple columns.
+   * @param columns - An array of column names to search within.
+   * @param options - Optional parameters for the multi-match query.
+   *  - `boosts`: An array of boost factors for each column (default is 1.0 for all).
+   */
+  constructor(
+    query: string,
+    columns: string[],
+    options?: {
+      boosts?: number[];
+    },
+  ) {
+    this.inner = JsFullTextQuery.multiMatchQuery(
+      query,
+      columns,
+      options?.boosts,
+    );
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.MultiMatch;
+  }
+}

nodejs/lancedb/table.ts

@@ -4,8 +4,10 @@
 import {
   Table as ArrowTable,
   Data,
+  DataType,
   IntoVector,
   Schema,
+  dataTypeToJson,
   fromDataToBuffer,
   tableFromIPC,
 } from "./arrow";
@@ -14,14 +16,27 @@ import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
 import { MergeInsertBuilder } from "./merge";
 import {
+  AddColumnsResult,
   AddColumnsSql,
-  ColumnAlteration,
+  AddResult,
+  AlterColumnsResult,
+  DeleteResult,
+  DropColumnsResult,
   IndexConfig,
   IndexStatistics,
   OptimizeStats,
+  TableStatistics,
+  Tags,
+  UpdateResult,
   Table as _NativeTable,
 } from "./native";
-import { Query, VectorQuery } from "./query";
+import {
+  FullTextQuery,
+  Query,
+  VectorQuery,
+  instanceOfFullTextQuery,
+} from "./query";
+import { sanitizeType } from "./sanitize";
 import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
 
@@ -117,12 +132,19 @@ export abstract class Table {
   /**
    * Insert records into this Table.
    * @param {Data} data Records to be inserted into the Table
+   * @returns {Promise<AddResult>} A promise that resolves to an object
+   * containing the new version number of the table
    */
-  abstract add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
+  abstract add(
+    data: Data,
+    options?: Partial<AddDataOptions>,
+  ): Promise<AddResult>;
   /**
    * Update existing records in the Table
    * @param opts.values The values to update. The keys are the column names and the values
    * are the values to set.
+   * @returns {Promise<UpdateResult>} A promise that resolves to an object containing
+   * the number of rows updated and the new version number
    * @example
    * ```ts
    * table.update({where:"x = 2", values:{"vector": [10, 10]}})
@@ -132,11 +154,13 @@ export abstract class Table {
     opts: {
       values: Map<string, IntoSql> | Record<string, IntoSql>;
     } & Partial<UpdateOptions>,
-  ): Promise<void>;
+  ): Promise<UpdateResult>;
   /**
    * Update existing records in the Table
    * @param opts.valuesSql The values to update. The keys are the column names and the values
    * are the values to set. The values are SQL expressions.
+   * @returns {Promise<UpdateResult>} A promise that resolves to an object containing
+   * the number of rows updated and the new version number
    * @example
    * ```ts
    * table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
@@ -146,7 +170,7 @@ export abstract class Table {
     opts: {
       valuesSql: Map<string, string> | Record<string, string>;
     } & Partial<UpdateOptions>,
-  ): Promise<void>;
+  ): Promise<UpdateResult>;
   /**
    * Update existing records in the Table
    *
@@ -164,6 +188,8 @@ export abstract class Table {
    * repeatedly calilng this method.
    * @param {Map<string, string> | Record<string, string>} updates - the
    * columns to update
+   * @returns {Promise<UpdateResult>} A promise that resolves to an object
+   * containing the number of rows updated and the new version number
    *
    * Keys in the map should specify the name of the column to update.
    * Values in the map provide the new value of the column.  These can
@@ -175,12 +201,16 @@ export abstract class Table {
   abstract update(
     updates: Map<string, string> | Record<string, string>,
     options?: Partial<UpdateOptions>,
-  ): Promise<void>;
+  ): Promise<UpdateResult>;
 
   /** Count the total number of rows in the dataset. */
   abstract countRows(filter?: string): Promise<number>;
-  /** Delete the rows that satisfy the predicate. */
+  /**
-  abstract delete(predicate: string): Promise<void>;
+   * Delete the rows that satisfy the predicate.
+   * @returns {Promise<DeleteResult>} A promise that resolves to an object
+   * containing the new version number of the table
+   */
+  abstract delete(predicate: string): Promise<DeleteResult>;
   /**
    * Create an index to speed up queries.
    *
@@ -228,6 +258,30 @@ export abstract class Table {
    */
   abstract dropIndex(name: string): Promise<void>;
 
+  /**
+   * Prewarm an index in the table.
+   *
+   * @param name The name of the index.
+   *
+   * This will load the index into memory.  This may reduce the cold-start time for
+   * future queries.  If the index does not fit in the cache then this call may be
+   * wasteful.
+   */
+  abstract prewarmIndex(name: string): Promise<void>;
+
+  /**
+   * Waits for asynchronous indexing to complete on the table.
+   *
+   * @param indexNames The name of the indices to wait for
+   * @param timeoutSeconds The number of seconds to wait before timing out
+   *
+   * This will raise an error if the indices are not created and fully indexed within the timeout.
+   */
+  abstract waitForIndex(
+    indexNames: string[],
+    timeoutSeconds: number,
+  ): Promise<void>;
+
   /**
    * Create a {@link Query} Builder.
    *
@@ -292,7 +346,7 @@ export abstract class Table {
    * if the query is a string and no embedding function is defined, it will be treated as a full text search query
    */
   abstract search(
-    query: string | IntoVector,
+    query: string | IntoVector | FullTextQuery,
     queryType?: string,
     ftsColumns?: string | string[],
   ): VectorQuery | Query;
@@ -310,15 +364,23 @@ export abstract class Table {
    * the SQL expression to use to calculate the value of the new column. These
    * expressions will be evaluated for each row in the table, and can
    * reference existing columns in the table.
+   * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
+   * containing the new version number of the table after adding the columns.
    */
-  abstract addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void>;
+  abstract addColumns(
+    newColumnTransforms: AddColumnsSql[],
+  ): Promise<AddColumnsResult>;
 
   /**
    * Alter the name or nullability of columns.
    * @param {ColumnAlteration[]} columnAlterations One or more alterations to
    * apply to columns.
+   * @returns {Promise<AlterColumnsResult>} A promise that resolves to an object
+   * containing the new version number of the table after altering the columns.
    */
-  abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
+  abstract alterColumns(
+    columnAlterations: ColumnAlteration[],
+  ): Promise<AlterColumnsResult>;
   /**
    * Drop one or more columns from the dataset
    *
@@ -329,8 +391,10 @@ export abstract class Table {
    * @param {string[]} columnNames The names of the columns to drop. These can
    * be nested column references (e.g. "a.b.c") or top-level column names
    * (e.g. "a").
+   * @returns {Promise<DropColumnsResult>} A promise that resolves to an object
+   * containing the new version number of the table after dropping the columns.
    */
-  abstract dropColumns(columnNames: string[]): Promise<void>;
+  abstract dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
   /** Retrieve the version of the table */
 
   abstract version(): Promise<number>;
@@ -343,7 +407,7 @@ export abstract class Table {
    *
    * Calling this method will set the table into time-travel mode. If you
    * wish to return to standard mode, call `checkoutLatest`.
-   * @param {number} version The version to checkout
+   * @param {number | string} version The version to checkout, could be version number or tag
    * @example
    * ```typescript
    * import * as lancedb from "@lancedb/lancedb"
@@ -359,7 +423,8 @@ export abstract class Table {
    * console.log(await table.version()); // 2
    * ```
    */
-  abstract checkout(version: number): Promise<void>;
+  abstract checkout(version: number | string): Promise<void>;
+
   /**
    * Checkout the latest version of the table. _This is an in-place operation._
    *
@@ -373,6 +438,23 @@ export abstract class Table {
    */
   abstract listVersions(): Promise<Version[]>;
 
+  /**
+   * Get a tags manager for this table.
+   *
+   * Tags allow you to label specific versions of a table with a human-readable name.
+   * The returned tags manager can be used to list, create, update, or delete tags.
+   *
+   * @returns {Tags} A tags manager for this table
+   * @example
+   * ```typescript
+   * const tagsManager = await table.tags();
+   * await tagsManager.create("v1", 1);
+   * const tags = await tagsManager.list();
+   * console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
+   * ```
+   */
+  abstract tags(): Promise<Tags>;
+
   /**
    * Restore the table to the currently checked out version
    *
@@ -432,6 +514,13 @@ export abstract class Table {
    * Use {@link Table.listIndices} to find the names of the indices.
    */
   abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
+
+  /** Returns table and fragment statistics
+   *
+   * @returns {TableStatistics} The table and fragment statistics
+   *
+   */
+  abstract stats(): Promise<TableStatistics>;
 }
 
 export class LocalTable extends Table {
@@ -471,12 +560,12 @@ export class LocalTable extends Table {
     return tbl.schema;
   }
 
-  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+  async add(data: Data, options?: Partial<AddDataOptions>): Promise<AddResult> {
     const mode = options?.mode ?? "append";
     const schema = await this.schema();
 
     const buffer = await fromDataToBuffer(data, undefined, schema);
-    await this.inner.add(buffer, mode);
+    return await this.inner.add(buffer, mode);
   }
 
   async update(
@@ -489,7 +578,7 @@ export class LocalTable extends Table {
           valuesSql: Map<string, string> | Record<string, string>;
         } & Partial<UpdateOptions>),
     options?: Partial<UpdateOptions>,
-  ) {
+  ): Promise<UpdateResult> {
     const isValues =
       "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
     const isValuesSql =
@@ -536,38 +625,54 @@ export class LocalTable extends Table {
         columns = Object.entries(optsOrUpdates as Record<string, string>);
         predicate = options?.where;
     }
-    await this.inner.update(predicate, columns);
+    return await this.inner.update(predicate, columns);
   }
 
   async countRows(filter?: string): Promise<number> {
     return await this.inner.countRows(filter);
   }
 
-  async delete(predicate: string): Promise<void> {
+  async delete(predicate: string): Promise<DeleteResult> {
-    await this.inner.delete(predicate);
+    return await this.inner.delete(predicate);
   }
 
   async createIndex(column: string, options?: Partial<IndexOptions>) {
     // Bit of a hack to get around the fact that TS has no package-scope.
     // biome-ignore lint/suspicious/noExplicitAny: skip
     const nativeIndex = (options?.config as any)?.inner;
-    await this.inner.createIndex(nativeIndex, column, options?.replace);
+    await this.inner.createIndex(
+      nativeIndex,
+      column,
+      options?.replace,
+      options?.waitTimeoutSeconds,
+    );
   }
 
   async dropIndex(name: string): Promise<void> {
     await this.inner.dropIndex(name);
   }
 
+  async prewarmIndex(name: string): Promise<void> {
+    await this.inner.prewarmIndex(name);
+  }
+
+  async waitForIndex(
+    indexNames: string[],
+    timeoutSeconds: number,
+  ): Promise<void> {
+    await this.inner.waitForIndex(indexNames, timeoutSeconds);
+  }
+
   query(): Query {
     return new Query(this.inner);
   }
 
   search(
-    query: string | IntoVector,
+    query: string | IntoVector | FullTextQuery,
     queryType: string = "auto",
     ftsColumns?: string | string[],
   ): VectorQuery | Query {
-    if (typeof query !== "string") {
+    if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
       if (queryType === "fts") {
         throw new Error("Cannot perform full text search on a vector query");
       }
@@ -583,7 +688,10 @@ export class LocalTable extends Table {
 
     // The query type is auto or vector
     // fall back to full text search if no embedding functions are defined and the query is a string
-    if (queryType === "auto" && getRegistry().length() === 0) {
+    if (
+      queryType === "auto" &&
+      (getRegistry().length() === 0 || instanceOfFullTextQuery(query))
+    ) {
       return this.query().fullTextSearch(query, {
         columns: ftsColumns,
       });
@@ -613,24 +721,51 @@ export class LocalTable extends Table {
 
   // TODO: Support BatchUDF
 
-  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
+  async addColumns(
-    await this.inner.addColumns(newColumnTransforms);
+    newColumnTransforms: AddColumnsSql[],
+  ): Promise<AddColumnsResult> {
+    return await this.inner.addColumns(newColumnTransforms);
   }
 
-  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
+  async alterColumns(
-    await this.inner.alterColumns(columnAlterations);
+    columnAlterations: ColumnAlteration[],
+  ): Promise<AlterColumnsResult> {
+    const processedAlterations = columnAlterations.map((alteration) => {
+      if (typeof alteration.dataType === "string") {
+        return {
+          ...alteration,
+          dataType: JSON.stringify({ type: alteration.dataType }),
+        };
+      } else if (alteration.dataType === undefined) {
+        return {
+          ...alteration,
+          dataType: undefined,
+        };
+      } else {
+        const dataType = sanitizeType(alteration.dataType);
+        return {
+          ...alteration,
+          dataType: JSON.stringify(dataTypeToJson(dataType)),
+        };
+      }
+    });
+
+    return await this.inner.alterColumns(processedAlterations);
   }
 
-  async dropColumns(columnNames: string[]): Promise<void> {
+  async dropColumns(columnNames: string[]): Promise<DropColumnsResult> {
-    await this.inner.dropColumns(columnNames);
+    return await this.inner.dropColumns(columnNames);
   }
 
   async version(): Promise<number> {
     return await this.inner.version();
   }
 
-  async checkout(version: number): Promise<void> {
+  async checkout(version: number | string): Promise<void> {
-    await this.inner.checkout(version);
+    if (typeof version === "string") {
+      return this.inner.checkoutTag(version);
+    }
+    return this.inner.checkout(version);
   }
 
   async checkoutLatest(): Promise<void> {
@@ -649,6 +784,10 @@ export class LocalTable extends Table {
     await this.inner.restore();
   }
 
+  async tags(): Promise<Tags> {
+    return await this.inner.tags();
+  }
+
   async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
     let cleanupOlderThanMs;
     if (
@@ -679,6 +818,11 @@ export class LocalTable extends Table {
     }
     return stats;
   }
+
+  async stats(): Promise<TableStatistics> {
+    return await this.inner.stats();
+  }
+
   mergeInsert(on: string | string[]): MergeInsertBuilder {
     on = Array.isArray(on) ? on : [on];
     return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());
@@ -711,3 +855,38 @@ export class LocalTable extends Table {
     await this.inner.migrateManifestPathsV2();
   }
 }
+
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string;
+  /**
+   * The new name of the column. If not provided then the name will not be changed.
+   * This must be distinct from the names of all other columns in the table.
+   */
+  rename?: string;
+  /**
+   * A new data type for the column. If not provided then the data type will not be changed.
+   * Changing data types is limited to casting to the same general type. For example, these
+   * changes are valid:
+   * * `int32` -> `int64` (integers)
+   * * `double` -> `float` (floats)
+   * * `string` -> `large_string` (strings)
+   * But these changes are not:
+   * * `int32` -> `double` (mix integers and floats)
+   * * `string` -> `int32` (mix strings and integers)
+   */
+  dataType?: string | DataType;
+  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+  nullable?: boolean;
+}

nodejs/lancedb/util.ts

@@ -35,6 +35,16 @@ export function toSQL(value: IntoSql): string {
   }
 }
 
+export function packBits(data: Array<number>): Array<number> {
+  const packed = Array(data.length >> 3).fill(0);
+  for (let i = 0; i < data.length; i++) {
+    const byte = i >> 3;
+    const bit = i & 7;
+    packed[byte] |= data[i] << bit;
+  }
+  return packed;
+}
+
 export class TTLCache {
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   private readonly cache: Map<string, { value: any; expires: number }>;

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.18.0-beta.0",
+	"version": "0.19.1-beta.5",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",