Bump version: 0.17.1-beta.2 → 0.17.1-beta.3

User reported on Discord, when using
`table.vector_search([np.float16(1.0), np.float16(2.0), ...])`, it
yields `TypeError: 'numpy.float16' object is not iterable`

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.13.0"
+current_version = "0.14.1-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/docs.yml

@@ -72,9 +72,9 @@ jobs:
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.github/workflows/npm-publish.yml

@@ -133,7 +133,7 @@ jobs:
           free -h
       - name: Build Linux Artifacts
         run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -185,7 +185,7 @@ jobs:
       - name: Build Linux Artifacts
         run: |
           source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }}
+          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-musl
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -334,109 +334,50 @@ jobs:
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
-  # TODO: re-enable once working https://github.com/lancedb/lancedb/pull/1831
-  # node-windows-arm64:
-  #   name: vectordb win32-arm64-msvc
-  #   runs-on: windows-4x-arm
-  #   if: startsWith(github.ref, 'refs/tags/v')
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install Git
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-  #         Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-  #       shell: powershell
-  #     - name: Add Git to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-  #         $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-  #       shell: powershell
-  #     - name: Configure Git symlinks
-  #       run: git config --global core.symlinks true
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-python@v5
-  #       with:
-  #         python-version: "3.13"
-  #     - name: Install Visual Studio Build Tools
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-  #         Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-  #           "--installPath", "C:\BuildTools", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-  #           "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-  #       shell: powershell
-  #     - name: Add Visual Studio Build Tools to PATH
-  #       run: |
-  #         $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-  #         $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
-
-  #         # Add MSVC runtime libraries to LIB
-  #         $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
-  #                    "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
-  #                    "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-  #         Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
-
-  #         # Add INCLUDE paths
-  #         $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
-  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
-  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
-  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
-  #         Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
-  #       shell: powershell
-  #     - name: Install Rust
-  #       run: |
-  #         Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-  #         .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-  #       shell: powershell
-  #     - name: Add Rust to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-  #       shell: powershell
-
-  #     - uses: Swatinem/rust-cache@v2
-  #       with:
-  #         workspaces: rust
-  #     - name: Install 7-Zip ARM
-  #       run: |
-  #         New-Item -Path 'C:\7zip' -ItemType Directory
-  #         Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-  #         Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-  #       shell: powershell
-  #     - name: Add 7-Zip to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-  #       shell: powershell
-  #     - name: Install Protoc v21.12
-  #       working-directory: C:\
-  #       run: |
-  #         if (Test-Path 'C:\protoc') {
-  #             Write-Host "Protoc directory exists, skipping installation"
-  #             return
-  #         }
-  #         New-Item -Path 'C:\protoc' -ItemType Directory
-  #         Set-Location C:\protoc
-  #         Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-  #         & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-  #       shell: powershell
-  #     - name: Add Protoc to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-  #       shell: powershell
-  #     - name: Build Windows native node modules
-  #       run: .\ci\build_windows_artifacts.ps1 aarch64-pc-windows-msvc
-  #     - name: Upload Windows ARM64 Artifacts
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: node-native-windows-arm64
-  #         path: |
-  #           node/dist/*.node
+  node-windows-arm64:
+    name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          # - arch: x86_64
+          - arch: aarch64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          echo "source $HOME/.cargo/env" >> saved_env
+          echo "export CC=clang" >> saved_env
+          echo "export AR=llvm-ar" >> saved_env
+          source "$HOME/.cargo/env"
+          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc --toolchain 1.80.0
+          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
+          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
+          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
+      - name: Configure x86_64 build
+        if: ${{ matrix.config.arch == 'x86_64' }}
+        run: |
+          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
+      - name: Build Windows Artifacts
+        run: |
+          source ./saved_env
+          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-pc-windows-msvc
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-windows-${{ matrix.config.arch }}
+          path: |
+            node/dist/lancedb-vectordb-win32*.tgz
 
   nodejs-windows:
     name: lancedb ${{ matrix.target }}
@@ -472,103 +413,57 @@ jobs:
           path: |
             nodejs/dist/*.node
 
-  # TODO: re-enable once working https://github.com/lancedb/lancedb/pull/1831
-  # nodejs-windows-arm64:
-  #   name: lancedb win32-arm64-msvc
-  #   runs-on: windows-4x-arm
-  #   if: startsWith(github.ref, 'refs/tags/v')
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install Git
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-  #         Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-  #       shell: powershell
-  #     - name: Add Git to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-  #         $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-  #       shell: powershell
-  #     - name: Configure Git symlinks
-  #       run: git config --global core.symlinks true
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-python@v5
-  #       with:
-  #         python-version: "3.13"
-  #     - name: Install Visual Studio Build Tools
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-  #         Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-  #           "--installPath", "C:\BuildTools", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-  #           "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-  #       shell: powershell
-  #     - name: Add Visual Studio Build Tools to PATH
-  #       run: |
-  #         $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-  #         $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
-
-  #         $env:LIB = ""
-  #         Add-Content $env:GITHUB_ENV "LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-  #       shell: powershell
-  #     - name: Install Rust
-  #       run: |
-  #         Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-  #         .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-  #       shell: powershell
-  #     - name: Add Rust to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-  #       shell: powershell
-
-  #     - uses: Swatinem/rust-cache@v2
-  #       with:
-  #         workspaces: rust
-  #     - name: Install 7-Zip ARM
-  #       run: |
-  #         New-Item -Path 'C:\7zip' -ItemType Directory
-  #         Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-  #         Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-  #       shell: powershell
-  #     - name: Add 7-Zip to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-  #       shell: powershell
-  #     - name: Install Protoc v21.12
-  #       working-directory: C:\
-  #       run: |
-  #         if (Test-Path 'C:\protoc') {
-  #             Write-Host "Protoc directory exists, skipping installation"
-  #             return
-  #         }
-  #         New-Item -Path 'C:\protoc' -ItemType Directory
-  #         Set-Location C:\protoc
-  #         Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-  #         & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-  #       shell: powershell
-  #     - name: Add Protoc to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-  #       shell: powershell
-  #     - name: Build Windows native node modules
-  #       run: .\ci\build_windows_artifacts_nodejs.ps1 aarch64-pc-windows-msvc
-  #     - name: Upload Windows ARM64 Artifacts
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: nodejs-native-windows-arm64
-  #         path: |
-  #           nodejs/dist/*.node
+  nodejs-windows-arm64:
+    name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          # - arch: x86_64
+          - arch: aarch64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          echo "source $HOME/.cargo/env" >> saved_env
+          echo "export CC=clang" >> saved_env
+          echo "export AR=llvm-ar" >> saved_env
+          source "$HOME/.cargo/env"
+          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc --toolchain 1.80.0
+          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
+          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
+          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
+          printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin
+          chmod u+x $HOME/.cargo/bin/cargo-xwin
+      - name: Configure x86_64 build
+        if: ${{ matrix.config.arch == 'x86_64' }}
+        run: |
+          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
+      - name: Build Windows Artifacts
+        run: |
+          source ./saved_env
+          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
 
   release:
     name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows]
+    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows, node-windows-arm64]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -608,7 +503,7 @@ jobs:
 
   release-nodejs:
     name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows]
+    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows, nodejs-windows-arm64]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -666,6 +561,7 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   update-package-lock:
+    if: startsWith(github.ref, 'refs/tags/v')
     needs: [release]
     runs-on: ubuntu-latest
     permissions:
@@ -683,6 +579,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   update-package-lock-nodejs:
+    if: startsWith(github.ref, 'refs/tags/v')
     needs: [release-nodejs]
     runs-on: ubuntu-latest
     permissions:
@@ -700,6 +597,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/pypi-publish.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - uses: ./.github/workflows/build_windows_wheel
         with:
           python-minor-version: 8

.github/workflows/upload_wheel/action.yml

@@ -17,6 +17,7 @@ runs:
     run: |
       python -m pip install --upgrade pip
       pip install twine
+      python3 -m pip install --upgrade pkginfo
   - name: Choose repo
     shell: bash
     id: choose_repo

Cargo.toml

@@ -21,28 +21,29 @@ categories = ["database-implementations"]
 rust-version = "1.80.0"                                                     # TODO: lower this once we upgrade Lance again.
 
 [workspace.dependencies]
-lance = { "version" = "=0.20.0", "features" = [
+lance = { "version" = "=0.21.0", "features" = [
     "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
-lance-index = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
-lance-linalg = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
-lance-table = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
-lance-testing = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
-lance-datafusion = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
-lance-encoding = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+], git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-io = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-index = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-linalg = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-table = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-testing = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-datafusion = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
+lance-encoding = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.2" }
 # Note that this one does not include pyarrow
-arrow = { version = "52.2", optional = false }
-arrow-array = "52.2"
-arrow-data = "52.2"
-arrow-ipc = "52.2"
-arrow-ord = "52.2"
-arrow-schema = "52.2"
-arrow-arith = "52.2"
-arrow-cast = "52.2"
+arrow = { version = "53.2", optional = false }
+arrow-array = "53.2"
+arrow-data = "53.2"
+arrow-ipc = "53.2"
+arrow-ord = "53.2"
+arrow-schema = "53.2"
+arrow-arith = "53.2"
+arrow-cast = "53.2"
 async-trait = "0"
 chrono = "0.4.35"
-datafusion-common = "41.0"
-datafusion-physical-plan = "41.0"
+datafusion-common = "42.0"
+datafusion-physical-plan = "42.0"
 env_logger = "0.10"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",

ci/build_linux_artifacts.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 set -e
 ARCH=${1:-x86_64}
+TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
-# We pass down the current user so that when we later mount the local files 
+# We pass down the current user so that when we later mount the local files
 # into the container, the files are accessible by the current user.
 pushd ci/manylinux_node
 docker build \
@@ -18,4 +19,4 @@ docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
     lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH
+    bash ci/manylinux_node/build_vectordb.sh $ARCH $TARGET_TRIPLE

ci/manylinux_node/build_vectordb.sh

@@ -2,6 +2,7 @@
 # Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
 set -e
 ARCH=${1:-x86_64}
+TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
 if [ "$ARCH" = "x86_64" ]; then
     export OPENSSL_LIB_DIR=/usr/local/lib64/
@@ -17,4 +18,4 @@ FILE=$HOME/.bashrc && test -f $FILE && source $FILE
 cd node
 npm ci
 npm run build-release
-npm run pack-build
+npm run pack-build -- -t $TARGET_TRIPLE

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+
+# fwpuclnt.lib arm64rt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# ARM intrinsics
+# original dir: MSVC/
+
+# '__n128x4' redefined in arm_neon.h
+# "arm64_neon.h" included from intrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+# .lib
+
+# _Interlocked intrinsics
+# must always link with arm64rt.lib
+# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
+# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
+
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib

ci/sysroot-x86_64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
+
+# dbghelp.lib fwpuclnt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# x86 intrinsics
+# original dir: MSVC/
+
+# '_mm_movemask_epi8' defined in emmintrin.h
+# '__v4sf' defined in xmmintrin.h
+# '__v2si' defined in mmintrin.h
+# '__m128d' redefined in immintrin.h
+# '__m128i' redefined in intrin.h
+# '_mm_comlt_epu8' defined in ammintrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+# .lib
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib

docs/mkdocs.yml

@@ -55,6 +55,9 @@ plugins:
             show_signature_annotations: true
             show_root_heading: true
             members_order: source
+            docstring_section_style: list
+            signature_crossrefs: true
+            separate_signature: true
           import:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv

docs/src/ann_indexes.md

@@ -83,6 +83,7 @@ The following IVF_PQ paramters can be specified:
 - **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
   For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
   a single PQ code. The default is the dimension of the vector divided by 16.
+- **num_bits**: The number of bits used to encode each sub-vector. Only 4 and 8 are supported. The higher the number of bits, the higher the accuracy of the index, also the slower search. The default is 8.
 
 !!! note
 
@@ -142,11 +143,11 @@ There are a couple of parameters that can be used to fine-tune the search:
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
   Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
     - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
-  
+
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
   A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
     - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
-!!! note 
+!!! note
     Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
 
 
@@ -288,4 +289,4 @@ less space distortion, and thus yields better accuracy. However, a higher `num_s
 
 `m` determines the number of connections a new node establishes with its closest neighbors upon entering the graph. Typically, `m` falls within the range of 5 to 48. Lower `m` values are suitable for low-dimensional data or scenarios where recall is less critical. Conversely, higher `m` values are beneficial for high-dimensional data or when high recall is required. In essence, a larger `m` results in a denser graph with increased connectivity, but at the expense of higher memory consumption.
 
-`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase
+`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase

docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md

@@ -6,6 +6,7 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
 | `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
+| `use_azure` | bool | `False` | Set true to use Azure OpenAPI SDK |
 
 
 ```python

docs/src/guides/scalar_index.md

@@ -1,23 +1,35 @@
-# Building Scalar Index
+# Building a Scalar Index
 
-Similar to many SQL databases, LanceDB supports several types of Scalar indices to accelerate search
+Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
+
+Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
 over scalar columns.
 
-- `BTREE`: The most common type is BTREE. This index is inspired by the btree data structure
-  although only the first few layers of the btree are cached in memory.
-  It will perform well on columns with a large number of unique values and few rows per value.
-- `BITMAP`: this index stores a bitmap for each unique value in the column.
-  This index is useful for columns with a finite number of unique values and many rows per value.
-  For example, columns that represent "categories", "labels", or "tags"
-- `LABEL_LIST`: a special index that is used to index list columns whose values have a finite set of possibilities.
+- `BTREE`: The most common type is BTREE. The index stores a copy of the
+  column in sorted order. This sorted copy allows a binary search to be used to
+  satisfy queries.
+- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
+  uses a series of bits to indicate whether a value is present in a row of a table
+- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
+  support queries with `array_contains_all` and `array_contains_any`
+  using an underlying bitmap index.
   For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.
 
+!!! tips "How to choose the right scalar index type"
+
+    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
+    
+    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
+    
+    `LABEL_LIST`: This index should be used for columns containing list-type data.
+
 | Data Type                                                       | Filter                                    | Index Type   |
 | --------------------------------------------------------------- | ----------------------------------------- | ------------ |
 | Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
 | Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
 | List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |
 
+### Create a scalar index
 === "Python"
 
     ```python
@@ -46,7 +58,7 @@ over scalar columns.
         await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
         ```
 
-For example, the following scan will be faster if the column `my_col` has a scalar index:
+The following scan will be faster if the column `book_id` has a scalar index:
 
 === "Python"
 
@@ -106,3 +118,30 @@ Scalar indices can also speed up scans containing a vector search or full text s
           .limit(10)
           .toArray();
         ```
+### Update a scalar index
+Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
+=== "Python"
+
+    ```python
+    table.add([{"vector": [7, 8], "book_id": 4}])
+    table.optimize()
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
+    await tbl.optimize();
+    ```
+
+=== "Rust"
+
+    ```rust
+    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    tbl.add(more_data).execute().await?;
+    tbl.optimize(OptimizeAction::All).execute().await?;
+    ```
+
+!!! note
+
+    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.

docs/src/guides/storage.md

@@ -27,10 +27,13 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 
     Azure Blob Storage:
 
+    <!-- skip-test -->
     ```python
     import lancedb
     db = lancedb.connect("az://bucket/path")
     ```
+    Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
+
 
 === "TypeScript"
 
@@ -87,11 +90,6 @@ In most cases, when running in the respective cloud and permissions are set up c
 export TIMEOUT=60s
 ```
 
-!!! note "`storage_options` availability"
-
-    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
-    It is not yet supported in the Python synchronous API.
-
 If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
 
 === "Python"

docs/src/guides/tables.md

@@ -790,6 +790,101 @@ Use the `drop_table()` method on the database to remove a table.
       This permanently removes the table and is not recoverable, unlike deleting rows.
       If the table does not exist an exception is raised.
 
+## Changing schemas
+
+While tables must have a schema specified when they are created, you can
+change the schema over time. There's three methods to alter the schema of
+a table:
+
+* `add_columns`: Add new columns to the table
+* `alter_columns`: Alter the name, nullability, or data type of a column
+* `drop_columns`: Drop columns from the table
+
+### Adding new columns
+
+You can add new columns to the table with the `add_columns` method. New columns
+are filled with values based on a SQL expression. For example, you can add a new
+column `y` to the table and fill it with the value of `x + 1`.
+
+=== "Python"
+
+    ```python
+    table.add_columns({"double_price": "price * 2"})
+    ```
+    **API Reference:** [lancedb.table.Table.add_columns][]
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/basic.test.ts:add_columns"
+    ```
+    **API Reference:** [lancedb.Table.addColumns](../js/classes/Table.md/#addcolumns)
+
+If you want to fill it with null, you can use `cast(NULL as <data_type>)` as
+the SQL expression to fill the column with nulls, while controlling the data
+type of the column. Available data types are base on the
+[DataFusion data types](https://datafusion.apache.org/user-guide/sql/data_types.html).
+You can use any of the SQL types, such as `BIGINT`:
+
+```sql
+cast(NULL as BIGINT)
+```
+
+Using Arrow data types and the `arrow_typeof` function is not yet supported.
+
+<!-- TODO: we could provide a better formula for filling with nulls:
+   https://github.com/lancedb/lance/issues/3175
+-->
+
+### Altering existing columns
+
+You can alter the name, nullability, or data type of a column with the `alter_columns`
+method.
+
+Changing the name or nullability of a column just updates the metadata. Because
+of this, it's a fast operation. Changing the data type of a column requires
+rewriting the column, which can be a heavy operation.
+
+=== "Python"
+
+    ```python
+    import pyarrow as pa
+    table.alter_column({"path": "double_price", "rename": "dbl_price",
+                        "data_type": pa.float32(), "nullable": False})
+    ```
+    **API Reference:** [lancedb.table.Table.alter_columns][]
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/basic.test.ts:alter_columns"
+    ```
+    **API Reference:** [lancedb.Table.alterColumns](../js/classes/Table.md/#altercolumns)
+
+### Dropping columns
+
+You can drop columns from the table with the `drop_columns` method. This will
+will remove the column from the schema.
+
+<!-- TODO: Provide guidance on how to reduce disk usage once optimize helps here
+    waiting on: https://github.com/lancedb/lance/issues/3177
+-->
+
+=== "Python"
+
+    ```python
+    table.drop_columns(["dbl_price"])
+    ```
+    **API Reference:** [lancedb.table.Table.drop_columns][]
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/basic.test.ts:drop_columns"
+    ```
+    **API Reference:** [lancedb.Table.dropColumns](../js/classes/Table.md/#altercolumns)
+
+
 ## Handling bad vectors
 
 In LanceDB Python, you can use the `on_bad_vectors` parameter to choose how

docs/src/js/.nojekyll

@@ -1 +0,0 @@
-TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/js/classes/Connection.md

@@ -27,7 +27,9 @@ the underlying connection has been closed.
 
 ### new Connection()
 
-> **new Connection**(): [`Connection`](Connection.md)
+```ts
+new Connection(): Connection
+```
 
 #### Returns
 
@@ -37,7 +39,9 @@ the underlying connection has been closed.
 
 ### close()
 
-> `abstract` **close**(): `void`
+```ts
+abstract close(): void
+```
 
 Close the connection, releasing any underlying resources.
 
@@ -53,21 +57,24 @@ Any attempt to use the connection after it is closed will result in an error.
 
 ### createEmptyTable()
 
-> `abstract` **createEmptyTable**(`name`, `schema`, `options`?): `Promise`<[`Table`](Table.md)>
+```ts
+abstract createEmptyTable(
+   name,
+   schema,
+   options?): Promise<Table>
+```
 
 Creates a new empty Table
 
 #### Parameters
 
-• **name**: `string`
+* **name**: `string`
+    The name of the table.
 
-The name of the table.
+* **schema**: `SchemaLike`
+    The schema of the table
 
-• **schema**: `SchemaLike`
-
-The schema of the table
-
-• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
 
 #### Returns
 
@@ -79,15 +86,16 @@ The schema of the table
 
 #### createTable(options)
 
-> `abstract` **createTable**(`options`): `Promise`&lt;[`Table`](Table.md)&gt;
+```ts
+abstract createTable(options): Promise<Table>
+```
 
 Creates a new Table and initialize it with new data.
 
 ##### Parameters
 
-• **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
-
-The options object.
+* **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+    The options object.
 
 ##### Returns
 
@@ -95,22 +103,25 @@ The options object.
 
 #### createTable(name, data, options)
 
-> `abstract` **createTable**(`name`, `data`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;
+```ts
+abstract createTable(
+   name,
+   data,
+   options?): Promise<Table>
+```
 
 Creates a new Table and initialize it with new data.
 
 ##### Parameters
 
-• **name**: `string`
+* **name**: `string`
+    The name of the table.
 
-The name of the table.
+* **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
+    Non-empty Array of Records
+    to be inserted into the table
 
-• **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
-
-Non-empty Array of Records
-to be inserted into the table
-
-• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
 
 ##### Returns
 
@@ -120,7 +131,9 @@ to be inserted into the table
 
 ### display()
 
-> `abstract` **display**(): `string`
+```ts
+abstract display(): string
+```
 
 Return a brief description of the connection
 
@@ -132,15 +145,16 @@ Return a brief description of the connection
 
 ### dropTable()
 
-> `abstract` **dropTable**(`name`): `Promise`&lt;`void`&gt;
+```ts
+abstract dropTable(name): Promise<void>
+```
 
 Drop an existing table.
 
 #### Parameters
 
-• **name**: `string`
-
-The name of the table to drop.
+* **name**: `string`
+    The name of the table to drop.
 
 #### Returns
 
@@ -150,7 +164,9 @@ The name of the table to drop.
 
 ### isOpen()
 
-> `abstract` **isOpen**(): `boolean`
+```ts
+abstract isOpen(): boolean
+```
 
 Return true if the connection has not been closed
 
@@ -162,17 +178,18 @@ Return true if the connection has not been closed
 
 ### openTable()
 
-> `abstract` **openTable**(`name`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;
+```ts
+abstract openTable(name, options?): Promise<Table>
+```
 
 Open a table in the database.
 
 #### Parameters
 
-• **name**: `string`
+* **name**: `string`
+    The name of the table
 
-The name of the table
-
-• **options?**: `Partial`&lt;`OpenTableOptions`&gt;
+* **options?**: `Partial`&lt;`OpenTableOptions`&gt;
 
 #### Returns
 
@@ -182,7 +199,9 @@ The name of the table
 
 ### tableNames()
 
-> `abstract` **tableNames**(`options`?): `Promise`&lt;`string`[]&gt;
+```ts
+abstract tableNames(options?): Promise<string[]>
+```
 
 List all the table names in this database.
 
@@ -190,10 +209,9 @@ Tables will be returned in lexicographical order.
 
 #### Parameters
 
-• **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
-
-options to control the
-paging / start point
+* **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
+    options to control the
+    paging / start point
 
 #### Returns
 

docs/src/js/classes/Index.md

@@ -8,9 +8,30 @@
 
 ## Methods
 
+### bitmap()
+
+```ts
+static bitmap(): Index
+```
+
+Create a bitmap index.
+
+A `Bitmap` index stores a bitmap for each distinct value in the column for every row.
+
+This index works best for low-cardinality columns, where the number of unique values
+is small (i.e., less than a few hundreds).
+
+#### Returns
+
+[`Index`](Index.md)
+
+***
+
 ### btree()
 
-> `static` **btree**(): [`Index`](Index.md)
+```ts
+static btree(): Index
+```
 
 Create a btree index
 
@@ -36,9 +57,82 @@ block size may be added in the future.
 
 ***
 
+### fts()
+
+```ts
+static fts(options?): Index
+```
+
+Create a full text search index
+
+A full text search index is an index on a string column, so that you can conduct full
+text searches on the column.
+
+The results of a full text search are ordered by relevance measured by BM25.
+
+You can combine filters with full text search.
+
+For now, the full text search index only supports English, and doesn't support phrase search.
+
+#### Parameters
+
+* **options?**: `Partial`<`FtsOptions`>
+
+#### Returns
+
+[`Index`](Index.md)
+
+***
+
+### hnswPq()
+
+```ts
+static hnswPq(options?): Index
+```
+
+Create a hnswPq index
+
+HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+It is a variant of the HNSW algorithm that uses product quantization to compress
+the vectors.
+
+#### Parameters
+
+* **options?**: `Partial`<`HnswPqOptions`>
+
+#### Returns
+
+[`Index`](Index.md)
+
+***
+
+### hnswSq()
+
+```ts
+static hnswSq(options?): Index
+```
+
+Create a hnswSq index
+
+HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+It is a variant of the HNSW algorithm that uses scalar quantization to compress
+the vectors.
+
+#### Parameters
+
+* **options?**: `Partial`<`HnswSqOptions`>
+
+#### Returns
+
+[`Index`](Index.md)
+
+***
+
 ### ivfPq()
 
-> `static` **ivfPq**(`options`?): [`Index`](Index.md)
+```ts
+static ivfPq(options?): Index
+```
 
 Create an IvfPq index
 
@@ -63,29 +157,25 @@ currently is also a memory intensive operation.
 
 #### Parameters
 
-• **options?**: `Partial`<[`IvfPqOptions`](../interfaces/IvfPqOptions.md)>
+* **options?**: `Partial`<[`IvfPqOptions`](../interfaces/IvfPqOptions.md)>
 
 #### Returns
 
 [`Index`](Index.md)
 
-### fts()
+***
 
-> `static` **fts**(`options`?): [`Index`](Index.md)
+### labelList()
 
-Create a full text search index
+```ts
+static labelList(): Index
+```
 
-This index is used to search for text data.  The index is created by tokenizing the text
-into words and then storing occurrences of these words in a data structure called inverted index
-that allows for fast search.
+Create a label list index.
 
-During a search the query is tokenized and the inverted index is used to find the rows that
-contain the query words.  The rows are then scored based on BM25 and the top scoring rows are
-sorted and returned.
-
-#### Parameters
-
-• **options?**: `Partial`<[`FtsOptions`](../interfaces/FtsOptions.md)>
+LabelList index is a scalar index that can be used on `List<T>` columns to
+support queries with `array_contains_all` and `array_contains_any`
+using an underlying bitmap index.
 
 #### Returns
 

docs/src/js/classes/MakeArrowTableOptions.md

@@ -12,11 +12,13 @@ Options to control the makeArrowTable call.
 
 ### new MakeArrowTableOptions()
 
-> **new MakeArrowTableOptions**(`values`?): [`MakeArrowTableOptions`](MakeArrowTableOptions.md)
+```ts
+new MakeArrowTableOptions(values?): MakeArrowTableOptions
+```
 
 #### Parameters
 
-• **values?**: `Partial`<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)>
+* **values?**: `Partial`<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)>
 
 #### Returns
 
@@ -26,7 +28,9 @@ Options to control the makeArrowTable call.
 
 ### dictionaryEncodeStrings
 
-> **dictionaryEncodeStrings**: `boolean` = `false`
+```ts
+dictionaryEncodeStrings: boolean = false;
+```
 
 If true then string columns will be encoded with dictionary encoding
 
@@ -40,22 +44,30 @@ If `schema` is provided then this property is ignored.
 
 ### embeddingFunction?
 
-> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)
+```ts
+optional embeddingFunction: EmbeddingFunctionConfig;
+```
 
 ***
 
 ### embeddings?
 
-> `optional` **embeddings**: [`EmbeddingFunction`](../namespaces/embedding/classes/EmbeddingFunction.md)<`unknown`, `FunctionOptions`>
+```ts
+optional embeddings: EmbeddingFunction<unknown, FunctionOptions>;
+```
 
 ***
 
 ### schema?
 
-> `optional` **schema**: `SchemaLike`
+```ts
+optional schema: SchemaLike;
+```
 
 ***
 
 ### vectorColumns
 
-> **vectorColumns**: `Record`&lt;`string`, [`VectorColumnOptions`](VectorColumnOptions.md)&gt;
+```ts
+vectorColumns: Record<string, VectorColumnOptions>;
+```

docs/src/js/classes/Query.md

@@ -16,11 +16,13 @@ A builder for LanceDB queries.
 
 ### new Query()
 
-> **new Query**(`tbl`): [`Query`](Query.md)
+```ts
+new Query(tbl): Query
+```
 
 #### Parameters
 
-• **tbl**: `Table`
+* **tbl**: `Table`
 
 #### Returns
 
@@ -34,7 +36,9 @@ A builder for LanceDB queries.
 
 ### inner
 
-> `protected` **inner**: `Query` \| `Promise`<`Query`>
+```ts
+protected inner: Query | Promise<Query>;
+```
 
 #### Inherited from
 
@@ -44,7 +48,9 @@ A builder for LanceDB queries.
 
 ### \[asyncIterator\]()
 
-> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+```ts
+asyncIterator: AsyncIterator<RecordBatch<any>, any, undefined>
+```
 
 #### Returns
 
@@ -58,11 +64,13 @@ A builder for LanceDB queries.
 
 ### doCall()
 
-> `protected` **doCall**(`fn`): `void`
+```ts
+protected doCall(fn): void
+```
 
 #### Parameters
 
-• **fn**
+* **fn**
 
 #### Returns
 
@@ -76,13 +84,15 @@ A builder for LanceDB queries.
 
 ### execute()
 
-> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+```ts
+protected execute(options?): RecordBatchIterator
+```
 
 Execute the query and return the results as an
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -108,15 +118,16 @@ single query)
 
 ### explainPlan()
 
-> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;
+```ts
+explainPlan(verbose): Promise<string>
+```
 
 Generates an explanation of the query execution plan.
 
 #### Parameters
 
-• **verbose**: `boolean` = `false`
-
-If true, provides a more detailed explanation. Defaults to false.
+* **verbose**: `boolean` = `false`
+    If true, provides a more detailed explanation. Defaults to false.
 
 #### Returns
 
@@ -141,15 +152,38 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 
 ***
 
+### fastSearch()
+
+```ts
+fastSearch(): this
+```
+
+Skip searching un-indexed data. This can make search faster, but will miss
+any data that is not yet indexed.
+
+Use lancedb.Table#optimize to index all un-indexed data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
+
+***
+
 ### ~~filter()~~
 
-> **filter**(`predicate`): `this`
+```ts
+filter(predicate): this
+```
 
 A filter statement to be applied to this query.
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -169,9 +203,33 @@ Use `where` instead
 
 ***
 
+### fullTextSearch()
+
+```ts
+fullTextSearch(query, options?): this
+```
+
+#### Parameters
+
+* **query**: `string`
+
+* **options?**: `Partial`&lt;`FullTextSearchOptions`&gt;
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
+
+***
+
 ### limit()
 
-> **limit**(`limit`): `this`
+```ts
+limit(limit): this
+```
 
 Set the maximum number of results to return.
 
@@ -180,7 +238,7 @@ called then every valid row from the table will be returned.
 
 #### Parameters
 
-• **limit**: `number`
+* **limit**: `number`
 
 #### Returns
 
@@ -194,11 +252,13 @@ called then every valid row from the table will be returned.
 
 ### nativeExecute()
 
-> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
+```ts
+protected nativeExecute(options?): Promise<RecordBatchIterator>
+```
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -212,7 +272,9 @@ called then every valid row from the table will be returned.
 
 ### nearestTo()
 
-> **nearestTo**(`vector`): [`VectorQuery`](VectorQuery.md)
+```ts
+nearestTo(vector): VectorQuery
+```
 
 Find the nearest vectors to the given query vector.
 
@@ -232,7 +294,7 @@ If there is more than one vector column you must use
 
 #### Parameters
 
-• **vector**: `IntoVector`
+* **vector**: `IntoVector`
 
 #### Returns
 
@@ -264,9 +326,49 @@ a default `limit` of 10 will be used.
 
 ***
 
+### nearestToText()
+
+```ts
+nearestToText(query, columns?): Query
+```
+
+#### Parameters
+
+* **query**: `string`
+
+* **columns?**: `string`[]
+
+#### Returns
+
+[`Query`](Query.md)
+
+***
+
+### offset()
+
+```ts
+offset(offset): this
+```
+
+#### Parameters
+
+* **offset**: `number`
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
+
+***
+
 ### select()
 
-> **select**(`columns`): `this`
+```ts
+select(columns): this
+```
 
 Return only the specified columns.
 
@@ -290,7 +392,7 @@ input to this method would be:
 
 #### Parameters
 
-• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
 
 #### Returns
 
@@ -317,13 +419,15 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 ### toArray()
 
-> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
+```ts
+toArray(options?): Promise<any[]>
+```
 
 Collect the results as an array of objects.
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -337,13 +441,15 @@ Collect the results as an array of objects.
 
 ### toArrow()
 
-> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+```ts
+toArrow(options?): Promise<Table<any>>
+```
 
 Collect the results as an Arrow
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -361,7 +467,9 @@ ArrowTable.
 
 ### where()
 
-> **where**(`predicate`): `this`
+```ts
+where(predicate): this
+```
 
 A filter statement to be applied to this query.
 
@@ -369,7 +477,7 @@ The filter should be supplied as an SQL query string.  For example:
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -389,3 +497,25 @@ on the filter column(s).
 #### Inherited from
 
 [`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
+
+***
+
+### withRowId()
+
+```ts
+withRowId(): this
+```
+
+Whether to return the row id in the results.
+
+This column can be used to match results between different queries. For
+example, to match results from a full text search and a vector search in
+order to perform hybrid search.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)

docs/src/js/classes/QueryBase.md

@@ -25,11 +25,13 @@ Common methods supported by all query types
 
 ### new QueryBase()
 
-> `protected` **new QueryBase**<`NativeQueryType`>(`inner`): [`QueryBase`](QueryBase.md)<`NativeQueryType`>
+```ts
+protected new QueryBase<NativeQueryType>(inner): QueryBase<NativeQueryType>
+```
 
 #### Parameters
 
-• **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;
+* **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;
 
 #### Returns
 
@@ -39,13 +41,17 @@ Common methods supported by all query types
 
 ### inner
 
-> `protected` **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;
+```ts
+protected inner: NativeQueryType | Promise<NativeQueryType>;
+```
 
 ## Methods
 
 ### \[asyncIterator\]()
 
-> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+```ts
+asyncIterator: AsyncIterator<RecordBatch<any>, any, undefined>
+```
 
 #### Returns
 
@@ -59,11 +65,13 @@ Common methods supported by all query types
 
 ### doCall()
 
-> `protected` **doCall**(`fn`): `void`
+```ts
+protected doCall(fn): void
+```
 
 #### Parameters
 
-• **fn**
+* **fn**
 
 #### Returns
 
@@ -73,13 +81,15 @@ Common methods supported by all query types
 
 ### execute()
 
-> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+```ts
+protected execute(options?): RecordBatchIterator
+```
 
 Execute the query and return the results as an
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -101,15 +111,16 @@ single query)
 
 ### explainPlan()
 
-> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;
+```ts
+explainPlan(verbose): Promise<string>
+```
 
 Generates an explanation of the query execution plan.
 
 #### Parameters
 
-• **verbose**: `boolean` = `false`
-
-If true, provides a more detailed explanation. Defaults to false.
+* **verbose**: `boolean` = `false`
+    If true, provides a more detailed explanation. Defaults to false.
 
 #### Returns
 
@@ -130,15 +141,34 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 
 ***
 
+### fastSearch()
+
+```ts
+fastSearch(): this
+```
+
+Skip searching un-indexed data. This can make search faster, but will miss
+any data that is not yet indexed.
+
+Use lancedb.Table#optimize to index all un-indexed data.
+
+#### Returns
+
+`this`
+
+***
+
 ### ~~filter()~~
 
-> **filter**(`predicate`): `this`
+```ts
+filter(predicate): this
+```
 
 A filter statement to be applied to this query.
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -154,9 +184,29 @@ Use `where` instead
 
 ***
 
+### fullTextSearch()
+
+```ts
+fullTextSearch(query, options?): this
+```
+
+#### Parameters
+
+* **query**: `string`
+
+* **options?**: `Partial`&lt;`FullTextSearchOptions`&gt;
+
+#### Returns
+
+`this`
+
+***
+
 ### limit()
 
-> **limit**(`limit`): `this`
+```ts
+limit(limit): this
+```
 
 Set the maximum number of results to return.
 
@@ -165,7 +215,7 @@ called then every valid row from the table will be returned.
 
 #### Parameters
 
-• **limit**: `number`
+* **limit**: `number`
 
 #### Returns
 
@@ -175,11 +225,13 @@ called then every valid row from the table will be returned.
 
 ### nativeExecute()
 
-> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
+```ts
+protected nativeExecute(options?): Promise<RecordBatchIterator>
+```
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -187,9 +239,27 @@ called then every valid row from the table will be returned.
 
 ***
 
+### offset()
+
+```ts
+offset(offset): this
+```
+
+#### Parameters
+
+* **offset**: `number`
+
+#### Returns
+
+`this`
+
+***
+
 ### select()
 
-> **select**(`columns`): `this`
+```ts
+select(columns): this
+```
 
 Return only the specified columns.
 
@@ -213,7 +283,7 @@ input to this method would be:
 
 #### Parameters
 
-• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
 
 #### Returns
 
@@ -236,13 +306,15 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 ### toArray()
 
-> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
+```ts
+toArray(options?): Promise<any[]>
+```
 
 Collect the results as an array of objects.
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -252,13 +324,15 @@ Collect the results as an array of objects.
 
 ### toArrow()
 
-> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+```ts
+toArrow(options?): Promise<Table<any>>
+```
 
 Collect the results as an Arrow
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -272,7 +346,9 @@ ArrowTable.
 
 ### where()
 
-> **where**(`predicate`): `this`
+```ts
+where(predicate): this
+```
 
 A filter statement to be applied to this query.
 
@@ -280,7 +356,7 @@ The filter should be supplied as an SQL query string.  For example:
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -296,3 +372,21 @@ x > 5 OR y = 'test'
 Filtering performance can often be improved by creating a scalar index
 on the filter column(s).
 ```
+
+***
+
+### withRowId()
+
+```ts
+withRowId(): this
+```
+
+Whether to return the row id in the results.
+
+This column can be used to match results between different queries. For
+example, to match results from a full text search and a vector search in
+order to perform hybrid search.
+
+#### Returns
+
+`this`

docs/src/js/classes/RecordBatchIterator.md

@@ -14,11 +14,13 @@
 
 ### new RecordBatchIterator()
 
-> **new RecordBatchIterator**(`promise`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+```ts
+new RecordBatchIterator(promise?): RecordBatchIterator
+```
 
 #### Parameters
 
-• **promise?**: `Promise`<`RecordBatchIterator`>
+* **promise?**: `Promise`<`RecordBatchIterator`>
 
 #### Returns
 
@@ -28,7 +30,9 @@
 
 ### next()
 
-> **next**(): `Promise`<`IteratorResult`<`RecordBatch`<`any`>, `any`>>
+```ts
+next(): Promise<IteratorResult<RecordBatch<any>, any>>
+```
 
 #### Returns
 

docs/src/js/classes/Table.md

@@ -21,7 +21,9 @@ collected.
 
 ### new Table()
 
-> **new Table**(): [`Table`](Table.md)
+```ts
+new Table(): Table
+```
 
 #### Returns
 
@@ -31,7 +33,9 @@ collected.
 
 ### name
 
-> `get` `abstract` **name**(): `string`
+```ts
+get abstract name(): string
+```
 
 Returns the name of the table
 
@@ -43,17 +47,18 @@ Returns the name of the table
 
 ### add()
 
-> `abstract` **add**(`data`, `options`?): `Promise`<`void`>
+```ts
+abstract add(data, options?): Promise<void>
+```
 
 Insert records into this Table.
 
 #### Parameters
 
-• **data**: [`Data`](../type-aliases/Data.md)
+* **data**: [`Data`](../type-aliases/Data.md)
+    Records to be inserted into the Table
 
-Records to be inserted into the Table
-
-• **options?**: `Partial`&lt;[`AddDataOptions`](../interfaces/AddDataOptions.md)&gt;
+* **options?**: `Partial`&lt;[`AddDataOptions`](../interfaces/AddDataOptions.md)&gt;
 
 #### Returns
 
@@ -63,18 +68,19 @@ Records to be inserted into the Table
 
 ### addColumns()
 
-> `abstract` **addColumns**(`newColumnTransforms`): `Promise`&lt;`void`&gt;
+```ts
+abstract addColumns(newColumnTransforms): Promise<void>
+```
 
 Add new columns with defined values.
 
 #### Parameters
 
-• **newColumnTransforms**: [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
-
-pairs of column names and
-the SQL expression to use to calculate the value of the new column. These
-expressions will be evaluated for each row in the table, and can
-reference existing columns in the table.
+* **newColumnTransforms**: [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
+    pairs of column names and
+    the SQL expression to use to calculate the value of the new column. These
+    expressions will be evaluated for each row in the table, and can
+    reference existing columns in the table.
 
 #### Returns
 
@@ -84,16 +90,17 @@ reference existing columns in the table.
 
 ### alterColumns()
 
-> `abstract` **alterColumns**(`columnAlterations`): `Promise`&lt;`void`&gt;
+```ts
+abstract alterColumns(columnAlterations): Promise<void>
+```
 
 Alter the name or nullability of columns.
 
 #### Parameters
 
-• **columnAlterations**: [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[]
-
-One or more alterations to
-apply to columns.
+* **columnAlterations**: [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[]
+    One or more alterations to
+    apply to columns.
 
 #### Returns
 
@@ -103,7 +110,9 @@ apply to columns.
 
 ### checkout()
 
-> `abstract` **checkout**(`version`): `Promise`&lt;`void`&gt;
+```ts
+abstract checkout(version): Promise<void>
+```
 
 Checks out a specific version of the table _This is an in-place operation._
 
@@ -116,9 +125,8 @@ wish to return to standard mode, call `checkoutLatest`.
 
 #### Parameters
 
-• **version**: `number`
-
-The version to checkout
+* **version**: `number`
+    The version to checkout
 
 #### Returns
 
@@ -144,7 +152,9 @@ console.log(await table.version()); // 2
 
 ### checkoutLatest()
 
-> `abstract` **checkoutLatest**(): `Promise`&lt;`void`&gt;
+```ts
+abstract checkoutLatest(): Promise<void>
+```
 
 Checkout the latest version of the table. _This is an in-place operation._
 
@@ -159,7 +169,9 @@ version of the table.
 
 ### close()
 
-> `abstract` **close**(): `void`
+```ts
+abstract close(): void
+```
 
 Close the table, releasing any underlying resources.
 
@@ -175,13 +187,15 @@ Any attempt to use the table after it is closed will result in an error.
 
 ### countRows()
 
-> `abstract` **countRows**(`filter`?): `Promise`&lt;`number`&gt;
+```ts
+abstract countRows(filter?): Promise<number>
+```
 
 Count the total number of rows in the dataset.
 
 #### Parameters
 
-• **filter?**: `string`
+* **filter?**: `string`
 
 #### Returns
 
@@ -191,7 +205,9 @@ Count the total number of rows in the dataset.
 
 ### createIndex()
 
-> `abstract` **createIndex**(`column`, `options`?): `Promise`&lt;`void`&gt;
+```ts
+abstract createIndex(column, options?): Promise<void>
+```
 
 Create an index to speed up queries.
 
@@ -202,9 +218,9 @@ vector and non-vector searches)
 
 #### Parameters
 
-• **column**: `string`
+* **column**: `string`
 
-• **options?**: `Partial`&lt;[`IndexOptions`](../interfaces/IndexOptions.md)&gt;
+* **options?**: `Partial`&lt;[`IndexOptions`](../interfaces/IndexOptions.md)&gt;
 
 #### Returns
 
@@ -245,13 +261,15 @@ await table.createIndex("my_float_col");
 
 ### delete()
 
-> `abstract` **delete**(`predicate`): `Promise`&lt;`void`&gt;
+```ts
+abstract delete(predicate): Promise<void>
+```
 
 Delete the rows that satisfy the predicate.
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -261,7 +279,9 @@ Delete the rows that satisfy the predicate.
 
 ### display()
 
-> `abstract` **display**(): `string`
+```ts
+abstract display(): string
+```
 
 Return a brief description of the table
 
@@ -273,7 +293,9 @@ Return a brief description of the table
 
 ### dropColumns()
 
-> `abstract` **dropColumns**(`columnNames`): `Promise`&lt;`void`&gt;
+```ts
+abstract dropColumns(columnNames): Promise<void>
+```
 
 Drop one or more columns from the dataset
 
@@ -284,11 +306,10 @@ then call ``cleanup_files`` to remove the old files.
 
 #### Parameters
 
-• **columnNames**: `string`[]
-
-The names of the columns to drop. These can
-be nested column references (e.g. "a.b.c") or top-level column names
-(e.g. "a").
+* **columnNames**: `string`[]
+    The names of the columns to drop. These can
+    be nested column references (e.g. "a.b.c") or top-level column names
+    (e.g. "a").
 
 #### Returns
 
@@ -298,15 +319,16 @@ be nested column references (e.g. "a.b.c") or top-level column names
 
 ### indexStats()
 
-> `abstract` **indexStats**(`name`): `Promise`&lt;`undefined` \| [`IndexStatistics`](../interfaces/IndexStatistics.md)&gt;
+```ts
+abstract indexStats(name): Promise<undefined | IndexStatistics>
+```
 
 List all the stats of a specified index
 
 #### Parameters
 
-• **name**: `string`
-
-The name of the index.
+* **name**: `string`
+    The name of the index.
 
 #### Returns
 
@@ -318,7 +340,9 @@ The stats of the index. If the index does not exist, it will return undefined
 
 ### isOpen()
 
-> `abstract` **isOpen**(): `boolean`
+```ts
+abstract isOpen(): boolean
+```
 
 Return true if the table has not been closed
 
@@ -330,7 +354,9 @@ Return true if the table has not been closed
 
 ### listIndices()
 
-> `abstract` **listIndices**(): `Promise`&lt;[`IndexConfig`](../interfaces/IndexConfig.md)[]&gt;
+```ts
+abstract listIndices(): Promise<IndexConfig[]>
+```
 
 List all indices that have been created with [Table.createIndex](Table.md#createindex)
 
@@ -340,13 +366,29 @@ List all indices that have been created with [Table.createIndex](Table.md#create
 
 ***
 
+### listVersions()
+
+```ts
+abstract listVersions(): Promise<Version[]>
+```
+
+List all the versions of the table
+
+#### Returns
+
+`Promise`&lt;`Version`[]&gt;
+
+***
+
 ### mergeInsert()
 
-> `abstract` **mergeInsert**(`on`): `MergeInsertBuilder`
+```ts
+abstract mergeInsert(on): MergeInsertBuilder
+```
 
 #### Parameters
 
-• **on**: `string` \| `string`[]
+* **on**: `string` \| `string`[]
 
 #### Returns
 
@@ -356,7 +398,9 @@ List all indices that have been created with [Table.createIndex](Table.md#create
 
 ### optimize()
 
-> `abstract` **optimize**(`options`?): `Promise`&lt;`OptimizeStats`&gt;
+```ts
+abstract optimize(options?): Promise<OptimizeStats>
+```
 
 Optimize the on-disk data and indices for better performance.
 
@@ -388,7 +432,7 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`OptimizeOptions`&gt;
+* **options?**: `Partial`&lt;[`OptimizeOptions`](../interfaces/OptimizeOptions.md)&gt;
 
 #### Returns
 
@@ -398,7 +442,9 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 ### query()
 
-> `abstract` **query**(): [`Query`](Query.md)
+```ts
+abstract query(): Query
+```
 
 Create a [Query](Query.md) Builder.
 
@@ -466,7 +512,9 @@ for await (const batch of table.query()) {
 
 ### restore()
 
-> `abstract` **restore**(): `Promise`&lt;`void`&gt;
+```ts
+abstract restore(): Promise<void>
+```
 
 Restore the table to the currently checked out version
 
@@ -487,7 +535,9 @@ out state and the read_consistency_interval, if any, will apply.
 
 ### schema()
 
-> `abstract` **schema**(): `Promise`&lt;`Schema`&lt;`any`&gt;&gt;
+```ts
+abstract schema(): Promise<Schema<any>>
+```
 
 Get the schema of the table.
 
@@ -499,61 +549,41 @@ Get the schema of the table.
 
 ### search()
 
-#### search(query)
-
-> `abstract` **search**(`query`, `queryType`, `ftsColumns`): [`VectorQuery`](VectorQuery.md)
+```ts
+abstract search(
+   query,
+   queryType?,
+   ftsColumns?): VectorQuery | Query
+```
 
 Create a search query to find the nearest neighbors
-of the given query vector, or the documents
-with the highest relevance to the query string.
+of the given query
 
-##### Parameters
+#### Parameters
 
-• **query**: `string`
+* **query**: `string` \| `IntoVector`
+    the query, a vector or string
 
-the query. This will be converted to a vector using the table's provided embedding function,
-or the query string for full-text search if `queryType` is "fts".
+* **queryType?**: `string`
+    the type of the query, "vector", "fts", or "auto"
 
-• **queryType**: `string` =  `"auto"` \| `"fts"`
+* **ftsColumns?**: `string` \| `string`[]
+    the columns to search in for full text search
+    for now, only one column can be searched at a time.
+    when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
+    if the query is a string and no embedding function is defined, it will be treated as a full text search query
 
-the type of query to run. If "auto", the query type will be determined based on the query.
+#### Returns
 
-• **ftsColumns**: `string[] | str` =  undefined
-
-the columns to search in. If not provided, all indexed columns will be searched.
-
-For now, this can support to search only one column.
-
-##### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-##### Note
-
-If no embedding functions are defined in the table, this will error when collecting the results.
-
-#### search(query)
-
-> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
-
-Create a search query to find the nearest neighbors
-of the given query vector
-
-##### Parameters
-
-• **query**: `IntoVector`
-
-the query vector
-
-##### Returns
-
-[`VectorQuery`](VectorQuery.md)
+[`VectorQuery`](VectorQuery.md) \| [`Query`](Query.md)
 
 ***
 
 ### toArrow()
 
-> `abstract` **toArrow**(): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+```ts
+abstract toArrow(): Promise<Table<any>>
+```
 
 Return the table as an arrow table
 
@@ -567,13 +597,15 @@ Return the table as an arrow table
 
 #### update(opts)
 
-> `abstract` **update**(`opts`): `Promise`&lt;`void`&gt;
+```ts
+abstract update(opts): Promise<void>
+```
 
 Update existing records in the Table
 
 ##### Parameters
 
-• **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+* **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
 
 ##### Returns
 
@@ -587,13 +619,15 @@ table.update({where:"x = 2", values:{"vector": [10, 10]}})
 
 #### update(opts)
 
-> `abstract` **update**(`opts`): `Promise`&lt;`void`&gt;
+```ts
+abstract update(opts): Promise<void>
+```
 
 Update existing records in the Table
 
 ##### Parameters
 
-• **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+* **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
 
 ##### Returns
 
@@ -607,7 +641,9 @@ table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
 
 #### update(updates, options)
 
-> `abstract` **update**(`updates`, `options`?): `Promise`&lt;`void`&gt;
+```ts
+abstract update(updates, options?): Promise<void>
+```
 
 Update existing records in the Table
 
@@ -626,20 +662,17 @@ repeatedly calilng this method.
 
 ##### Parameters
 
-• **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+    the
+    columns to update
+    Keys in the map should specify the name of the column to update.
+    Values in the map provide the new value of the column.  These can
+    be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+    based on the row being updated (e.g. "my_col + 1")
 
-the
-columns to update
-
-Keys in the map should specify the name of the column to update.
-Values in the map provide the new value of the column.  These can
-be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
-based on the row being updated (e.g. "my_col + 1")
-
-• **options?**: `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
-
-additional options to control
-the update behavior
+* **options?**: `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+    additional options to control
+    the update behavior
 
 ##### Returns
 
@@ -649,7 +682,9 @@ the update behavior
 
 ### vectorSearch()
 
-> `abstract` **vectorSearch**(`vector`): [`VectorQuery`](VectorQuery.md)
+```ts
+abstract vectorSearch(vector): VectorQuery
+```
 
 Search the table with a given query vector.
 
@@ -659,7 +694,7 @@ by `query`.
 
 #### Parameters
 
-• **vector**: `IntoVector`
+* **vector**: `IntoVector`
 
 #### Returns
 
@@ -673,7 +708,9 @@ by `query`.
 
 ### version()
 
-> `abstract` **version**(): `Promise`&lt;`number`&gt;
+```ts
+abstract version(): Promise<number>
+```
 
 Retrieve the version of the table
 
@@ -685,15 +722,20 @@ Retrieve the version of the table
 
 ### parseTableData()
 
-> `static` **parseTableData**(`data`, `options`?, `streaming`?): `Promise`&lt;`object`&gt;
+```ts
+static parseTableData(
+   data,
+   options?,
+   streaming?): Promise<object>
+```
 
 #### Parameters
 
-• **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
+* **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
 
-• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
 
-• **streaming?**: `boolean` = `false`
+* **streaming?**: `boolean` = `false`
 
 #### Returns
 
@@ -701,8 +743,12 @@ Retrieve the version of the table
 
 ##### buf
 
-> **buf**: `Buffer`
+```ts
+buf: Buffer;
+```
 
 ##### mode
 
-> **mode**: `string`
+```ts
+mode: string;
+```

docs/src/js/classes/VectorColumnOptions.md

@@ -10,11 +10,13 @@
 
 ### new VectorColumnOptions()
 
-> **new VectorColumnOptions**(`values`?): [`VectorColumnOptions`](VectorColumnOptions.md)
+```ts
+new VectorColumnOptions(values?): VectorColumnOptions
+```
 
 #### Parameters
 
-• **values?**: `Partial`<[`VectorColumnOptions`](VectorColumnOptions.md)>
+* **values?**: `Partial`<[`VectorColumnOptions`](VectorColumnOptions.md)>
 
 #### Returns
 
@@ -24,6 +26,8 @@
 
 ### type
 
-> **type**: `Float`<`Floats`>
+```ts
+type: Float<Floats>;
+```
 
 Vector column type.

docs/src/js/classes/VectorQuery.md

@@ -18,11 +18,13 @@ This builder can be reused to execute the query many times.
 
 ### new VectorQuery()
 
-> **new VectorQuery**(`inner`): [`VectorQuery`](VectorQuery.md)
+```ts
+new VectorQuery(inner): VectorQuery
+```
 
 #### Parameters
 
-• **inner**: `VectorQuery` \| `Promise`<`VectorQuery`>
+* **inner**: `VectorQuery` \| `Promise`<`VectorQuery`>
 
 #### Returns
 
@@ -36,7 +38,9 @@ This builder can be reused to execute the query many times.
 
 ### inner
 
-> `protected` **inner**: `VectorQuery` \| `Promise`<`VectorQuery`>
+```ts
+protected inner: VectorQuery | Promise<VectorQuery>;
+```
 
 #### Inherited from
 
@@ -46,7 +50,9 @@ This builder can be reused to execute the query many times.
 
 ### \[asyncIterator\]()
 
-> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+```ts
+asyncIterator: AsyncIterator<RecordBatch<any>, any, undefined>
+```
 
 #### Returns
 
@@ -58,9 +64,27 @@ This builder can be reused to execute the query many times.
 
 ***
 
+### addQueryVector()
+
+```ts
+addQueryVector(vector): VectorQuery
+```
+
+#### Parameters
+
+* **vector**: `IntoVector`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
 ### bypassVectorIndex()
 
-> **bypassVectorIndex**(): [`VectorQuery`](VectorQuery.md)
+```ts
+bypassVectorIndex(): VectorQuery
+```
 
 If this is called then any vector index is skipped
 
@@ -78,7 +102,9 @@ calculate your recall to select an appropriate value for nprobes.
 
 ### column()
 
-> **column**(`column`): [`VectorQuery`](VectorQuery.md)
+```ts
+column(column): VectorQuery
+```
 
 Set the vector column to query
 
@@ -87,7 +113,7 @@ the call to
 
 #### Parameters
 
-• **column**: `string`
+* **column**: `string`
 
 #### Returns
 
@@ -104,7 +130,9 @@ whose data type is a fixed-size-list of floats.
 
 ### distanceType()
 
-> **distanceType**(`distanceType`): [`VectorQuery`](VectorQuery.md)
+```ts
+distanceType(distanceType): VectorQuery
+```
 
 Set the distance metric to use
 
@@ -114,7 +142,7 @@ use.  See
 
 #### Parameters
 
-• **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
+* **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
 
 #### Returns
 
@@ -135,11 +163,13 @@ By default "l2" is used.
 
 ### doCall()
 
-> `protected` **doCall**(`fn`): `void`
+```ts
+protected doCall(fn): void
+```
 
 #### Parameters
 
-• **fn**
+* **fn**
 
 #### Returns
 
@@ -151,15 +181,41 @@ By default "l2" is used.
 
 ***
 
+### ef()
+
+```ts
+ef(ef): VectorQuery
+```
+
+Set the number of candidates to consider during the search
+
+This argument is only used when the vector column has an HNSW index.
+If there is no index then this value is ignored.
+
+Increasing this value will increase the recall of your query but will
+also increase the latency of your query. The default value is 1.5*limit.
+
+#### Parameters
+
+* **ef**: `number`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
 ### execute()
 
-> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+```ts
+protected execute(options?): RecordBatchIterator
+```
 
 Execute the query and return the results as an
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -185,15 +241,16 @@ single query)
 
 ### explainPlan()
 
-> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;
+```ts
+explainPlan(verbose): Promise<string>
+```
 
 Generates an explanation of the query execution plan.
 
 #### Parameters
 
-• **verbose**: `boolean` = `false`
-
-If true, provides a more detailed explanation. Defaults to false.
+* **verbose**: `boolean` = `false`
+    If true, provides a more detailed explanation. Defaults to false.
 
 #### Returns
 
@@ -218,15 +275,38 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 
 ***
 
+### fastSearch()
+
+```ts
+fastSearch(): this
+```
+
+Skip searching un-indexed data. This can make search faster, but will miss
+any data that is not yet indexed.
+
+Use lancedb.Table#optimize to index all un-indexed data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
+
+***
+
 ### ~~filter()~~
 
-> **filter**(`predicate`): `this`
+```ts
+filter(predicate): this
+```
 
 A filter statement to be applied to this query.
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -246,9 +326,33 @@ Use `where` instead
 
 ***
 
+### fullTextSearch()
+
+```ts
+fullTextSearch(query, options?): this
+```
+
+#### Parameters
+
+* **query**: `string`
+
+* **options?**: `Partial`&lt;`FullTextSearchOptions`&gt;
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
+
+***
+
 ### limit()
 
-> **limit**(`limit`): `this`
+```ts
+limit(limit): this
+```
 
 Set the maximum number of results to return.
 
@@ -257,7 +361,7 @@ called then every valid row from the table will be returned.
 
 #### Parameters
 
-• **limit**: `number`
+* **limit**: `number`
 
 #### Returns
 
@@ -271,11 +375,13 @@ called then every valid row from the table will be returned.
 
 ### nativeExecute()
 
-> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
+```ts
+protected nativeExecute(options?): Promise<RecordBatchIterator>
+```
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -289,7 +395,9 @@ called then every valid row from the table will be returned.
 
 ### nprobes()
 
-> **nprobes**(`nprobes`): [`VectorQuery`](VectorQuery.md)
+```ts
+nprobes(nprobes): VectorQuery
+```
 
 Set the number of partitions to search (probe)
 
@@ -314,7 +422,7 @@ you the desired recall.
 
 #### Parameters
 
-• **nprobes**: `number`
+* **nprobes**: `number`
 
 #### Returns
 
@@ -322,9 +430,31 @@ you the desired recall.
 
 ***
 
+### offset()
+
+```ts
+offset(offset): this
+```
+
+#### Parameters
+
+* **offset**: `number`
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
+
+***
+
 ### postfilter()
 
-> **postfilter**(): [`VectorQuery`](VectorQuery.md)
+```ts
+postfilter(): VectorQuery
+```
 
 If this is called then filtering will happen after the vector search instead of
 before.
@@ -356,7 +486,9 @@ factor can often help restore some of the results lost by post filtering.
 
 ### refineFactor()
 
-> **refineFactor**(`refineFactor`): [`VectorQuery`](VectorQuery.md)
+```ts
+refineFactor(refineFactor): VectorQuery
+```
 
 A multiplier to control how many additional rows are taken during the refine step
 
@@ -388,7 +520,7 @@ distance between the query vector and the actual uncompressed vector.
 
 #### Parameters
 
-• **refineFactor**: `number`
+* **refineFactor**: `number`
 
 #### Returns
 
@@ -398,7 +530,9 @@ distance between the query vector and the actual uncompressed vector.
 
 ### select()
 
-> **select**(`columns`): `this`
+```ts
+select(columns): this
+```
 
 Return only the specified columns.
 
@@ -422,7 +556,7 @@ input to this method would be:
 
 #### Parameters
 
-• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
 
 #### Returns
 
@@ -449,13 +583,15 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 ### toArray()
 
-> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
+```ts
+toArray(options?): Promise<any[]>
+```
 
 Collect the results as an array of objects.
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -469,13 +605,15 @@ Collect the results as an array of objects.
 
 ### toArrow()
 
-> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+```ts
+toArrow(options?): Promise<Table<any>>
+```
 
 Collect the results as an Arrow
 
 #### Parameters
 
-• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+* **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
@@ -493,7 +631,9 @@ ArrowTable.
 
 ### where()
 
-> **where**(`predicate`): `this`
+```ts
+where(predicate): this
+```
 
 A filter statement to be applied to this query.
 
@@ -501,7 +641,7 @@ The filter should be supplied as an SQL query string.  For example:
 
 #### Parameters
 
-• **predicate**: `string`
+* **predicate**: `string`
 
 #### Returns
 
@@ -521,3 +661,25 @@ on the filter column(s).
 #### Inherited from
 
 [`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
+
+***
+
+### withRowId()
+
+```ts
+withRowId(): this
+```
+
+Whether to return the row id in the results.
+
+This column can be used to match results between different queries. For
+example, to match results from a full text search and a vector search in
+order to perform hybrid search.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)

docs/src/js/enumerations/WriteMode.md

@@ -12,16 +12,22 @@ Write mode for writing a table.
 
 ### Append
 
-> **Append**: `"Append"`
+```ts
+Append: "Append";
+```
 
 ***
 
 ### Create
 
-> **Create**: `"Create"`
+```ts
+Create: "Create";
+```
 
 ***
 
 ### Overwrite
 
-> **Overwrite**: `"Overwrite"`
+```ts
+Overwrite: "Overwrite";
+```

docs/src/js/functions/connect.md

@@ -8,7 +8,9 @@
 
 ## connect(uri, opts)
 
-> **connect**(`uri`, `opts`?): `Promise`<[`Connection`](../classes/Connection.md)>
+```ts
+function connect(uri, opts?): Promise<Connection>
+```
 
 Connect to a LanceDB instance at the given URI.
 
@@ -20,12 +22,11 @@ Accepted formats:
 
 ### Parameters
 
-• **uri**: `string`
+* **uri**: `string`
+    The uri of the database. If the database uri starts
+    with `db://` then it connects to a remote database.
 
-The uri of the database. If the database uri starts
-with `db://` then it connects to a remote database.
-
-• **opts?**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`&gt;
+* **opts?**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md)&gt;
 
 ### Returns
 
@@ -50,7 +51,9 @@ const conn = await connect(
 
 ## connect(opts)
 
-> **connect**(`opts`): `Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+```ts
+function connect(opts): Promise<Connection>
+```
 
 Connect to a LanceDB instance at the given URI.
 
@@ -62,7 +65,7 @@ Accepted formats:
 
 ### Parameters
 
-• **opts**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`&gt; & `object`
+* **opts**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md)&gt; & `object`
 
 ### Returns
 

docs/src/js/functions/makeArrowTable.md

@@ -6,7 +6,12 @@
 
 # Function: makeArrowTable()
 
-> **makeArrowTable**(`data`, `options`?, `metadata`?): `ArrowTable`
+```ts
+function makeArrowTable(
+   data,
+   options?,
+   metadata?): ArrowTable
+```
 
 An enhanced version of the makeTable function from Apache Arrow
 that supports nested fields and embeddings columns.
@@ -40,11 +45,11 @@ rules are as follows:
 
 ## Parameters
 
-• **data**: `Record`<`string`, `unknown`>[]
+* **data**: `Record`<`string`, `unknown`>[]
 
-• **options?**: `Partial`<[`MakeArrowTableOptions`](../classes/MakeArrowTableOptions.md)>
+* **options?**: `Partial`<[`MakeArrowTableOptions`](../classes/MakeArrowTableOptions.md)>
 
-• **metadata?**: `Map`<`string`, `string`>
+* **metadata?**: `Map`<`string`, `string`>
 
 ## Returns
 

docs/src/js/globals.md

@@ -28,17 +28,19 @@
 
 - [AddColumnsSql](interfaces/AddColumnsSql.md)
 - [AddDataOptions](interfaces/AddDataOptions.md)
+- [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [ExecutableQuery](interfaces/ExecutableQuery.md)
 - [IndexConfig](interfaces/IndexConfig.md)
-- [IndexMetadata](interfaces/IndexMetadata.md)
 - [IndexOptions](interfaces/IndexOptions.md)
 - [IndexStatistics](interfaces/IndexStatistics.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
-- [FtsOptions](interfaces/FtsOptions.md)
+- [OptimizeOptions](interfaces/OptimizeOptions.md)
+- [RetryConfig](interfaces/RetryConfig.md)
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
+- [TimeoutConfig](interfaces/TimeoutConfig.md)
 - [UpdateOptions](interfaces/UpdateOptions.md)
 - [WriteOptions](interfaces/WriteOptions.md)
 

docs/src/js/interfaces/AddColumnsSql.md

@@ -12,7 +12,9 @@ A definition of a new column to add to a table.
 
 ### name
 
-> **name**: `string`
+```ts
+name: string;
+```
 
 The name of the new column.
 
@@ -20,7 +22,9 @@ The name of the new column.
 
 ### valueSql
 
-> **valueSql**: `string`
+```ts
+valueSql: string;
+```
 
 The values to populate the new column with, as a SQL expression.
 The expression can reference other columns in the table.

docs/src/js/interfaces/AddDataOptions.md

@@ -12,7 +12,9 @@ Options for adding data to a table.
 
 ### mode
 
-> **mode**: `"append"` \| `"overwrite"`
+```ts
+mode: "append" | "overwrite";
+```
 
 If "append" (the default) then the new data will be added to the table
 

docs/src/js/interfaces/ClientConfig.md

@@ -0,0 +1,31 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ClientConfig
+
+# Interface: ClientConfig
+
+## Properties
+
+### retryConfig?
+
+```ts
+optional retryConfig: RetryConfig;
+```
+
+***
+
+### timeoutConfig?
+
+```ts
+optional timeoutConfig: TimeoutConfig;
+```
+
+***
+
+### userAgent?
+
+```ts
+optional userAgent: string;
+```

docs/src/js/interfaces/ColumnAlteration.md

@@ -13,9 +13,29 @@ must be provided.
 
 ## Properties
 
+### dataType?
+
+```ts
+optional dataType: string;
+```
+
+A new data type for the column. If not provided then the data type will not be changed.
+Changing data types is limited to casting to the same general type. For example, these
+changes are valid:
+* `int32` -> `int64` (integers)
+* `double` -> `float` (floats)
+* `string` -> `large_string` (strings)
+But these changes are not:
+* `int32` -> `double` (mix integers and floats)
+* `string` -> `int32` (mix strings and integers)
+
+***
+
 ### nullable?
 
-> `optional` **nullable**: `boolean`
+```ts
+optional nullable: boolean;
+```
 
 Set the new nullability. Note that a nullable column cannot be made non-nullable.
 
@@ -23,7 +43,9 @@ Set the new nullability. Note that a nullable column cannot be made non-nullable
 
 ### path
 
-> **path**: `string`
+```ts
+path: string;
+```
 
 The path to the column to alter. This is a dot-separated path to the column.
 If it is a top-level column then it is just the name of the column. If it is
@@ -34,7 +56,9 @@ a nested column then it is the path to the column, e.g. "a.b.c" for a column
 
 ### rename?
 
-> `optional` **rename**: `string`
+```ts
+optional rename: string;
+```
 
 The new name of the column. If not provided then the name will not be changed.
 This must be distinct from the names of all other columns in the table.

docs/src/js/interfaces/ConnectionOptions.md

@@ -8,9 +8,44 @@
 
 ## Properties
 
+### apiKey?
+
+```ts
+optional apiKey: string;
+```
+
+(For LanceDB cloud only): the API key to use with LanceDB Cloud.
+
+Can also be set via the environment variable `LANCEDB_API_KEY`.
+
+***
+
+### clientConfig?
+
+```ts
+optional clientConfig: ClientConfig;
+```
+
+(For LanceDB cloud only): configuration for the remote HTTP client.
+
+***
+
+### hostOverride?
+
+```ts
+optional hostOverride: string;
+```
+
+(For LanceDB cloud only): the host to use for LanceDB cloud. Used
+for testing purposes.
+
+***
+
 ### readConsistencyInterval?
 
-> `optional` **readConsistencyInterval**: `number`
+```ts
+optional readConsistencyInterval: number;
+```
 
 (For LanceDB OSS only): The interval, in seconds, at which to check for
 updates to the table from other processes. If None, then consistency is not
@@ -24,9 +59,22 @@ always consistent.
 
 ***
 
+### region?
+
+```ts
+optional region: string;
+```
+
+(For LanceDB cloud only): the region to use for LanceDB cloud.
+Defaults to 'us-east-1'.
+
+***
+
 ### storageOptions?
 
-> `optional` **storageOptions**: `Record`<`string`, `string`>
+```ts
+optional storageOptions: Record<string, string>;
+```
 
 (For LanceDB OSS only): configuration for object storage.
 

docs/src/js/interfaces/CreateTableOptions.md

@@ -8,15 +8,46 @@
 
 ## Properties
 
+### dataStorageVersion?
+
+```ts
+optional dataStorageVersion: string;
+```
+
+The version of the data storage format to use.
+
+The default is `stable`.
+Set to "legacy" to use the old format.
+
+***
+
 ### embeddingFunction?
 
-> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)
+```ts
+optional embeddingFunction: EmbeddingFunctionConfig;
+```
+
+***
+
+### enableV2ManifestPaths?
+
+```ts
+optional enableV2ManifestPaths: boolean;
+```
+
+Use the new V2 manifest paths. These paths provide more efficient
+opening of datasets with many versions on object stores.  WARNING:
+turning this on will make the dataset unreadable for older versions
+of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
+use the LocalTable#migrateManifestPathsV2 method.
 
 ***
 
 ### existOk
 
-> **existOk**: `boolean`
+```ts
+existOk: boolean;
+```
 
 If this is true and the table already exists and the mode is "create"
 then no error will be raised.
@@ -25,7 +56,9 @@ then no error will be raised.
 
 ### mode
 
-> **mode**: `"overwrite"` \| `"create"`
+```ts
+mode: "overwrite" | "create";
+```
 
 The mode to use when creating the table.
 
@@ -39,13 +72,17 @@ If this is set to "overwrite" then any existing table will be replaced.
 
 ### schema?
 
-> `optional` **schema**: `SchemaLike`
+```ts
+optional schema: SchemaLike;
+```
 
 ***
 
 ### storageOptions?
 
-> `optional` **storageOptions**: `Record`<`string`, `string`>
+```ts
+optional storageOptions: Record<string, string>;
+```
 
 Configuration for object storage.
 
@@ -58,8 +95,12 @@ The available options are described at https://lancedb.github.io/lancedb/guides/
 
 ### useLegacyFormat?
 
-> `optional` **useLegacyFormat**: `boolean`
+```ts
+optional useLegacyFormat: boolean;
+```
 
 If true then data files will be written with the legacy format
 
-The default is true while the new format is in beta
+The default is false.
+
+Deprecated. Use data storage version instead.

docs/src/js/interfaces/FtsOptions.md

@@ -1,25 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / FtsOptions
-
-# Interface: FtsOptions
-
-Options to create an `FTS` index
-
-## Properties
-
-### withPosition?
-
-> `optional` **withPosition**: `boolean`
-
-Whether to store the positions of the term in the document.
-
-If this is true then the index will store the positions of the term in the document.
-This allows phrase queries to be run. But it also increases the size of the index,
-and the time to build the index.
-
-The default value is true.
-
-***

docs/src/js/interfaces/IndexConfig.md

@@ -12,7 +12,9 @@ A description of an index currently configured on a column
 
 ### columns
 
-> **columns**: `string`[]
+```ts
+columns: string[];
+```
 
 The columns in the index
 
@@ -23,7 +25,9 @@ be more columns to represent composite indices.
 
 ### indexType
 
-> **indexType**: `string`
+```ts
+indexType: string;
+```
 
 The type of the index
 
@@ -31,6 +35,8 @@ The type of the index
 
 ### name
 
-> **name**: `string`
+```ts
+name: string;
+```
 
 The name of the index

docs/src/js/interfaces/IndexMetadata.md

@@ -1,19 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / IndexMetadata
-
-# Interface: IndexMetadata
-
-## Properties
-
-### indexType?
-
-> `optional` **indexType**: `string`
-
-***
-
-### metricType?
-
-> `optional` **metricType**: `string`

docs/src/js/interfaces/IndexOptions.md

@@ -10,7 +10,9 @@
 
 ### config?
 
-> `optional` **config**: [`Index`](../classes/Index.md)
+```ts
+optional config: Index;
+```
 
 Advanced index configuration
 
@@ -26,7 +28,9 @@ will be used to determine the most useful kind of index to create.
 
 ### replace?
 
-> `optional` **replace**: `boolean`
+```ts
+optional replace: boolean;
+```
 
 Whether to replace the existing index
 

docs/src/js/interfaces/IndexStatistics.md

@@ -8,32 +8,52 @@
 
 ## Properties
 
-### indexType?
+### distanceType?
 
-> `optional` **indexType**: `string`
+```ts
+optional distanceType: string;
+```
+
+The type of the distance function used by the index. This is only
+present for vector indices. Scalar and full text search indices do
+not have a distance function.
+
+***
+
+### indexType
+
+```ts
+indexType: string;
+```
 
 The type of the index
 
 ***
 
-### indices
-
-> **indices**: [`IndexMetadata`](IndexMetadata.md)[]
-
-The metadata for each index
-
-***
-
 ### numIndexedRows
 
-> **numIndexedRows**: `number`
+```ts
+numIndexedRows: number;
+```
 
 The number of rows indexed by the index
 
 ***
 
+### numIndices?
+
+```ts
+optional numIndices: number;
+```
+
+The number of parts this index is split into.
+
+***
+
 ### numUnindexedRows
 
-> **numUnindexedRows**: `number`
+```ts
+numUnindexedRows: number;
+```
 
 The number of rows not indexed

docs/src/js/interfaces/IvfPqOptions.md

@@ -12,7 +12,9 @@ Options to create an `IVF_PQ` index
 
 ### distanceType?
 
-> `optional` **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
+```ts
+optional distanceType: "l2" | "cosine" | "dot";
+```
 
 Distance type to use to build the index.
 
@@ -50,7 +52,9 @@ L2 norm is 1), then dot distance is equivalent to the cosine distance.
 
 ### maxIterations?
 
-> `optional` **maxIterations**: `number`
+```ts
+optional maxIterations: number;
+```
 
 Max iteration to train IVF kmeans.
 
@@ -66,7 +70,9 @@ The default value is 50.
 
 ### numPartitions?
 
-> `optional` **numPartitions**: `number`
+```ts
+optional numPartitions: number;
+```
 
 The number of IVF partitions to create.
 
@@ -82,7 +88,9 @@ part of the search (searching within a partition) will be slow.
 
 ### numSubVectors?
 
-> `optional` **numSubVectors**: `number`
+```ts
+optional numSubVectors: number;
+```
 
 Number of sub-vectors of PQ.
 
@@ -101,7 +109,9 @@ will likely result in poor performance.
 
 ### sampleRate?
 
-> `optional` **sampleRate**: `number`
+```ts
+optional sampleRate: number;
+```
 
 The number of vectors, per partition, to sample when training IVF kmeans.
 

docs/src/js/interfaces/OptimizeOptions.md

@@ -0,0 +1,39 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / OptimizeOptions
+
+# Interface: OptimizeOptions
+
+## Properties
+
+### cleanupOlderThan
+
+```ts
+cleanupOlderThan: Date;
+```
+
+If set then all versions older than the given date
+be removed.  The current version will never be removed.
+The default is 7 days
+
+#### Example
+
+```ts
+// Delete all versions older than 1 day
+const olderThan = new Date();
+olderThan.setDate(olderThan.getDate() - 1));
+tbl.cleanupOlderVersions(olderThan);
+
+// Delete all versions except the current version
+tbl.cleanupOlderVersions(new Date());
+```
+
+***
+
+### deleteUnverified
+
+```ts
+deleteUnverified: boolean;
+```

docs/src/js/interfaces/RetryConfig.md

@@ -0,0 +1,90 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RetryConfig
+
+# Interface: RetryConfig
+
+Retry configuration for the remote HTTP client.
+
+## Properties
+
+### backoffFactor?
+
+```ts
+optional backoffFactor: number;
+```
+
+The backoff factor to apply between retries. Default is 0.25. Between each retry
+the client will wait for the amount of seconds:
+`{backoff factor} * (2 ** ({number of previous retries}))`. So for the default
+of 0.25, the first retry will wait 0.25 seconds, the second retry will wait 0.5
+seconds, the third retry will wait 1 second, etc.
+
+You can also set this via the environment variable
+`LANCE_CLIENT_RETRY_BACKOFF_FACTOR`.
+
+***
+
+### backoffJitter?
+
+```ts
+optional backoffJitter: number;
+```
+
+The jitter to apply to the backoff factor, in seconds. Default is 0.25.
+
+A random value between 0 and `backoff_jitter` will be added to the backoff
+factor in seconds. So for the default of 0.25 seconds, between 0 and 250
+milliseconds will be added to the sleep between each retry.
+
+You can also set this via the environment variable
+`LANCE_CLIENT_RETRY_BACKOFF_JITTER`.
+
+***
+
+### connectRetries?
+
+```ts
+optional connectRetries: number;
+```
+
+The maximum number of retries for connection errors. Default is 3. You
+can also set this via the environment variable `LANCE_CLIENT_CONNECT_RETRIES`.
+
+***
+
+### readRetries?
+
+```ts
+optional readRetries: number;
+```
+
+The maximum number of retries for read errors. Default is 3. You can also
+set this via the environment variable `LANCE_CLIENT_READ_RETRIES`.
+
+***
+
+### retries?
+
+```ts
+optional retries: number;
+```
+
+The maximum number of retries for a request. Default is 3. You can also
+set this via the environment variable `LANCE_CLIENT_MAX_RETRIES`.
+
+***
+
+### statuses?
+
+```ts
+optional statuses: number[];
+```
+
+The HTTP status codes for which to retry the request. Default is
+[429, 500, 502, 503].
+
+You can also set this via the environment variable
+`LANCE_CLIENT_RETRY_STATUSES`. Use a comma-separated list of integers.

docs/src/js/interfaces/TableNamesOptions.md

@@ -10,7 +10,9 @@
 
 ### limit?
 
-> `optional` **limit**: `number`
+```ts
+optional limit: number;
+```
 
 An optional limit to the number of results to return.
 
@@ -18,7 +20,9 @@ An optional limit to the number of results to return.
 
 ### startAfter?
 
-> `optional` **startAfter**: `string`
+```ts
+optional startAfter: string;
+```
 
 If present, only return names that come lexicographically after the
 supplied value.

docs/src/js/interfaces/TimeoutConfig.md

@@ -0,0 +1,46 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TimeoutConfig
+
+# Interface: TimeoutConfig
+
+Timeout configuration for remote HTTP client.
+
+## Properties
+
+### connectTimeout?
+
+```ts
+optional connectTimeout: number;
+```
+
+The timeout for establishing a connection in seconds. Default is 120
+seconds (2 minutes). This can also be set via the environment variable
+`LANCE_CLIENT_CONNECT_TIMEOUT`, as an integer number of seconds.
+
+***
+
+### poolIdleTimeout?
+
+```ts
+optional poolIdleTimeout: number;
+```
+
+The timeout for keeping idle connections in the connection pool in seconds.
+Default is 300 seconds (5 minutes). This can also be set via the
+environment variable `LANCE_CLIENT_CONNECTION_TIMEOUT`, as an integer
+number of seconds.
+
+***
+
+### readTimeout?
+
+```ts
+optional readTimeout: number;
+```
+
+The timeout for reading data from the server in seconds. Default is 300
+seconds (5 minutes). This can also be set via the environment variable
+`LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.

docs/src/js/interfaces/UpdateOptions.md

@@ -10,7 +10,9 @@
 
 ### where
 
-> **where**: `string`
+```ts
+where: string;
+```
 
 A filter that limits the scope of the update.
 

docs/src/js/interfaces/WriteOptions.md

@@ -12,6 +12,8 @@ Write options when creating a Table.
 
 ### mode?
 
-> `optional` **mode**: [`WriteMode`](../enumerations/WriteMode.md)
+```ts
+optional mode: WriteMode;
+```
 
 Write mode for writing to a table.

docs/src/js/namespaces/embedding/README.md

@@ -12,16 +12,12 @@
 
 - [EmbeddingFunction](classes/EmbeddingFunction.md)
 - [EmbeddingFunctionRegistry](classes/EmbeddingFunctionRegistry.md)
-- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
+- [TextEmbeddingFunction](classes/TextEmbeddingFunction.md)
 
 ### Interfaces
 
 - [EmbeddingFunctionConfig](interfaces/EmbeddingFunctionConfig.md)
 
-### Type Aliases
-
-- [OpenAIOptions](type-aliases/OpenAIOptions.md)
-
 ### Functions
 
 - [LanceSchema](functions/LanceSchema.md)

docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md

@@ -10,7 +10,7 @@ An embedding function that automatically creates vector representation for a giv
 
 ## Extended by
 
-- [`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+- [`TextEmbeddingFunction`](TextEmbeddingFunction.md)
 
 ## Type Parameters
 
@@ -22,7 +22,9 @@ An embedding function that automatically creates vector representation for a giv
 
 ### new EmbeddingFunction()
 
-> **new EmbeddingFunction**<`T`, `M`>(): [`EmbeddingFunction`](EmbeddingFunction.md)<`T`, `M`>
+```ts
+new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
+```
 
 #### Returns
 
@@ -32,13 +34,15 @@ An embedding function that automatically creates vector representation for a giv
 
 ### computeQueryEmbeddings()
 
-> **computeQueryEmbeddings**(`data`): `Promise`&lt;`number`[] \| `Float32Array` \| `Float64Array`&gt;
+```ts
+computeQueryEmbeddings(data): Promise<number[] | Float32Array | Float64Array>
+```
 
 Compute the embeddings for a single query
 
 #### Parameters
 
-• **data**: `T`
+* **data**: `T`
 
 #### Returns
 
@@ -48,13 +52,15 @@ Compute the embeddings for a single query
 
 ### computeSourceEmbeddings()
 
-> `abstract` **computeSourceEmbeddings**(`data`): `Promise`&lt;`number`[][] \| `Float32Array`[] \| `Float64Array`[]&gt;
+```ts
+abstract computeSourceEmbeddings(data): Promise<number[][] | Float32Array[] | Float64Array[]>
+```
 
 Creates a vector representation for the given values.
 
 #### Parameters
 
-• **data**: `T`[]
+* **data**: `T`[]
 
 #### Returns
 
@@ -64,7 +70,9 @@ Creates a vector representation for the given values.
 
 ### embeddingDataType()
 
-> `abstract` **embeddingDataType**(): `Float`&lt;`Floats`&gt;
+```ts
+abstract embeddingDataType(): Float<Floats>
+```
 
 The datatype of the embeddings
 
@@ -74,9 +82,23 @@ The datatype of the embeddings
 
 ***
 
+### init()?
+
+```ts
+optional init(): Promise<void>
+```
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### ndims()
 
-> **ndims**(): `undefined` \| `number`
+```ts
+ndims(): undefined | number
+```
 
 The number of dimensions of the embeddings
 
@@ -88,15 +110,16 @@ The number of dimensions of the embeddings
 
 ### sourceField()
 
-> **sourceField**(`optionsOrDatatype`): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+```ts
+sourceField(optionsOrDatatype): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
+```
 
 sourceField is used in combination with `LanceSchema` to provide a declarative data model
 
 #### Parameters
 
-• **optionsOrDatatype**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
-
-The options for the field or the datatype
+* **optionsOrDatatype**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
+    The options for the field or the datatype
 
 #### Returns
 
@@ -110,7 +133,9 @@ lancedb.LanceSchema
 
 ### toJSON()
 
-> `abstract` **toJSON**(): `Partial`&lt;`M`&gt;
+```ts
+abstract toJSON(): Partial<M>
+```
 
 Convert the embedding function to a JSON object
 It is used to serialize the embedding function to the schema
@@ -145,13 +170,15 @@ class MyEmbeddingFunction extends EmbeddingFunction {
 
 ### vectorField()
 
-> **vectorField**(`optionsOrDatatype`?): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+```ts
+vectorField(optionsOrDatatype?): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
+```
 
 vectorField is used in combination with `LanceSchema` to provide a declarative data model
 
 #### Parameters
 
-• **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
+* **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
 
 #### Returns
 

docs/src/js/namespaces/embedding/classes/EmbeddingFunctionRegistry.md

@@ -15,7 +15,9 @@ or TextEmbeddingFunction and registering it with the registry
 
 ### new EmbeddingFunctionRegistry()
 
-> **new EmbeddingFunctionRegistry**(): [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+```ts
+new EmbeddingFunctionRegistry(): EmbeddingFunctionRegistry
+```
 
 #### Returns
 
@@ -25,11 +27,13 @@ or TextEmbeddingFunction and registering it with the registry
 
 ### functionToMetadata()
 
-> **functionToMetadata**(`conf`): `Record`<`string`, `any`>
+```ts
+functionToMetadata(conf): Record<string, any>
+```
 
 #### Parameters
 
-• **conf**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)
+* **conf**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)
 
 #### Returns
 
@@ -39,7 +43,9 @@ or TextEmbeddingFunction and registering it with the registry
 
 ### get()
 
-> **get**&lt;`T`, `Name`&gt;(`name`): `Name` *extends* `"openai"` ? `EmbeddingFunctionCreate`&lt;[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)&gt; : `undefined` \| `EmbeddingFunctionCreate`&lt;`T`&gt;
+```ts
+get<T>(name): undefined | EmbeddingFunctionCreate<T>
+```
 
 Fetch an embedding function by name
 
@@ -47,27 +53,26 @@ Fetch an embedding function by name
 
 • **T** *extends* [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`unknown`, `FunctionOptions`&gt;
 
-• **Name** *extends* `string` = `""`
-
 #### Parameters
 
-• **name**: `Name` *extends* `"openai"` ? `"openai"` : `string`
-
-The name of the function
+* **name**: `string`
+    The name of the function
 
 #### Returns
 
-`Name` *extends* `"openai"` ? `EmbeddingFunctionCreate`&lt;[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)&gt; : `undefined` \| `EmbeddingFunctionCreate`&lt;`T`&gt;
+`undefined` \| `EmbeddingFunctionCreate`&lt;`T`&gt;
 
 ***
 
 ### getTableMetadata()
 
-> **getTableMetadata**(`functions`): `Map`&lt;`string`, `string`&gt;
+```ts
+getTableMetadata(functions): Map<string, string>
+```
 
 #### Parameters
 
-• **functions**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)[]
+* **functions**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)[]
 
 #### Returns
 
@@ -75,9 +80,25 @@ The name of the function
 
 ***
 
+### length()
+
+```ts
+length(): number
+```
+
+Get the number of registered functions
+
+#### Returns
+
+`number`
+
+***
+
 ### register()
 
-> **register**&lt;`T`&gt;(`this`, `alias`?): (`ctor`) => `any`
+```ts
+register<T>(this, alias?): (ctor) => any
+```
 
 Register an embedding function
 
@@ -87,9 +108,9 @@ Register an embedding function
 
 #### Parameters
 
-• **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+* **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
 
-• **alias?**: `string`
+* **alias?**: `string`
 
 #### Returns
 
@@ -97,7 +118,7 @@ Register an embedding function
 
 ##### Parameters
 
-• **ctor**: `T`
+* **ctor**: `T`
 
 ##### Returns
 
@@ -111,13 +132,15 @@ Error if the function is already registered
 
 ### reset()
 
-> **reset**(`this`): `void`
+```ts
+reset(this): void
+```
 
 reset the registry to the initial state
 
 #### Parameters
 
-• **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+* **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
 
 #### Returns
 

docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md

@@ -2,31 +2,33 @@
 
 ***
 
-[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / OpenAIEmbeddingFunction
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / TextEmbeddingFunction
 
-# Class: OpenAIEmbeddingFunction
+# Class: `abstract` TextEmbeddingFunction<M>
 
-An embedding function that automatically creates vector representation for a given column.
+an abstract class for implementing embedding functions that take text as input
 
 ## Extends
 
-- [`EmbeddingFunction`](EmbeddingFunction.md)<`string`, `Partial`<[`OpenAIOptions`](../type-aliases/OpenAIOptions.md)>>
+- [`EmbeddingFunction`](EmbeddingFunction.md)<`string`, `M`>
+
+## Type Parameters
+
+• **M** *extends* `FunctionOptions` = `FunctionOptions`
 
 ## Constructors
 
-### new OpenAIEmbeddingFunction()
+### new TextEmbeddingFunction()
 
-> **new OpenAIEmbeddingFunction**(`options`): [`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
-
-#### Parameters
-
-• **options**: `Partial`<[`OpenAIOptions`](../type-aliases/OpenAIOptions.md)> = `...`
+```ts
+new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
+```
 
 #### Returns
 
-[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+[`TextEmbeddingFunction`](TextEmbeddingFunction.md)&lt;`M`&gt;
 
-#### Overrides
+#### Inherited from
 
 [`EmbeddingFunction`](EmbeddingFunction.md).[`constructor`](EmbeddingFunction.md#constructors)
 
@@ -34,17 +36,19 @@ An embedding function that automatically creates vector representation for a giv
 
 ### computeQueryEmbeddings()
 
-> **computeQueryEmbeddings**(`data`): `Promise`&lt;`number`[]&gt;
+```ts
+computeQueryEmbeddings(data): Promise<number[] | Float32Array | Float64Array>
+```
 
 Compute the embeddings for a single query
 
 #### Parameters
 
-• **data**: `string`
+* **data**: `string`
 
 #### Returns
 
-`Promise`&lt;`number`[]&gt;
+`Promise`&lt;`number`[] \| `Float32Array` \| `Float64Array`&gt;
 
 #### Overrides
 
@@ -54,17 +58,19 @@ Compute the embeddings for a single query
 
 ### computeSourceEmbeddings()
 
-> **computeSourceEmbeddings**(`data`): `Promise`&lt;`number`[][]&gt;
+```ts
+computeSourceEmbeddings(data): Promise<number[][] | Float32Array[] | Float64Array[]>
+```
 
 Creates a vector representation for the given values.
 
 #### Parameters
 
-• **data**: `string`[]
+* **data**: `string`[]
 
 #### Returns
 
-`Promise`&lt;`number`[][]&gt;
+`Promise`&lt;`number`[][] \| `Float32Array`[] \| `Float64Array`[]&gt;
 
 #### Overrides
 
@@ -74,7 +80,9 @@ Creates a vector representation for the given values.
 
 ### embeddingDataType()
 
-> **embeddingDataType**(): `Float`&lt;`Floats`&gt;
+```ts
+embeddingDataType(): Float<Floats>
+```
 
 The datatype of the embeddings
 
@@ -88,17 +96,53 @@ The datatype of the embeddings
 
 ***
 
+### generateEmbeddings()
+
+```ts
+abstract generateEmbeddings(texts, ...args): Promise<number[][] | Float32Array[] | Float64Array[]>
+```
+
+#### Parameters
+
+* **texts**: `string`[]
+
+* ...**args**: `any`[]
+
+#### Returns
+
+`Promise`&lt;`number`[][] \| `Float32Array`[] \| `Float64Array`[]&gt;
+
+***
+
+### init()?
+
+```ts
+optional init(): Promise<void>
+```
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+#### Inherited from
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`init`](EmbeddingFunction.md#init)
+
+***
+
 ### ndims()
 
-> **ndims**(): `number`
+```ts
+ndims(): undefined | number
+```
 
 The number of dimensions of the embeddings
 
 #### Returns
 
-`number`
+`undefined` \| `number`
 
-#### Overrides
+#### Inherited from
 
 [`EmbeddingFunction`](EmbeddingFunction.md).[`ndims`](EmbeddingFunction.md#ndims)
 
@@ -106,16 +150,12 @@ The number of dimensions of the embeddings
 
 ### sourceField()
 
-> **sourceField**(`optionsOrDatatype`): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+```ts
+sourceField(): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
+```
 
 sourceField is used in combination with `LanceSchema` to provide a declarative data model
 
-#### Parameters
-
-• **optionsOrDatatype**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
-
-The options for the field or the datatype
-
 #### Returns
 
 [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
@@ -124,7 +164,7 @@ The options for the field or the datatype
 
 lancedb.LanceSchema
 
-#### Inherited from
+#### Overrides
 
 [`EmbeddingFunction`](EmbeddingFunction.md).[`sourceField`](EmbeddingFunction.md#sourcefield)
 
@@ -132,7 +172,9 @@ lancedb.LanceSchema
 
 ### toJSON()
 
-> **toJSON**(): `object`
+```ts
+abstract toJSON(): Partial<M>
+```
 
 Convert the embedding function to a JSON object
 It is used to serialize the embedding function to the schema
@@ -144,11 +186,7 @@ If it does not, the embedding function will not be able to be recreated, or coul
 
 #### Returns
 
-`object`
-
-##### model
-
-> **model**: `string` & `object` \| `"text-embedding-ada-002"` \| `"text-embedding-3-small"` \| `"text-embedding-3-large"`
+`Partial`&lt;`M`&gt;
 
 #### Example
 
@@ -167,7 +205,7 @@ class MyEmbeddingFunction extends EmbeddingFunction {
 }
 ```
 
-#### Overrides
+#### Inherited from
 
 [`EmbeddingFunction`](EmbeddingFunction.md).[`toJSON`](EmbeddingFunction.md#tojson)
 
@@ -175,13 +213,15 @@ class MyEmbeddingFunction extends EmbeddingFunction {
 
 ### vectorField()
 
-> **vectorField**(`optionsOrDatatype`?): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+```ts
+vectorField(optionsOrDatatype?): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
+```
 
 vectorField is used in combination with `LanceSchema` to provide a declarative data model
 
 #### Parameters
 
-• **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
+* **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
 
 #### Returns
 

docs/src/js/namespaces/embedding/functions/LanceSchema.md

@@ -6,13 +6,15 @@
 
 # Function: LanceSchema()
 
-> **LanceSchema**(`fields`): `Schema`
+```ts
+function LanceSchema(fields): Schema
+```
 
 Create a schema with embedding functions.
 
 ## Parameters
 
-• **fields**: `Record`<`string`, `object` \| [`object`, `Map`<`string`, [`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>>]>
+* **fields**: `Record`<`string`, `object` \| [`object`, `Map`<`string`, [`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>>]>
 
 ## Returns
 

docs/src/js/namespaces/embedding/functions/getRegistry.md

@@ -6,7 +6,9 @@
 
 # Function: getRegistry()
 
-> **getRegistry**(): [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
+```ts
+function getRegistry(): EmbeddingFunctionRegistry
+```
 
 Utility function to get the global instance of the registry
 

docs/src/js/namespaces/embedding/functions/register.md

@@ -6,11 +6,13 @@
 
 # Function: register()
 
-> **register**(`name`?): (`ctor`) => `any`
+```ts
+function register(name?): (ctor) => any
+```
 
 ## Parameters
 
-• **name?**: `string`
+* **name?**: `string`
 
 ## Returns
 
@@ -18,7 +20,7 @@
 
 ### Parameters
 
-• **ctor**: `EmbeddingFunctionConstructor`<[`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>>
+* **ctor**: `EmbeddingFunctionConstructor`<[`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>>
 
 ### Returns
 

docs/src/js/namespaces/embedding/interfaces/EmbeddingFunctionConfig.md

@@ -10,16 +10,22 @@
 
 ### function
 
-> **function**: [`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>
+```ts
+function: EmbeddingFunction<any, FunctionOptions>;
+```
 
 ***
 
 ### sourceColumn
 
-> **sourceColumn**: `string`
+```ts
+sourceColumn: string;
+```
 
 ***
 
 ### vectorColumn?
 
-> `optional` **vectorColumn**: `string`
+```ts
+optional vectorColumn: string;
+```

docs/src/js/namespaces/embedding/type-aliases/OpenAIOptions.md

@@ -1,19 +0,0 @@
-[**@lancedb/lancedb**](../../../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / OpenAIOptions
-
-# Type Alias: OpenAIOptions
-
-> **OpenAIOptions**: `object`
-
-## Type declaration
-
-### apiKey
-
-> **apiKey**: `string`
-
-### model
-
-> **model**: `EmbeddingCreateParams`\[`"model"`\]

docs/src/js/type-aliases/Data.md

@@ -6,6 +6,8 @@
 
 # Type Alias: Data
 
-> **Data**: `Record`<`string`, `unknown`>[] \| `TableLike`
+```ts
+type Data: Record<string, unknown>[] | TableLike;
+```
 
 Data type accepted by NodeJS SDK

docs/src/python/python.md

@@ -1,6 +1,16 @@
 # Python API Reference
 
-This section contains the API reference for the OSS Python API.
+This section contains the API reference for the Python API. There is a
+synchronous and an asynchronous API client.
+
+The general flow of using the API is:
+
+1. Use [lancedb.connect][] or [lancedb.connect_async][] to connect to a database.
+2. Use the returned [lancedb.DBConnection][] or [lancedb.AsyncConnection][] to
+   create or open tables.
+3. Use the returned [lancedb.table.Table][] or [lancedb.AsyncTable][] to query
+   or modify tables.
+
 
 ## Installation
 

docs/src/sql.md

@@ -7,6 +7,10 @@ performed on the top-k results returned by the vector search. However, pre-filte
 option that performs the filter prior to vector search. This can be useful to narrow down on
 the search space on a very large dataset to reduce query latency.
 
+Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to set `use_index=false` if you know that the filter is highly selective.
+
+Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain.
+
 <!-- Setup Code
 ```python
 import lancedb
@@ -57,6 +61,9 @@ const tbl = await db.createTable('myVectors', data)
         ```ts
         --8<-- "docs/src/sql_legacy.ts:search"
         ```
+!!! note
+
+    Creating a [scalar index](guides/scalar_index.md) accelerates filtering
 
 ## SQL filters
 

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.13.0-final.0</version>
+        <version>0.14.1-beta.2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.13.0-final.0</version>
+    <version>0.14.1-beta.2</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.13.0",
+  "version": "0.14.1-beta.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.13.0",
+      "version": "0.14.1-beta.2",
       "cpu": [
         "x64",
         "arm64"
@@ -52,12 +52,14 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.13.0",
-        "@lancedb/vectordb-darwin-x64": "0.13.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.13.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.13.0",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.13.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.13.0"
+        "@lancedb/vectordb-darwin-arm64": "0.14.1-beta.2",
+        "@lancedb/vectordb-darwin-x64": "0.14.1-beta.2",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.14.1-beta.2",
+        "@lancedb/vectordb-linux-arm64-musl": "0.14.1-beta.2",
+        "@lancedb/vectordb-linux-x64-gnu": "0.14.1-beta.2",
+        "@lancedb/vectordb-linux-x64-musl": "0.14.1-beta.2",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.14.1-beta.2",
+        "@lancedb/vectordb-win32-x64-msvc": "0.14.1-beta.2"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -327,66 +329,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.13.0.tgz",
-      "integrity": "sha512-8hdcjkRmgrdQYf1jN+DyZae40LIv8UUfnWy70Uid5qy63sSvRW/+MvIdqIPFr9QlLUXmpyyQuX0y3bZhUR99cQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.13.0.tgz",
-      "integrity": "sha512-fWzAY4l5SQtNfMYh80v+M66ugZHhdxbkpk5mNEv6Zsug3DL6kRj3Uv31/i0wgzY6F5G3LUlbjZerN+eTnDLwOw==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.13.0.tgz",
-      "integrity": "sha512-ltwAT9baOSuR5YiGykQXPC8/HGYF13vpI47qxhP9yfgiz9pA8EUn8p8YrBRzq7J4DIZ4b8JSVDXQnMIqEtB4Kg==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.13.0.tgz",
-      "integrity": "sha512-MiT/RBlMPGGRh7BX+MXwRuNiiUnKmuDcHH8nm88IH28T7TQxXIbA9w6UpSg5m9f3DgKQI2K8oLi29oKIB8ZwDQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.13.0.tgz",
-      "integrity": "sha512-SovP/hwWYLJIy65DKbVuXlBPTb/nwvVpTO6dh9zRch+L5ek6JmVAkwsfeTS2p5bMa8VPujsCXYUAVuCDEJU8wg==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",

node/package.json

@@ -1,7 +1,8 @@
 {
   "name": "vectordb",
-  "version": "0.13.0",
+  "version": "0.14.1-beta.2",
   "description": " Serverless, low-latency vector database for AI applications",
+  "private": false,
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
@@ -84,18 +85,20 @@
       "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
       "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
       "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
+      "x86_64-unknown-linux-musl": "@lancedb/vectordb-linux-x64-musl",
+      "aarch64-unknown-linux-musl": "@lancedb/vectordb-linux-arm64-musl",
       "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc",
       "aarch64-pc-windows-msvc": "@lancedb/vectordb-win32-arm64-msvc"
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.13.0",
-    "@lancedb/vectordb-darwin-arm64": "0.13.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.13.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.13.0",
-    "@lancedb/vectordb-linux-x64-musl": "0.13.0",
-    "@lancedb/vectordb-linux-arm64-musl": "0.13.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.13.0",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.13.0"
+    "@lancedb/vectordb-darwin-x64": "0.14.1-beta.2",
+    "@lancedb/vectordb-darwin-arm64": "0.14.1-beta.2",
+    "@lancedb/vectordb-linux-x64-gnu": "0.14.1-beta.2",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.14.1-beta.2",
+    "@lancedb/vectordb-linux-x64-musl": "0.14.1-beta.2",
+    "@lancedb/vectordb-linux-arm64-musl": "0.14.1-beta.2",
+    "@lancedb/vectordb-win32-x64-msvc": "0.14.1-beta.2",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.14.1-beta.2"
   }
 }

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.13.0"
+version = "0.14.1-beta.2"
 license.workspace = true
 description.workspace = true
 repository.workspace = true

nodejs/__test__/arrow.test.ts

@@ -13,11 +13,10 @@ import { Schema } from "apache-arrow";
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import * as arrow13 from "apache-arrow-13";
-import * as arrow14 from "apache-arrow-14";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
+import * as arrow18 from "apache-arrow-18";
 
 import {
   convertToTable,
@@ -45,22 +44,16 @@ function sampleRecords(): Array<Record<string, any>> {
     },
   ];
 }
-describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
+describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Arrow",
   (
-    arrow:
-      | typeof arrow13
-      | typeof arrow14
-      | typeof arrow15
-      | typeof arrow16
-      | typeof arrow17,
+    arrow: typeof arrow15 | typeof arrow16 | typeof arrow17 | typeof arrow18,
   ) => {
     type ApacheArrow =
-      | typeof arrow13
-      | typeof arrow14
       | typeof arrow15
       | typeof arrow16
-      | typeof arrow17;
+      | typeof arrow17
+      | typeof arrow18;
     const {
       Schema,
       Field,
@@ -498,40 +491,40 @@ describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
 
     describe("when using two versions of arrow", function () {
       it("can still import data", async function () {
-        const schema = new arrow13.Schema([
-          new arrow13.Field("id", new arrow13.Int32()),
-          new arrow13.Field(
+        const schema = new arrow15.Schema([
+          new arrow15.Field("id", new arrow15.Int32()),
+          new arrow15.Field(
             "vector",
-            new arrow13.FixedSizeList(
+            new arrow15.FixedSizeList(
               1024,
-              new arrow13.Field("item", new arrow13.Float32(), true),
+              new arrow15.Field("item", new arrow15.Float32(), true),
             ),
           ),
-          new arrow13.Field(
+          new arrow15.Field(
             "struct",
-            new arrow13.Struct([
-              new arrow13.Field(
+            new arrow15.Struct([
+              new arrow15.Field(
                 "nested",
-                new arrow13.Dictionary(
-                  new arrow13.Utf8(),
-                  new arrow13.Int32(),
+                new arrow15.Dictionary(
+                  new arrow15.Utf8(),
+                  new arrow15.Int32(),
                   1,
                   true,
                 ),
               ),
-              new arrow13.Field(
+              new arrow15.Field(
                 "ts_with_tz",
-                new arrow13.TimestampNanosecond("some_tz"),
+                new arrow15.TimestampNanosecond("some_tz"),
               ),
-              new arrow13.Field(
+              new arrow15.Field(
                 "ts_no_tz",
-                new arrow13.TimestampNanosecond(null),
+                new arrow15.TimestampNanosecond(null),
               ),
             ]),
           ),
           // biome-ignore lint/suspicious/noExplicitAny: skip
         ]) as any;
-        schema.metadataVersion = arrow13.MetadataVersion.V5;
+        schema.metadataVersion = arrow15.MetadataVersion.V5;
         const table = makeArrowTable([], { schema });
 
         const buf = await fromTableToBuffer(table);
@@ -543,13 +536,13 @@ describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
         // Deep equality gets hung up on some very minor unimportant differences
         // between arrow version 13 and 15 which isn't really what we're testing for
         // and so we do our own comparison that just checks name/type/nullability
-        function compareFields(lhs: arrow13.Field, rhs: arrow13.Field) {
+        function compareFields(lhs: arrow15.Field, rhs: arrow15.Field) {
           expect(lhs.name).toEqual(rhs.name);
           expect(lhs.nullable).toEqual(rhs.nullable);
           expect(lhs.typeId).toEqual(rhs.typeId);
           if ("children" in lhs.type && lhs.type.children !== null) {
-            const lhsChildren = lhs.type.children as arrow13.Field[];
-            lhsChildren.forEach((child: arrow13.Field, idx) => {
+            const lhsChildren = lhs.type.children as arrow15.Field[];
+            lhsChildren.forEach((child: arrow15.Field, idx) => {
               compareFields(child, rhs.type.children[idx]);
             });
           }

nodejs/__test__/connection.test.ts

@@ -110,7 +110,10 @@ describe("given a connection", () => {
     let table = await db.createTable("test", data, { useLegacyFormat: true });
 
     const isV2 = async (table: Table) => {
-      const data = await table.query().toArrow({ maxBatchLength: 100000 });
+      const data = await table
+        .query()
+        .limit(10000)
+        .toArrow({ maxBatchLength: 100000 });
       console.log(data.batches.length);
       return data.batches.length < 5;
     };

nodejs/__test__/registry.test.ts

@@ -12,11 +12,10 @@ import * as apiArrow from "apache-arrow";
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-import * as arrow13 from "apache-arrow-13";
-import * as arrow14 from "apache-arrow-14";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
+import * as arrow18 from "apache-arrow-18";
 
 import * as tmp from "tmp";
 
@@ -24,154 +23,144 @@ import { connect } from "../lancedb";
 import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
 import { getRegistry, register } from "../lancedb/embedding/registry";
 
-describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
-  "LanceSchema",
-  (arrow) => {
-    test("should preserve input order", async () => {
-      const schema = LanceSchema({
-        id: new arrow.Int32(),
-        text: new arrow.Utf8(),
-        vector: new arrow.Float32(),
-      });
-      expect(schema.fields.map((x) => x.name)).toEqual([
-        "id",
-        "text",
-        "vector",
-      ]);
+describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
+  test("should preserve input order", async () => {
+    const schema = LanceSchema({
+      id: new arrow.Int32(),
+      text: new arrow.Utf8(),
+      vector: new arrow.Float32(),
     });
-  },
-);
+    expect(schema.fields.map((x) => x.name)).toEqual(["id", "text", "vector"]);
+  });
+});
 
-describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
-  "Registry",
-  (arrow) => {
-    let tmpDir: tmp.DirResult;
-    beforeEach(() => {
-      tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    });
+describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
 
-    afterEach(() => {
-      tmpDir.removeCallback();
-      getRegistry().reset();
-    });
+  afterEach(() => {
+    tmpDir.removeCallback();
+    getRegistry().reset();
+  });
 
-    it("should register a new item to the registry", async () => {
-      @register("mock-embedding")
-      class MockEmbeddingFunction extends EmbeddingFunction<string> {
-        toJSON(): object {
-          return {
-            someText: "hello",
-          };
-        }
-        constructor() {
-          super();
-        }
-        ndims() {
-          return 3;
-        }
-        embeddingDataType() {
-          return new arrow.Float32() as apiArrow.Float;
-        }
-        async computeSourceEmbeddings(data: string[]) {
-          return data.map(() => [1, 2, 3]);
-        }
+  it("should register a new item to the registry", async () => {
+    @register("mock-embedding")
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {
+          someText: "hello",
+        };
       }
-
-      const func = getRegistry()
-        .get<MockEmbeddingFunction>("mock-embedding")!
-        .create();
-
-      const schema = LanceSchema({
-        id: new arrow.Int32(),
-        text: func.sourceField(new arrow.Utf8() as apiArrow.DataType),
-        vector: func.vectorField(),
-      });
-
-      const db = await connect(tmpDir.name);
-      const table = await db.createTable(
-        "test",
-        [
-          { id: 1, text: "hello" },
-          { id: 2, text: "world" },
-        ],
-        { schema },
-      );
-      const expected = [
-        [1, 2, 3],
-        [1, 2, 3],
-      ];
-      const actual = await table.query().toArrow();
-      const vectors = actual.getChild("vector")!.toArray();
-      expect(JSON.parse(JSON.stringify(vectors))).toEqual(
-        JSON.parse(JSON.stringify(expected)),
-      );
-    });
-    test("should error if registering with the same name", async () => {
-      class MockEmbeddingFunction extends EmbeddingFunction<string> {
-        toJSON(): object {
-          return {
-            someText: "hello",
-          };
-        }
-        constructor() {
-          super();
-        }
-        ndims() {
-          return 3;
-        }
-        embeddingDataType() {
-          return new arrow.Float32() as apiArrow.Float;
-        }
-        async computeSourceEmbeddings(data: string[]) {
-          return data.map(() => [1, 2, 3]);
-        }
+      constructor() {
+        super();
       }
-      register("mock-embedding")(MockEmbeddingFunction);
-      expect(() => register("mock-embedding")(MockEmbeddingFunction)).toThrow(
-        'Embedding function with alias "mock-embedding" already exists',
-      );
-    });
-    test("schema should contain correct metadata", async () => {
-      class MockEmbeddingFunction extends EmbeddingFunction<string> {
-        toJSON(): object {
-          return {
-            someText: "hello",
-          };
-        }
-        constructor() {
-          super();
-        }
-        ndims() {
-          return 3;
-        }
-        embeddingDataType() {
-          return new arrow.Float32() as apiArrow.Float;
-        }
-        async computeSourceEmbeddings(data: string[]) {
-          return data.map(() => [1, 2, 3]);
-        }
+      ndims() {
+        return 3;
       }
-      const func = new MockEmbeddingFunction();
+      embeddingDataType() {
+        return new arrow.Float32() as apiArrow.Float;
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
 
-      const schema = LanceSchema({
-        id: new arrow.Int32(),
-        text: func.sourceField(new arrow.Utf8() as apiArrow.DataType),
-        vector: func.vectorField(),
-      });
-      const expectedMetadata = new Map<string, string>([
-        [
-          "embedding_functions",
-          JSON.stringify([
-            {
-              sourceColumn: "text",
-              vectorColumn: "vector",
-              name: "MockEmbeddingFunction",
-              model: { someText: "hello" },
-            },
-          ]),
-        ],
-      ]);
-      expect(schema.metadata).toEqual(expectedMetadata);
+    const func = getRegistry()
+      .get<MockEmbeddingFunction>("mock-embedding")!
+      .create();
+
+    const schema = LanceSchema({
+      id: new arrow.Int32(),
+      text: func.sourceField(new arrow.Utf8() as apiArrow.DataType),
+      vector: func.vectorField(),
     });
-  },
-);
+
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable(
+      "test",
+      [
+        { id: 1, text: "hello" },
+        { id: 2, text: "world" },
+      ],
+      { schema },
+    );
+    const expected = [
+      [1, 2, 3],
+      [1, 2, 3],
+    ];
+    const actual = await table.query().toArrow();
+    const vectors = actual.getChild("vector")!.toArray();
+    expect(JSON.parse(JSON.stringify(vectors))).toEqual(
+      JSON.parse(JSON.stringify(expected)),
+    );
+  });
+  test("should error if registering with the same name", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {
+          someText: "hello",
+        };
+      }
+      constructor() {
+        super();
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType() {
+        return new arrow.Float32() as apiArrow.Float;
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    register("mock-embedding")(MockEmbeddingFunction);
+    expect(() => register("mock-embedding")(MockEmbeddingFunction)).toThrow(
+      'Embedding function with alias "mock-embedding" already exists',
+    );
+  });
+  test("schema should contain correct metadata", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {
+          someText: "hello",
+        };
+      }
+      constructor() {
+        super();
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType() {
+        return new arrow.Float32() as apiArrow.Float;
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    const func = new MockEmbeddingFunction();
+
+    const schema = LanceSchema({
+      id: new arrow.Int32(),
+      text: func.sourceField(new arrow.Utf8() as apiArrow.DataType),
+      vector: func.vectorField(),
+    });
+    const expectedMetadata = new Map<string, string>([
+      [
+        "embedding_functions",
+        JSON.stringify([
+          {
+            sourceColumn: "text",
+            vectorColumn: "vector",
+            name: "MockEmbeddingFunction",
+            model: { someText: "hello" },
+          },
+        ]),
+      ],
+    ]);
+    expect(schema.metadata).toEqual(expectedMetadata);
+  });
+});

nodejs/__test__/table.test.ts

@@ -16,11 +16,10 @@ import * as fs from "fs";
 import * as path from "path";
 import * as tmp from "tmp";
 
-import * as arrow13 from "apache-arrow-13";
-import * as arrow14 from "apache-arrow-14";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
+import * as arrow18 from "apache-arrow-18";
 
 import { Table, connect } from "../lancedb";
 import {
@@ -44,7 +43,7 @@ import {
 } from "../lancedb/embedding";
 import { Index } from "../lancedb/indices";
 
-describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
+describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Given a table",
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   (arrow: any) => {
@@ -52,11 +51,10 @@ describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
     let table: Table;
 
     const schema:
-      | import("apache-arrow-13").Schema
-      | import("apache-arrow-14").Schema
       | import("apache-arrow-15").Schema
       | import("apache-arrow-16").Schema
-      | import("apache-arrow-17").Schema = new arrow.Schema([
+      | import("apache-arrow-17").Schema
+      | import("apache-arrow-18").Schema = new arrow.Schema([
       new arrow.Field("id", new arrow.Float64(), true),
     ]);
 
@@ -569,6 +567,15 @@ describe("When creating an index", () => {
     // TODO: Verify parameters when we can load index config as part of list indices
   });
 
+  it("should be able to create 4bit IVF_PQ", async () => {
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({
+        numPartitions: 10,
+        numBits: 4,
+      }),
+    });
+  });
+
   it("should allow me to replace (or not) an existing index", async () => {
     await tbl.createIndex("id");
     // Default is replace=true
@@ -585,11 +592,11 @@ describe("When creating an index", () => {
     expect(fs.readdirSync(indexDir)).toHaveLength(1);
 
     for await (const r of tbl.query().where("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(298);
+      expect(r.numRows).toBe(10);
     }
     // should also work with 'filter' alias
     for await (const r of tbl.query().filter("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(298);
+      expect(r.numRows).toBe(10);
     }
   });
 
@@ -825,6 +832,18 @@ describe("schema evolution", function () {
       new Field("price", new Float64(), true),
     ]);
     expect(await table.schema()).toEqual(expectedSchema);
+
+    await table.alterColumns([{ path: "new_id", dataType: "int32" }]);
+    const expectedSchema2 = new Schema([
+      new Field("new_id", new Int32(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32(), true)),
+        true,
+      ),
+      new Field("price", new Float64(), true),
+    ]);
+    expect(await table.schema()).toEqual(expectedSchema2);
   });
 
   it("can drop a column from the schema", async function () {
@@ -927,7 +946,7 @@ describe("when optimizing a dataset", () => {
   });
 });
 
-describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
+describe.each([arrow15, arrow16, arrow17, arrow18])(
   "when optimizing a dataset",
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   (arrow: any) => {

nodejs/examples/basic.test.ts

@@ -116,6 +116,26 @@ test("basic table examples", async () => {
       await tbl.add(data);
       // --8<-- [end:add_data]
     }
+
+    {
+      // --8<-- [start:add_columns]
+      await tbl.addColumns([{ name: "double_price", valueSql: "price * 2" }]);
+      // --8<-- [end:add_columns]
+      // --8<-- [start:alter_columns]
+      await tbl.alterColumns([
+        {
+          path: "double_price",
+          rename: "dbl_price",
+          dataType: "float",
+          nullable: true,
+        },
+      ]);
+      // --8<-- [end:alter_columns]
+      // --8<-- [start:drop_columns]
+      await tbl.dropColumns(["dbl_price"]);
+      // --8<-- [end:drop_columns]
+    }
+
     {
       // --8<-- [start:vector_search]
       const res = await tbl.search([100, 100]).limit(2).toArray();

nodejs/lancedb/indices.ts

@@ -47,6 +47,16 @@ export interface IvfPqOptions {
    */
   numSubVectors?: number;
 
+  /**
+   * Number of bits per sub-vector.
+   *
+   * This value controls how much each subvector is compressed.  The more bits the more
+   * accurate the index will be but the slower search.  The default is 8 bits.
+   *
+   * The number of bits must be 4 or 8.
+   */
+  numBits?: number;
+
   /**
    * Distance type to use to build the index.
    *

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.13.0",
+  "version": "0.14.1-beta.2",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.13.0",
+	"version": "0.14.1-beta.2",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.13.0",
+      "version": "0.14.0",
       "cpu": [
         "x64",
         "arm64"
@@ -31,11 +31,10 @@
         "@types/jest": "^29.1.2",
         "@types/node": "^22.7.4",
         "@types/tmp": "^0.2.6",
-        "apache-arrow-13": "npm:apache-arrow@13.0.0",
-        "apache-arrow-14": "npm:apache-arrow@14.0.0",
         "apache-arrow-15": "npm:apache-arrow@15.0.0",
         "apache-arrow-16": "npm:apache-arrow@16.0.0",
         "apache-arrow-17": "npm:apache-arrow@17.0.0",
+        "apache-arrow-18": "npm:apache-arrow@18.0.0",
         "eslint": "^8.57.0",
         "jest": "^29.7.0",
         "shx": "^0.3.4",
@@ -54,7 +53,7 @@
         "openai": "^4.29.2"
       },
       "peerDependencies": {
-        "apache-arrow": ">=13.0.0 <=17.0.0"
+        "apache-arrow": ">=15.0.0 <=18.1.0"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -5146,12 +5145,6 @@
       "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
       "devOptional": true
     },
-    "node_modules/@types/pad-left": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
-      "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==",
-      "dev": true
-    },
     "node_modules/@types/semver": {
       "version": "7.5.6",
       "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz",
@@ -5341,74 +5334,6 @@
         "arrow2csv": "bin/arrow2csv.cjs"
       }
     },
-    "node_modules/apache-arrow-13": {
-      "name": "apache-arrow",
-      "version": "13.0.0",
-      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-13.0.0.tgz",
-      "integrity": "sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==",
-      "dev": true,
-      "dependencies": {
-        "@types/command-line-args": "5.2.0",
-        "@types/command-line-usage": "5.0.2",
-        "@types/node": "20.3.0",
-        "@types/pad-left": "2.1.1",
-        "command-line-args": "5.2.1",
-        "command-line-usage": "7.0.1",
-        "flatbuffers": "23.5.26",
-        "json-bignum": "^0.0.3",
-        "pad-left": "^2.1.0",
-        "tslib": "^2.5.3"
-      },
-      "bin": {
-        "arrow2csv": "bin/arrow2csv.js"
-      }
-    },
-    "node_modules/apache-arrow-13/node_modules/@types/command-line-args": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
-      "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==",
-      "dev": true
-    },
-    "node_modules/apache-arrow-13/node_modules/@types/node": {
-      "version": "20.3.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
-      "dev": true
-    },
-    "node_modules/apache-arrow-14": {
-      "name": "apache-arrow",
-      "version": "14.0.0",
-      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.0.tgz",
-      "integrity": "sha512-9cKE24YxkaqAZWJddrVnjUJMLwq6CokOjK+AHpm145rMJNsBZXQkzqouemQyEX0+/iHYRnGym6X6ZgNcHHrcWA==",
-      "dev": true,
-      "dependencies": {
-        "@types/command-line-args": "5.2.0",
-        "@types/command-line-usage": "5.0.2",
-        "@types/node": "20.3.0",
-        "@types/pad-left": "2.1.1",
-        "command-line-args": "5.2.1",
-        "command-line-usage": "7.0.1",
-        "flatbuffers": "23.5.26",
-        "json-bignum": "^0.0.3",
-        "pad-left": "^2.1.0",
-        "tslib": "^2.5.3"
-      },
-      "bin": {
-        "arrow2csv": "bin/arrow2csv.js"
-      }
-    },
-    "node_modules/apache-arrow-14/node_modules/@types/command-line-args": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
-      "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==",
-      "dev": true
-    },
-    "node_modules/apache-arrow-14/node_modules/@types/node": {
-      "version": "20.3.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
-      "dev": true
-    },
     "node_modules/apache-arrow-15": {
       "name": "apache-arrow",
       "version": "15.0.0",
@@ -5529,6 +5454,54 @@
       "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
       "dev": true
     },
+    "node_modules/apache-arrow-18": {
+      "name": "apache-arrow",
+      "version": "18.0.0",
+      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-18.0.0.tgz",
+      "integrity": "sha512-gFlPaqN9osetbB83zC29AbbZqGiCuFH1vyyPseJ+B7SIbfBtESV62mMT/CkiIt77W6ykC/nTWFzTXFs0Uldg4g==",
+      "dev": true,
+      "dependencies": {
+        "@swc/helpers": "^0.5.11",
+        "@types/command-line-args": "^5.2.3",
+        "@types/command-line-usage": "^5.0.4",
+        "@types/node": "^20.13.0",
+        "command-line-args": "^5.2.1",
+        "command-line-usage": "^7.0.1",
+        "flatbuffers": "^24.3.25",
+        "json-bignum": "^0.0.3",
+        "tslib": "^2.6.2"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/apache-arrow-18/node_modules/@types/command-line-usage": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz",
+      "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==",
+      "dev": true
+    },
+    "node_modules/apache-arrow-18/node_modules/@types/node": {
+      "version": "20.17.9",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.9.tgz",
+      "integrity": "sha512-0JOXkRyLanfGPE2QRCwgxhzlBAvaRdCNMcvbd7jFfpmD4eEXll7LRwy5ymJmyeZqk7Nh7eD2LeUyQ68BbndmXw==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
+    "node_modules/apache-arrow-18/node_modules/flatbuffers": {
+      "version": "24.3.25",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.3.25.tgz",
+      "integrity": "sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ==",
+      "dev": true
+    },
+    "node_modules/apache-arrow-18/node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true
+    },
     "node_modules/apache-arrow/node_modules/@types/node": {
       "version": "20.16.10",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.10.tgz",
@@ -8533,18 +8506,6 @@
       "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
       "optional": true
     },
-    "node_modules/pad-left": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
-      "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==",
-      "dev": true,
-      "dependencies": {
-        "repeat-string": "^1.5.4"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/parent-module": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -8885,15 +8846,6 @@
       "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz",
       "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="
     },
-    "node_modules/repeat-string": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
-      "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10"
-      }
-    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",

nodejs/package.json

@@ -10,7 +10,8 @@
     "vector database",
     "ann"
   ],
-  "version": "0.13.0",
+  "private": false,
+  "version": "0.14.1-beta.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -30,7 +31,8 @@
         "aarch64-unknown-linux-gnu",
         "x86_64-unknown-linux-musl",
         "aarch64-unknown-linux-musl",
-        "x86_64-pc-windows-msvc"
+        "x86_64-pc-windows-msvc",
+        "aarch64-pc-windows-msvc"
       ]
     }
   },
@@ -46,11 +48,10 @@
     "@types/jest": "^29.1.2",
     "@types/node": "^22.7.4",
     "@types/tmp": "^0.2.6",
-    "apache-arrow-13": "npm:apache-arrow@13.0.0",
-    "apache-arrow-14": "npm:apache-arrow@14.0.0",
     "apache-arrow-15": "npm:apache-arrow@15.0.0",
     "apache-arrow-16": "npm:apache-arrow@16.0.0",
     "apache-arrow-17": "npm:apache-arrow@17.0.0",
+    "apache-arrow-18": "npm:apache-arrow@18.0.0",
     "eslint": "^8.57.0",
     "jest": "^29.7.0",
     "shx": "^0.3.4",
@@ -77,6 +78,7 @@
     "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
     "docs": "typedoc --plugin typedoc-plugin-markdown --out ../docs/src/js lancedb/index.ts",
+    "postdocs": "node typedoc_post_process.js",
     "lint": "biome check . && biome format .",
     "lint-fix": "biome check --write . && biome format --write .",
     "prepublishOnly": "napi prepublish -t npm",
@@ -93,6 +95,6 @@
     "openai": "^4.29.2"
   },
   "peerDependencies": {
-    "apache-arrow": ">=13.0.0 <=17.0.0"
+    "apache-arrow": ">=15.0.0 <=18.1.0"
   }
 }

nodejs/src/index.rs

@@ -45,6 +45,7 @@ impl Index {
         distance_type: Option<String>,
         num_partitions: Option<u32>,
         num_sub_vectors: Option<u32>,
+        num_bits: Option<u32>,
         max_iterations: Option<u32>,
         sample_rate: Option<u32>,
     ) -> napi::Result<Self> {
@@ -59,6 +60,9 @@ impl Index {
         if let Some(num_sub_vectors) = num_sub_vectors {
             ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
         }
+        if let Some(num_bits) = num_bits {
+            ivf_pq_builder = ivf_pq_builder.num_bits(num_bits);
+        }
         if let Some(max_iterations) = max_iterations {
             ivf_pq_builder = ivf_pq_builder.max_iterations(max_iterations);
         }

nodejs/src/table.rs

@@ -178,16 +178,20 @@ impl Table {
     #[napi(catch_unwind)]
     pub async fn alter_columns(&self, alterations: Vec<ColumnAlteration>) -> napi::Result<()> {
         for alteration in &alterations {
-            if alteration.rename.is_none() && alteration.nullable.is_none() {
+            if alteration.rename.is_none()
+                && alteration.nullable.is_none()
+                && alteration.data_type.is_none()
+            {
                 return Err(napi::Error::from_reason(
-                    "Alteration must have a 'rename' or 'nullable' field.",
+                    "Alteration must have a 'rename', 'dataType', or 'nullable' field.",
                 ));
             }
         }
         let alterations = alterations
             .into_iter()
-            .map(LanceColumnAlteration::from)
-            .collect::<Vec<_>>();
+            .map(LanceColumnAlteration::try_from)
+            .collect::<std::result::Result<Vec<_>, String>>()
+            .map_err(napi::Error::from_reason)?;
 
         self.inner_ref()?
             .alter_columns(&alterations)
@@ -433,24 +437,43 @@ pub struct ColumnAlteration {
     /// The new name of the column. If not provided then the name will not be changed.
     /// This must be distinct from the names of all other columns in the table.
     pub rename: Option<String>,
+    /// A new data type for the column. If not provided then the data type will not be changed.
+    /// Changing data types is limited to casting to the same general type. For example, these
+    /// changes are valid:
+    /// * `int32` -> `int64` (integers)
+    /// * `double` -> `float` (floats)
+    /// * `string` -> `large_string` (strings)
+    /// But these changes are not:
+    /// * `int32` -> `double` (mix integers and floats)
+    /// * `string` -> `int32` (mix strings and integers)
+    pub data_type: Option<String>,
     /// Set the new nullability. Note that a nullable column cannot be made non-nullable.
     pub nullable: Option<bool>,
 }
 
-impl From<ColumnAlteration> for LanceColumnAlteration {
-    fn from(js: ColumnAlteration) -> Self {
+impl TryFrom<ColumnAlteration> for LanceColumnAlteration {
+    type Error = String;
+    fn try_from(js: ColumnAlteration) -> std::result::Result<Self, Self::Error> {
         let ColumnAlteration {
             path,
             rename,
             nullable,
+            data_type,
         } = js;
-        Self {
+        let data_type = if let Some(data_type) = data_type {
+            Some(
+                lancedb::utils::string_to_datatype(&data_type)
+                    .ok_or_else(|| format!("Invalid data type: {}", data_type))?,
+            )
+        } else {
+            None
+        };
+        Ok(Self {
             path,
             rename,
             nullable,
-            // TODO: wire up this field
-            data_type: None,
-        }
+            data_type,
+        })
     }
 }
 

nodejs/typedoc.json

@@ -8,5 +8,6 @@
     "lancedb/native.d.ts:Table"
   ],
   "useHTMLEncodedBrackets": true,
+  "useCodeBlocks": true,
   "disableSources": true
 }

nodejs/typedoc_post_process.js

@@ -0,0 +1,63 @@
+const fs = require("fs");
+const path = require("path");
+
+// Read all files in the directory
+function processDirectory(directoryPath) {
+  fs.readdir(directoryPath, { withFileTypes: true }, (err, files) => {
+    if (err) {
+      return console.error("Unable to scan directory: " + err);
+    }
+
+    files.forEach((file) => {
+      const filePath = path.join(directoryPath, file.name);
+
+      if (file.isDirectory()) {
+        // Recursively process subdirectory
+        processDirectory(filePath);
+      } else if (file.isFile()) {
+        // Read each file
+        fs.readFile(filePath, "utf8", (err, data) => {
+          if (err) {
+            return console.error("Unable to read file: " + err);
+          }
+
+          // Process the file content
+          const processedData = processContents(data);
+
+          // Write the processed content back to the file
+          fs.writeFile(filePath, processedData, "utf8", (err) => {
+            if (err) {
+              return console.error("Unable to write file: " + err);
+            }
+            console.log(`Processed file: ${filePath}`);
+          });
+        });
+      }
+    });
+  });
+}
+
+function processContents(contents) {
+  // This changes the parameters section to put the parameter description on
+  // the same line as the bullet with the parameter name and type.
+  return contents.replace(/(## Parameters[\s\S]*?)(?=##|$)/g, (match) => {
+    let lines = match
+      .split("\n")
+      .map((line) => line.trim())
+
+      .filter((line) => line !== "")
+      .map((line) => {
+        if (line.startsWith("##")) {
+          return line;
+        } else if (line.startsWith("•")) {
+          return "\n*" + line.substring(1);
+        } else {
+          return "    " + line;
+        }
+      });
+    return lines.join("\n") + "\n\n";
+  });
+}
+
+// Start processing from the root directory
+processDirectory("../docs/src/js");

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.1-beta.0"
+current_version = "0.17.1-beta.3"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.16.1-beta.0"
+version = "0.17.1-beta.3"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -14,29 +14,29 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "52.1", features = ["pyarrow"] }
+arrow = { version = "53.2", features = ["pyarrow"] }
 lancedb = { path = "../rust/lancedb", default-features = false }
 env_logger.workspace = true
-pyo3 = { version = "0.21", features = ["extension-module", "abi3-py38", "gil-refs"] }
-# Using this fork for now: https://github.com/awestlake87/pyo3-asyncio/issues/119
-# pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
-pyo3-asyncio-0-21 = { version = "0.21.0", features = ["attributes", "tokio-runtime"] }
-
+pyo3 = { version = "0.22.2", features = [
+    "extension-module",
+    "abi3-py39",
+    "gil-refs"
+] }
+pyo3-async-runtimes = { version = "0.22", features = ["attributes", "tokio-runtime"] }
 pin-project = "1.1.5"
 futures.workspace = true
-tokio = { version = "1.36.0", features = ["sync"] }
+tokio = { version = "1.40", features = ["sync"] }
 
 [build-dependencies]
 pyo3-build-config = { version = "0.20.3", features = [
     "extension-module",
-    "abi3-py38",
+    "abi3-py39",
 ] }
 
 [features]
 default = ["default-tls", "remote"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]
-
 # TLS
 default-tls = ["lancedb/default-tls"]
 native-tls = ["lancedb/native-tls"]

python/pyproject.toml

@@ -3,8 +3,7 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
     "deprecation",
-    "nest-asyncio~=1.0",
-    "pylance==0.20.0b2",
+    "pylance==0.21.0b2",
     "tqdm>=4.27.0",
     "pydantic>=1.10",
     "packaging",
@@ -31,7 +30,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",

python/python/lancedb/__init__.py

@@ -36,6 +36,7 @@ def connect(
     read_consistency_interval: Optional[timedelta] = None,
     request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
     client_config: Union[ClientConfig, Dict[str, Any], None] = None,
+    storage_options: Optional[Dict[str, str]] = None,
     **kwargs: Any,
 ) -> DBConnection:
     """Connect to a LanceDB database.
@@ -67,6 +68,9 @@ def connect(
         Configuration options for the LanceDB Cloud HTTP client. If a dict, then
         the keys are the attributes of the ClientConfig class. If None, then the
         default configuration is used.
+    storage_options: dict, optional
+        Additional options for the storage backend. See available options at
+        https://lancedb.github.io/lancedb/guides/storage/
 
     Examples
     --------
@@ -106,12 +110,17 @@ def connect(
             # TODO: remove this (deprecation warning downstream)
             request_thread_pool=request_thread_pool,
             client_config=client_config,
+            storage_options=storage_options,
             **kwargs,
         )
 
     if kwargs:
         raise ValueError(f"Unknown keyword arguments: {kwargs}")
-    return LanceDBConnection(uri, read_consistency_interval=read_consistency_interval)
+    return LanceDBConnection(
+        uri,
+        read_consistency_interval=read_consistency_interval,
+        storage_options=storage_options,
+    )
 
 
 async def connect_async(

python/python/lancedb/_lancedb.pyi

@@ -79,9 +79,21 @@ class Query:
     def limit(self, limit: int): ...
     def offset(self, offset: int): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
-    def nearest_to_text(self, query: dict) -> Query: ...
+    def nearest_to_text(self, query: dict) -> FTSQuery: ...
     async def execute(self, max_batch_legnth: Optional[int]) -> RecordBatchStream: ...
 
+class FTSQuery:
+    def where(self, filter: str): ...
+    def select(self, columns: List[str]): ...
+    def limit(self, limit: int): ...
+    def offset(self, offset: int): ...
+    def fast_search(self): ...
+    def with_row_id(self): ...
+    def postfilter(self): ...
+    def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
+    async def execute(self, max_batch_length: Optional[int]) -> RecordBatchStream: ...
+    async def explain_plan(self) -> str: ...
+
 class VectorQuery:
     async def execute(self) -> RecordBatchStream: ...
     def where(self, filter: str): ...
@@ -95,6 +107,24 @@ class VectorQuery:
     def refine_factor(self, refine_factor: int): ...
     def nprobes(self, nprobes: int): ...
     def bypass_vector_index(self): ...
+    def nearest_to_text(self, query: dict) -> HybridQuery: ...
+
+class HybridQuery:
+    def where(self, filter: str): ...
+    def select(self, columns: List[str]): ...
+    def limit(self, limit: int): ...
+    def offset(self, offset: int): ...
+    def fast_search(self): ...
+    def with_row_id(self): ...
+    def postfilter(self): ...
+    def distance_type(self, distance_type: str): ...
+    def refine_factor(self, refine_factor: int): ...
+    def nprobes(self, nprobes: int): ...
+    def bypass_vector_index(self): ...
+    def to_vector_query(self) -> VectorQuery: ...
+    def to_fts_query(self) -> FTSQuery: ...
+    def get_limit(self) -> int: ...
+    def get_with_row_id(self) -> bool: ...
 
 class CompactionStats:
     fragments_removed: int

python/python/lancedb/background_loop.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import asyncio
+import threading
+
+
+class BackgroundEventLoop:
+    """
+    A background event loop that can run futures.
+
+    Used to bridge sync and async code, without messing with users event loops.
+    """
+
+    def __init__(self):
+        self.loop = asyncio.new_event_loop()
+        self.thread = threading.Thread(
+            target=self.loop.run_forever,
+            name="LanceDBBackgroundEventLoop",
+            daemon=True,
+        )
+        self.thread.start()
+
+    def run(self, future):
+        return asyncio.run_coroutine_threadsafe(future, self.loop).result()

python/python/lancedb/db.py

@@ -13,34 +13,29 @@
 
 from __future__ import annotations
 
-import asyncio
-import os
 from abc import abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Union
 
-import pyarrow as pa
 from overrides import EnforceOverrides, override
-from pyarrow import fs
 
-from lancedb.common import data_to_reader, validate_schema
+from lancedb.common import data_to_reader, sanitize_uri, validate_schema
+from lancedb.background_loop import BackgroundEventLoop
 
 from ._lancedb import connect as lancedb_connect
 from .table import (
     AsyncTable,
     LanceTable,
     Table,
-    _table_path,
     sanitize_create_table,
 )
 from .util import (
-    fs_from_uri,
-    get_uri_location,
     get_uri_scheme,
     validate_table_name,
 )
 
 if TYPE_CHECKING:
+    import pyarrow as pa
     from .pydantic import LanceModel
     from datetime import timedelta
 
@@ -48,6 +43,8 @@ if TYPE_CHECKING:
     from .common import DATA, URI
     from .embeddings import EmbeddingFunctionConfig
 
+LOOP = BackgroundEventLoop()
+
 
 class DBConnection(EnforceOverrides):
     """An active LanceDB connection interface."""
@@ -180,6 +177,7 @@ class DBConnection(EnforceOverrides):
         control over how data is saved, either provide the PyArrow schema to
         convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
 
+        >>> import pyarrow as pa
         >>> custom_schema = pa.schema([
         ...   pa.field("vector", pa.list_(pa.float32(), 2)),
         ...   pa.field("lat", pa.float32()),
@@ -327,7 +325,11 @@ class LanceDBConnection(DBConnection):
     """
 
     def __init__(
-        self, uri: URI, *, read_consistency_interval: Optional[timedelta] = None
+        self,
+        uri: URI,
+        *,
+        read_consistency_interval: Optional[timedelta] = None,
+        storage_options: Optional[Dict[str, str]] = None,
     ):
         if not isinstance(uri, Path):
             scheme = get_uri_scheme(uri)
@@ -338,9 +340,27 @@ class LanceDBConnection(DBConnection):
             uri = uri.expanduser().absolute()
             Path(uri).mkdir(parents=True, exist_ok=True)
         self._uri = str(uri)
-
         self._entered = False
         self.read_consistency_interval = read_consistency_interval
+        self.storage_options = storage_options
+
+        if read_consistency_interval is not None:
+            read_consistency_interval_secs = read_consistency_interval.total_seconds()
+        else:
+            read_consistency_interval_secs = None
+
+        async def do_connect():
+            return await lancedb_connect(
+                sanitize_uri(uri),
+                None,
+                None,
+                None,
+                read_consistency_interval_secs,
+                None,
+                storage_options,
+            )
+
+        self._conn = AsyncConnection(LOOP.run(do_connect()))
 
     def __repr__(self) -> str:
         val = f"{self.__class__.__name__}({self._uri}"
@@ -364,32 +384,7 @@ class LanceDBConnection(DBConnection):
         Iterator of str.
             A list of table names.
         """
-        try:
-            asyncio.get_running_loop()
-            # User application is async.  Soon we will just tell them to use the
-            # async version.  Until then fallback to the old sync implementation.
-            try:
-                filesystem = fs_from_uri(self.uri)[0]
-            except pa.ArrowInvalid:
-                raise NotImplementedError("Unsupported scheme: " + self.uri)
-
-            try:
-                loc = get_uri_location(self.uri)
-                paths = filesystem.get_file_info(fs.FileSelector(loc))
-            except FileNotFoundError:
-                # It is ok if the file does not exist since it will be created
-                paths = []
-            tables = [
-                os.path.splitext(file_info.base_name)[0]
-                for file_info in paths
-                if file_info.extension == "lance"
-            ]
-            tables.sort()
-            return tables
-        except RuntimeError:
-            # User application is sync.  It is safe to use the async implementation
-            # under the hood.
-            return asyncio.run(self._async_get_table_names(page_token, limit))
+        return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -461,19 +456,16 @@ class LanceDBConnection(DBConnection):
             If True, ignore if the table does not exist.
         """
         try:
-            table_uri = _table_path(self.uri, name)
-            filesystem, path = fs_from_uri(table_uri)
-            filesystem.delete_dir(path)
-        except FileNotFoundError:
+            LOOP.run(self._conn.drop_table(name))
+        except ValueError as e:
             if not ignore_missing:
-                raise
+                raise e
+            if f"Table '{name}' was not found" not in str(e):
+                raise e
 
     @override
     def drop_database(self):
-        dummy_table_uri = _table_path(self.uri, "dummy")
-        uri = dummy_table_uri.removesuffix("dummy.lance")
-        filesystem, path = fs_from_uri(uri)
-        filesystem.delete_dir(path)
+        LOOP.run(self._conn.drop_database())
 
 
 class AsyncConnection(object):
@@ -689,6 +681,7 @@ class AsyncConnection(object):
         control over how data is saved, either provide the PyArrow schema to
         convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
 
+        >>> import pyarrow as pa
         >>> custom_schema = pa.schema([
         ...   pa.field("vector", pa.list_(pa.float32(), 2)),
         ...   pa.field("lat", pa.float32()),

python/python/lancedb/embeddings/openai.py

@@ -48,6 +48,9 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
     organization: Optional[str] = None
     api_key: Optional[str] = None
 
+    # Set true to use Azure OpenAI API
+    use_azure: bool = False
+
     def ndims(self):
         return self._ndims
 
@@ -83,25 +86,33 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
         """
         openai = attempt_import_or_raise("openai")
 
+        valid_texts = []
+        valid_indices = []
+        for idx, text in enumerate(texts):
+            if text:
+                valid_texts.append(text)
+                valid_indices.append(idx)
+
         # TODO retry, rate limit, token limit
         try:
-            if self.name == "text-embedding-ada-002":
-                rs = self._openai_client.embeddings.create(input=texts, model=self.name)
-            else:
-                kwargs = {
-                    "input": texts,
-                    "model": self.name,
-                }
-                if self.dim:
-                    kwargs["dimensions"] = self.dim
-                rs = self._openai_client.embeddings.create(**kwargs)
+            kwargs = {
+                "input": valid_texts,
+                "model": self.name,
+            }
+            if self.name != "text-embedding-ada-002":
+                kwargs["dimensions"] = self.dim
+
+            rs = self._openai_client.embeddings.create(**kwargs)
+            valid_embeddings = {
+                idx: v.embedding for v, idx in zip(rs.data, valid_indices)
+            }
         except openai.BadRequestError:
             logging.exception("Bad request: %s", texts)
             return [None] * len(texts)
         except Exception:
             logging.exception("OpenAI embeddings error")
             raise
-        return [v.embedding for v in rs.data]
+        return [valid_embeddings.get(idx, None) for idx in range(len(texts))]
 
     @cached_property
     def _openai_client(self):
@@ -115,4 +126,8 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
             kwargs["organization"] = self.organization
         if self.api_key:
             kwargs["api_key"] = self.api_key
-        return openai.OpenAI(**kwargs)
+
+        if self.use_azure:
+            return openai.AzureOpenAI(**kwargs)
+        else:
+            return openai.OpenAI(**kwargs)

python/python/lancedb/embeddings/voyageai.py

@@ -12,18 +12,22 @@
 #  limitations under the License.
 
 import os
-from typing import ClassVar, List, Union
+from typing import ClassVar, TYPE_CHECKING, List, Union
 
 import numpy as np
+import pyarrow as pa
 
 from ..util import attempt_import_or_raise
-from .base import TextEmbeddingFunction
+from .base import EmbeddingFunction
 from .registry import register
-from .utils import api_key_not_found_help, TEXT
+from .utils import api_key_not_found_help, IMAGES
+
+if TYPE_CHECKING:
+    import PIL
 
 
 @register("voyageai")
-class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
+class VoyageAIEmbeddingFunction(EmbeddingFunction):
     """
     An embedding function that uses the VoyageAI API
 
@@ -36,6 +40,7 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
 
             * voyage-3
             * voyage-3-lite
+            * voyage-multimodal-3
             * voyage-finance-2
             * voyage-multilingual-2
             * voyage-law-2
@@ -54,7 +59,7 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
         .create(name="voyage-3")
 
     class TextModel(LanceModel):
-        text: str = voyageai.SourceField()
+        data: str = voyageai.SourceField()
         vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
 
     data = [ { "text": "hello world" },
@@ -77,6 +82,7 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
             return 1536
         elif self.name in [
             "voyage-3",
+            "voyage-multimodal-3",
             "voyage-finance-2",
             "voyage-multilingual-2",
             "voyage-law-2",
@@ -85,19 +91,19 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
         else:
             raise ValueError(f"Model {self.name} not supported")
 
-    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
-        return self.compute_source_embeddings(query, input_type="query")
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
 
-    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
-        texts = self.sanitize_input(texts)
-        input_type = (
-            kwargs.get("input_type") or "document"
-        )  # assume source input type if not passed by `compute_query_embeddings`
-        return self.generate_embeddings(texts, input_type=input_type)
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray], *args, **kwargs
-    ) -> List[np.array]:
+    def generate_text_embeddings(self, text: str, **kwargs) -> np.ndarray:
         """
         Get the embeddings for the given texts
 
@@ -109,15 +115,55 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
 
         truncation: Optional[bool]
         """
-        VoyageAIEmbeddingFunction._init_client()
-        rs = VoyageAIEmbeddingFunction.client.embed(
-            texts=texts, model=self.name, **kwargs
-        )
+        if self.name in ["voyage-multimodal-3"]:
+            rs = VoyageAIEmbeddingFunction._get_client().multimodal_embed(
+                inputs=[[text]], model=self.name, **kwargs
+            )
+        else:
+            rs = VoyageAIEmbeddingFunction._get_client().embed(
+                texts=[text], model=self.name, **kwargs
+            )
 
-        return [emb for emb in rs.embeddings]
+        return rs.embeddings[0]
+
+    def generate_image_embedding(
+        self, image: "PIL.Image.Image", **kwargs
+    ) -> np.ndarray:
+        rs = VoyageAIEmbeddingFunction._get_client().multimodal_embed(
+            inputs=[[image]], model=self.name, **kwargs
+        )
+        return rs.embeddings[0]
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query, input_type="query")]
+        else:
+            PIL = attempt_import_or_raise("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query, input_type="query")]
+            else:
+                raise TypeError("Only text PIL images supported as query")
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        images = self.sanitize_input(images)
+        return [
+            self.generate_image_embedding(img, input_type="document") for img in images
+        ]
 
     @staticmethod
-    def _init_client():
+    def _get_client():
         if VoyageAIEmbeddingFunction.client is None:
             voyageai = attempt_import_or_raise("voyageai")
             if os.environ.get("VOYAGE_API_KEY") is None:
@@ -125,3 +171,4 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
             VoyageAIEmbeddingFunction.client = voyageai.Client(
                 os.environ["VOYAGE_API_KEY"]
             )
+        return VoyageAIEmbeddingFunction.client

python/python/lancedb/index.py

@@ -110,7 +110,16 @@ class FTS:
         remove_stop_words: bool = False,
         ascii_folding: bool = False,
     ):
-        self._inner = LanceDbIndex.fts(with_position=with_position)
+        self._inner = LanceDbIndex.fts(
+            with_position=with_position,
+            base_tokenizer=base_tokenizer,
+            language=language,
+            max_token_length=max_token_length,
+            lower_case=lower_case,
+            stem=stem,
+            remove_stop_words=remove_stop_words,
+            ascii_folding=ascii_folding,
+        )
 
 
 class HnswPq:
@@ -169,6 +178,12 @@ class HnswPq:
         If the dimension is not visible by 8 then we use 1 subvector.  This is not
         ideal and will likely result in poor performance.
 
+     num_bits: int, default 8
+        Number of bits to encode each sub-vector.
+
+        This value controls how much the sub-vectors are compressed.  The more bits
+        the more accurate the index but the slower search. Only 4 and 8 are supported.
+
     max_iterations, default 50
 
         Max iterations to train kmeans.
@@ -223,6 +238,7 @@ class HnswPq:
         distance_type: Optional[str] = None,
         num_partitions: Optional[int] = None,
         num_sub_vectors: Optional[int] = None,
+        num_bits: Optional[int] = None,
         max_iterations: Optional[int] = None,
         sample_rate: Optional[int] = None,
         m: Optional[int] = None,
@@ -232,6 +248,7 @@ class HnswPq:
             distance_type=distance_type,
             num_partitions=num_partitions,
             num_sub_vectors=num_sub_vectors,
+            num_bits=num_bits,
             max_iterations=max_iterations,
             sample_rate=sample_rate,
             m=m,
@@ -378,6 +395,7 @@ class IvfPq:
         distance_type: Optional[str] = None,
         num_partitions: Optional[int] = None,
         num_sub_vectors: Optional[int] = None,
+        num_bits: Optional[int] = None,
         max_iterations: Optional[int] = None,
         sample_rate: Optional[int] = None,
     ):
@@ -440,6 +458,12 @@ class IvfPq:
 
             If the dimension is not visible by 8 then we use 1 subvector.  This is not
             ideal and will likely result in poor performance.
+        num_bits: int, default 8
+            Number of bits to encode each sub-vector.
+
+            This value controls how much the sub-vectors are compressed.  The more bits
+            the more accurate the index but the slower search.  The default is 8
+            bits.  Only 4 and 8 are supported.
         max_iterations: int, default 50
             Max iteration to train kmeans.
 
@@ -473,6 +497,7 @@ class IvfPq:
             distance_type=distance_type,
             num_partitions=num_partitions,
             num_sub_vectors=num_sub_vectors,
+            num_bits=num_bits,
             max_iterations=max_iterations,
             sample_rate=sample_rate,
         )

python/python/lancedb/integrations/pyarrow.py

@@ -0,0 +1,248 @@
+import logging
+from typing import Any, List, Optional, Tuple, Union, Literal
+
+import pyarrow as pa
+
+from ..table import Table
+
+Filter = Union[str, pa.compute.Expression]
+Keys = Union[str, List[str]]
+JoinType = Literal[
+    "left semi",
+    "right semi",
+    "left anti",
+    "right anti",
+    "inner",
+    "left outer",
+    "right outer",
+    "full outer",
+]
+
+
+class PyarrowScannerAdapter(pa.dataset.Scanner):
+    def __init__(
+        self,
+        table: Table,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        self.table = table
+        self.columns = columns
+        self.filter = filter
+        self.batch_size = batch_size
+        if batch_readahead is not None:
+            logging.debug("ignoring batch_readahead which has no lance equivalent")
+        if fragment_readahead is not None:
+            logging.debug("ignoring fragment_readahead which has no lance equivalent")
+        if fragment_scan_options is not None:
+            raise NotImplementedError("fragment_scan_options not supported")
+        if use_threads is False:
+            raise NotImplementedError("use_threads=False not supported")
+        if memory_pool is not None:
+            raise NotImplementedError("memory_pool not supported")
+
+    def count_rows(self):
+        return self.table.count_rows(self.filter)
+
+    def from_batches(self, **kwargs):
+        raise NotImplementedError
+
+    def from_dataset(self, **kwargs):
+        raise NotImplementedError
+
+    def from_fragment(self, **kwargs):
+        raise NotImplementedError
+
+    def head(self, num_rows: int):
+        return self.to_reader(limit=num_rows).read_all()
+
+    @property
+    def projected_schema(self):
+        return self.head(1).schema
+
+    def scan_batches(self):
+        return self.to_reader()
+
+    def take(self, indices: List[int]):
+        raise NotImplementedError
+
+    def to_batches(self):
+        return self.to_reader()
+
+    def to_table(self):
+        return self.to_reader().read_all()
+
+    def to_reader(self, *, limit: Optional[int] = None):
+        query = self.table.search()
+        # Disable the builtin limit
+        if limit is None:
+            num_rows = self.count_rows()
+            query.limit(num_rows)
+        elif limit <= 0:
+            raise ValueError("limit must be positive")
+        else:
+            query.limit(limit)
+        if self.columns is not None:
+            query = query.select(self.columns)
+        if self.filter is not None:
+            query = query.where(self.filter, prefilter=True)
+        return query.to_batches(batch_size=self.batch_size)
+
+
+class PyarrowDatasetAdapter(pa.dataset.Dataset):
+    def __init__(self, table: Table):
+        self.table = table
+
+    def count_rows(self, filter: Optional[Filter] = None):
+        return self.table.count_rows(filter)
+
+    def get_fragments(self, filter: Optional[Filter] = None):
+        raise NotImplementedError
+
+    def head(
+        self,
+        num_rows: int,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return self.scanner(
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        ).head(num_rows)
+
+    def join(
+        self,
+        right_dataset: Any,
+        keys: Keys,
+        right_keys: Optional[Keys] = None,
+        join_type: Optional[JoinType] = None,
+        left_suffix: Optional[str] = None,
+        right_suffix: Optional[str] = None,
+        coalesce_keys: bool = True,
+        use_threads: bool = True,
+    ):
+        raise NotImplementedError
+
+    def join_asof(
+        self,
+        right_dataset: Any,
+        on: str,
+        by: Keys,
+        tolerance: int,
+        right_on: Optional[str] = None,
+        right_by: Optional[Keys] = None,
+    ):
+        raise NotImplementedError
+
+    @property
+    def partition_expression(self):
+        raise NotImplementedError
+
+    def replace_schema(self, schema: pa.Schema):
+        raise NotImplementedError
+
+    def scanner(
+        self,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return PyarrowScannerAdapter(
+            self.table,
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        )
+
+    @property
+    def schema(self):
+        return self.table.schema
+
+    def sort_by(self, sorting: Union[str, List[Tuple[str, bool]]]):
+        raise NotImplementedError
+
+    def take(
+        self,
+        indices: List[int],
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        raise NotImplementedError
+
+    def to_batches(
+        self,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return self.scanner(
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        ).to_batches()
+
+    def to_table(
+        self,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return self.scanner(
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        ).to_table()

python/python/lancedb/pydantic.py

@@ -1,15 +1,5 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 """Pydantic (v1 / v2) adapter for LanceDB"""
 
@@ -30,6 +20,7 @@ from typing import (
     Type,
     Union,
     _GenericAlias,
+    GenericAlias,
 )
 
 import numpy as np
@@ -75,7 +66,7 @@ def vector(dim: int, value_type: pa.DataType = pa.float32()):
 
 
 def Vector(
-    dim: int, value_type: pa.DataType = pa.float32()
+    dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True
 ) -> Type[FixedSizeListMixin]:
     """Pydantic Vector Type.
 
@@ -88,6 +79,8 @@ def Vector(
         The dimension of the vector.
     value_type : pyarrow.DataType, optional
         The value type of the vector, by default pa.float32()
+    nullable : bool, optional
+        Whether the vector is nullable, by default it is True.
 
     Examples
     --------
@@ -103,7 +96,7 @@ def Vector(
     >>> assert schema == pa.schema([
     ...     pa.field("id", pa.int64(), False),
     ...     pa.field("url", pa.utf8(), False),
-    ...     pa.field("embeddings", pa.list_(pa.float32(), 768), False)
+    ...     pa.field("embeddings", pa.list_(pa.float32(), 768))
     ... ])
     """
 
@@ -112,6 +105,10 @@ def Vector(
         def __repr__(self):
             return f"FixedSizeList(dim={dim})"
 
+        @staticmethod
+        def nullable() -> bool:
+            return nullable
+
         @staticmethod
         def dim() -> int:
             return dim
@@ -205,9 +202,7 @@ else:
 def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
     """Convert a Pydantic FieldInfo to Arrow DataType"""
 
-    if isinstance(field.annotation, _GenericAlias) or (
-        sys.version_info > (3, 9) and isinstance(field.annotation, types.GenericAlias)
-    ):
+    if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
         origin = field.annotation.__origin__
         args = field.annotation.__args__
         if origin is list:
@@ -235,7 +230,7 @@ def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
 
 def is_nullable(field: FieldInfo) -> bool:
     """Check if a Pydantic FieldInfo is nullable."""
-    if isinstance(field.annotation, _GenericAlias):
+    if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
         origin = field.annotation.__origin__
         args = field.annotation.__args__
         if origin == Union:
@@ -246,6 +241,10 @@ def is_nullable(field: FieldInfo) -> bool:
         for typ in args:
             if typ is type(None):
                 return True
+    elif inspect.isclass(field.annotation) and issubclass(
+        field.annotation, FixedSizeListMixin
+    ):
+        return field.annotation.nullable()
     return False
 
 

python/python/lancedb/query.py

@@ -1,15 +1,5 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 from __future__ import annotations
 
@@ -26,6 +16,7 @@ from typing import (
     Union,
 )
 
+import asyncio
 import deprecation
 import numpy as np
 import pyarrow as pa
@@ -44,6 +35,8 @@ if TYPE_CHECKING:
     import polars as pl
 
     from ._lancedb import Query as LanceQuery
+    from ._lancedb import FTSQuery as LanceFTSQuery
+    from ._lancedb import HybridQuery as LanceHybridQuery
     from ._lancedb import VectorQuery as LanceVectorQuery
     from .common import VEC
     from .pydantic import LanceModel
@@ -325,6 +318,14 @@ class LanceQueryBuilder(ABC):
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def to_batches(self, /, batch_size: Optional[int] = None) -> pa.Table:
+        """
+        Execute the query and return the results as a pyarrow
+        [RecordBatchReader](https://arrow.apache.org/docs/python/generated/pyarrow.RecordBatchReader.html)
+        """
+        raise NotImplementedError
+
     def to_list(self) -> List[dict]:
         """
         Execute the query and return the results as a list of dictionaries.
@@ -869,6 +870,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
             check_reranker_result(results)
         return results
 
+    def to_batches(self, /, batch_size: Optional[int] = None):
+        raise NotImplementedError("to_batches on an FTS query")
+
     def tantivy_to_arrow(self) -> pa.Table:
         try:
             import tantivy
@@ -971,6 +975,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
 
 class LanceEmptyQueryBuilder(LanceQueryBuilder):
     def to_arrow(self) -> pa.Table:
+        return self.to_batches().read_all()
+
+    def to_batches(self, /, batch_size: Optional[int] = None) -> pa.RecordBatchReader:
         query = Query(
             columns=self._columns,
             filter=self._where,
@@ -980,7 +987,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
             # not actually respected in remote query
             offset=self._offset or 0,
         )
-        return self._table._execute_query(query).read_all()
+        return self._table._execute_query(query)
 
     def rerank(self, reranker: Reranker) -> LanceEmptyQueryBuilder:
         """Rerank the results using the specified reranker.
@@ -1110,32 +1117,55 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             fts_results = fts_future.result()
             vector_results = vector_future.result()
 
-        # convert to ranks first if needed
-        if self._norm == "rank":
-            vector_results = self._rank(vector_results, "_distance")
-            fts_results = self._rank(fts_results, "_score")
+        return self._combine_hybrid_results(
+            fts_results=fts_results,
+            vector_results=vector_results,
+            norm=self._norm,
+            fts_query=self._fts_query._query,
+            reranker=self._reranker,
+            limit=self._limit,
+            with_row_ids=self._with_row_id,
+        )
+
+    @staticmethod
+    def _combine_hybrid_results(
+        fts_results: pa.Table,
+        vector_results: pa.Table,
+        norm: str,
+        fts_query: str,
+        reranker,
+        limit: int,
+        with_row_ids: bool,
+    ) -> pa.Table:
+        if norm == "rank":
+            vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
+            fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
 
         # normalize the scores to be between 0 and 1, 0 being most relevant
-        vector_results = self._normalize_scores(vector_results, "_distance")
+        vector_results = LanceHybridQueryBuilder._normalize_scores(
+            vector_results, "_distance"
+        )
 
         # In fts higher scores represent relevance. Not inverting them here as
         # rerankers might need to preserve this score to support `return_score="all"`
-        fts_results = self._normalize_scores(fts_results, "_score")
+        fts_results = LanceHybridQueryBuilder._normalize_scores(fts_results, "_score")
 
-        results = self._reranker.rerank_hybrid(
-            self._fts_query._query, vector_results, fts_results
-        )
+        results = reranker.rerank_hybrid(fts_query, vector_results, fts_results)
 
         check_reranker_result(results)
 
-        # apply limit after reranking
-        results = results.slice(length=self._limit)
+        results = results.slice(length=limit)
 
-        if not self._with_row_id:
+        if not with_row_ids:
             results = results.drop(["_rowid"])
+
         return results
 
-    def _rank(self, results: pa.Table, column: str, ascending: bool = True):
+    def to_batches(self):
+        raise NotImplementedError("to_batches not yet supported on a hybrid query")
+
+    @staticmethod
+    def _rank(results: pa.Table, column: str, ascending: bool = True):
         if len(results) == 0:
             return results
         # Get the _score column from results
@@ -1152,7 +1182,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         )
         return results
 
-    def _normalize_scores(self, results: pa.Table, column: str, invert=False):
+    @staticmethod
+    def _normalize_scores(results: pa.Table, column: str, invert=False):
         if len(results) == 0:
             return results
         # Get the _score column from results
@@ -1502,10 +1533,11 @@ class AsyncQueryBase(object):
         ...     print(plan)
         >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
-          FilterExec: _distance@2 IS NOT NULL
-            SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
-              KNNVectorDistance: metric=l2
-                LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
+          GlobalLimitExec: skip=0, fetch=10
+            FilterExec: _distance@2 IS NOT NULL
+              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+                KNNVectorDistance: metric=l2
+                  LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
 
         Parameters
         ----------
@@ -1602,7 +1634,7 @@ class AsyncQuery(AsyncQueryBase):
         if (
             isinstance(query_vector, list)
             and len(query_vector) > 0
-            and not isinstance(query_vector[0], (float, int))
+            and isinstance(query_vector[0], (list, np.ndarray, pa.Array))
         ):
             # multiple have been passed
             query_vectors = [AsyncQuery._query_vec_to_array(v) for v in query_vector]
@@ -1617,7 +1649,7 @@ class AsyncQuery(AsyncQueryBase):
 
     def nearest_to_text(
         self, query: str, columns: Union[str, List[str]] = []
-    ) -> AsyncQuery:
+    ) -> AsyncFTSQuery:
         """
         Find the documents that are most relevant to the given text query.
 
@@ -1640,8 +1672,90 @@ class AsyncQuery(AsyncQueryBase):
         """
         if isinstance(columns, str):
             columns = [columns]
-        self._inner.nearest_to_text({"query": query, "columns": columns})
-        return self
+        return AsyncFTSQuery(
+            self._inner.nearest_to_text({"query": query, "columns": columns})
+        )
+
+
+class AsyncFTSQuery(AsyncQueryBase):
+    """A query for full text search for LanceDB."""
+
+    def __init__(self, inner: LanceFTSQuery):
+        super().__init__(inner)
+        self._inner = inner
+
+    def get_query(self):
+        self._inner.get_query()
+
+    def nearest_to(
+        self,
+        query_vector: Union[VEC, Tuple, List[VEC]],
+    ) -> AsyncHybridQuery:
+        """
+        In addition doing text search on the LanceDB Table, also
+        find the nearest vectors to the given query vector.
+
+        This converts the query from a FTS Query to a Hybrid query. Results
+        from the vector search will be combined with results from the FTS query.
+
+        This method will attempt to convert the input to the query vector
+        expected by the embedding model.  If the input cannot be converted
+        then an error will be thrown.
+
+        By default, there is no embedding model, and the input should be
+        something that can be converted to a pyarrow array of floats.  This
+        includes lists, numpy arrays, and tuples.
+
+        If there is only one vector column (a column whose data type is a
+        fixed size list of floats) then the column does not need to be specified.
+        If there is more than one vector column you must use
+        [AsyncVectorQuery.column][lancedb.query.AsyncVectorQuery.column] to specify
+        which column you would like to compare with.
+
+        If no index has been created on the vector column then a vector query
+        will perform a distance comparison between the query vector and every
+        vector in the database and then sort the results.  This is sometimes
+        called a "flat search"
+
+        For small databases, with tens of thousands of vectors or less, this can
+        be reasonably fast.  In larger databases you should create a vector index
+        on the column.  If there is a vector index then an "approximate" nearest
+        neighbor search (frequently called an ANN search) will be performed.  This
+        search is much faster, but the results will be approximate.
+
+        The query can be further parameterized using the returned builder.  There
+        are various ANN search parameters that will let you fine tune your recall
+        accuracy vs search latency.
+
+        Hybrid searches always have a [limit][].  If `limit` has not been called then
+        a default `limit` of 10 will be used.
+
+        Typically, a single vector is passed in as the query. However, you can also
+        pass in multiple vectors.  This can be useful if you want to find the nearest
+        vectors to multiple query vectors. This is not expected to be faster than
+        making multiple queries concurrently; it is just a convenience method.
+        If multiple vectors are passed in then an additional column `query_index`
+        will be added to the results.  This column will contain the index of the
+        query vector that the result is nearest to.
+        """
+        if query_vector is None:
+            raise ValueError("query_vector can not be None")
+
+        if (
+            isinstance(query_vector, list)
+            and len(query_vector) > 0
+            and not isinstance(query_vector[0], (float, int))
+        ):
+            # multiple have been passed
+            query_vectors = [AsyncQuery._query_vec_to_array(v) for v in query_vector]
+            new_self = self._inner.nearest_to(query_vectors[0])
+            for v in query_vectors[1:]:
+                new_self.add_query_vector(v)
+            return AsyncHybridQuery(new_self)
+        else:
+            return AsyncHybridQuery(
+                self._inner.nearest_to(AsyncQuery._query_vec_to_array(query_vector))
+            )
 
 
 class AsyncVectorQuery(AsyncQueryBase):
@@ -1778,3 +1892,160 @@ class AsyncVectorQuery(AsyncQueryBase):
         """
         self._inner.bypass_vector_index()
         return self
+
+    def nearest_to_text(
+        self, query: str, columns: Union[str, List[str]] = []
+    ) -> AsyncHybridQuery:
+        """
+        Find the documents that are most relevant to the given text query,
+        in addition to vector search.
+
+        This converts the vector query into a hybrid query.
+
+        This search will perform a full text search on the table and return
+        the most relevant documents, combined with the vector query results.
+        The text relevance is determined by BM25.
+
+        The columns to search must be with native FTS index
+        (Tantivy-based can't work with this method).
+
+        By default, all indexed columns are searched,
+        now only one column can be searched at a time.
+
+        Parameters
+        ----------
+        query: str
+            The text query to search for.
+        columns: str or list of str, default None
+            The columns to search in. If None, all indexed columns are searched.
+            For now only one column can be searched at a time.
+        """
+        if isinstance(columns, str):
+            columns = [columns]
+        return AsyncHybridQuery(
+            self._inner.nearest_to_text({"query": query, "columns": columns})
+        )
+
+
+class AsyncHybridQuery(AsyncQueryBase):
+    """
+    A query builder that performs hybrid vector and full text search.
+    Results are combined and reranked based on the specified reranker.
+    By default, the results are reranked using the RRFReranker, which
+    uses reciprocal rank fusion score for reranking.
+
+    To make the vector and fts results comparable, the scores are normalized.
+    Instead of normalizing scores, the `normalize` parameter can be set to "rank"
+    in the `rerank` method to convert the scores to ranks and then normalize them.
+    """
+
+    def __init__(self, inner: LanceHybridQuery):
+        super().__init__(inner)
+        self._inner = inner
+        self._norm = "score"
+        self._reranker = RRFReranker()
+
+    def rerank(
+        self, reranker: Reranker = RRFReranker(), normalize: str = "score"
+    ) -> AsyncHybridQuery:
+        """
+        Rerank the hybrid search results using the specified reranker. The reranker
+        must be an instance of Reranker class.
+
+        Parameters
+        ----------
+        reranker: Reranker, default RRFReranker()
+            The reranker to use. Must be an instance of Reranker class.
+        normalize: str, default "score"
+            The method to normalize the scores. Can be "rank" or "score". If "rank",
+            the scores are converted to ranks and then normalized. If "score", the
+            scores are normalized directly.
+        Returns
+        -------
+        AsyncHybridQuery
+            The AsyncHybridQuery object.
+        """
+        if normalize not in ["rank", "score"]:
+            raise ValueError("normalize must be 'rank' or 'score'.")
+        if reranker and not isinstance(reranker, Reranker):
+            raise ValueError("reranker must be an instance of Reranker class.")
+
+        self._norm = normalize
+        self._reranker = reranker
+
+        return self
+
+    async def to_batches(self):
+        raise NotImplementedError("to_batches not yet supported on a hybrid query")
+
+    async def to_arrow(self) -> pa.Table:
+        fts_query = AsyncFTSQuery(self._inner.to_fts_query())
+        vec_query = AsyncVectorQuery(self._inner.to_vector_query())
+
+        # save the row ID choice that was made on the query builder and force it
+        # to actually fetch the row ids because we need this for reranking
+        with_row_ids = self._inner.get_with_row_id()
+        fts_query.with_row_id()
+        vec_query.with_row_id()
+
+        fts_results, vector_results = await asyncio.gather(
+            fts_query.to_arrow(),
+            vec_query.to_arrow(),
+        )
+
+        return LanceHybridQueryBuilder._combine_hybrid_results(
+            fts_results=fts_results,
+            vector_results=vector_results,
+            norm=self._norm,
+            fts_query=fts_query.get_query(),
+            reranker=self._reranker,
+            limit=self._inner.get_limit(),
+            with_row_ids=with_row_ids,
+        )
+
+    async def explain_plan(self, verbose: Optional[bool] = False):
+        """Return the execution plan for this query.
+
+        The output includes both the vector and FTS search plans.
+
+        Examples
+        --------
+        >>> import asyncio
+        >>> from lancedb import connect_async
+        >>> from lancedb.index import FTS
+        >>> async def doctest_example():
+        ...     conn = await connect_async("./.lancedb")
+        ...     table = await conn.create_table("my_table", [{"vector": [99, 99], "text": "hello world"}])
+        ...     await table.create_index("text", config=FTS(with_position=False))
+        ...     query = [100, 100]
+        ...     plan = await table.query().nearest_to([1, 2]).nearest_to_text("hello").explain_plan(True)
+        ...     print(plan)
+        >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+        Vector Search Plan:
+        ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance]
+            Take: columns="vector, _rowid, _distance, (text)"
+                CoalesceBatchesExec: target_batch_size=1024
+                GlobalLimitExec: skip=0, fetch=10
+                    FilterExec: _distance@2 IS NOT NULL
+                    SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+                        KNNVectorDistance: metric=l2
+                        LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
+        FTS Search Plan:
+        LanceScan: uri=..., projection=[vector, text], row_id=false, row_addr=false, ordered=true
+
+        Parameters
+        ----------
+        verbose : bool, default False
+            Use a verbose output format.
+
+        Returns
+        -------
+        plan
+        """  # noqa: E501
+
+        results = ["Vector Search Plan:"]
+        results.append(await self._inner.to_vector_query().explain_plan(verbose))
+        results.append("FTS Search Plan:")
+        results.append(await self._inner.to_fts_query().explain_plan(verbose))
+
+        return "\n".join(results)

python/python/lancedb/remote/db.py

@@ -11,7 +11,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import asyncio
 from datetime import timedelta
 import logging
 from concurrent.futures import ThreadPoolExecutor
@@ -25,7 +24,7 @@ import pyarrow as pa
 from overrides import override
 
 from ..common import DATA
-from ..db import DBConnection
+from ..db import DBConnection, LOOP
 from ..embeddings import EmbeddingFunctionConfig
 from ..pydantic import LanceModel
 from ..table import Table
@@ -45,9 +44,9 @@ class RemoteDBConnection(DBConnection):
         client_config: Union[ClientConfig, Dict[str, Any], None] = None,
         connection_timeout: Optional[float] = None,
         read_timeout: Optional[float] = None,
+        storage_options: Optional[Dict[str, str]] = None,
     ):
         """Connect to a remote LanceDB database."""
-
         if isinstance(client_config, dict):
             client_config = ClientConfig(**client_config)
         elif client_config is None:
@@ -86,24 +85,16 @@ class RemoteDBConnection(DBConnection):
             raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
         self.db_name = parsed.netloc
 
-        import nest_asyncio
-
-        nest_asyncio.apply()
-        try:
-            self._loop = asyncio.get_running_loop()
-        except RuntimeError:
-            self._loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(self._loop)
-
         self.client_config = client_config
 
-        self._conn = self._loop.run_until_complete(
+        self._conn = LOOP.run(
             connect_async(
                 db_url,
                 api_key=api_key,
                 region=region,
                 host_override=host_override,
                 client_config=client_config,
+                storage_options=storage_options,
             )
         )
 
@@ -127,9 +118,7 @@ class RemoteDBConnection(DBConnection):
         -------
         An iterator of table names.
         """
-        return self._loop.run_until_complete(
-            self._conn.table_names(start_after=page_token, limit=limit)
-        )
+        return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
 
     @override
     def open_table(self, name: str, *, index_cache_size: Optional[int] = None) -> Table:
@@ -152,8 +141,8 @@ class RemoteDBConnection(DBConnection):
                 " (there is no local cache to configure)"
             )
 
-        table = self._loop.run_until_complete(self._conn.open_table(name))
-        return RemoteTable(table, self.db_name, self._loop)
+        table = LOOP.run(self._conn.open_table(name))
+        return RemoteTable(table, self.db_name)
 
     @override
     def create_table(
@@ -268,7 +257,7 @@ class RemoteDBConnection(DBConnection):
 
         from .table import RemoteTable
 
-        table = self._loop.run_until_complete(
+        table = LOOP.run(
             self._conn.create_table(
                 name,
                 data,
@@ -278,7 +267,7 @@ class RemoteDBConnection(DBConnection):
                 fill_value=fill_value,
             )
         )
-        return RemoteTable(table, self.db_name, self._loop)
+        return RemoteTable(table, self.db_name)
 
     @override
     def drop_table(self, name: str):
@@ -289,7 +278,7 @@ class RemoteDBConnection(DBConnection):
         name: str
             The name of the table.
         """
-        self._loop.run_until_complete(self._conn.drop_table(name))
+        LOOP.run(self._conn.drop_table(name))
 
     @override
     def rename_table(self, cur_name: str, new_name: str):
@@ -302,7 +291,7 @@ class RemoteDBConnection(DBConnection):
         new_name: str
             The new name of the table.
         """
-        self._loop.run_until_complete(self._conn.rename_table(cur_name, new_name))
+        LOOP.run(self._conn.rename_table(cur_name, new_name))
 
     async def close(self):
         """Close the connection to the database."""