Bump version: 0.17.0 → 0.17.1-beta.0

db.connect with azure storage account name is supported in async connect
but not sync connect.
Add this functionality

---------

Co-authored-by: Will Jones <willjones127@gmail.com>

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.13.0-beta.1"
+current_version = "0.14.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -87,6 +87,16 @@ glob = "node/package.json"
 replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\""
 search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\""
 
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-linux-arm64-musl\": \"{new_version}\""
+search = "\"@lancedb/vectordb-linux-arm64-musl\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-linux-x64-musl\": \"{new_version}\""
+search = "\"@lancedb/vectordb-linux-x64-musl\": \"{current_version}\""
+
 [[tool.bumpversion.files]]
 glob = "node/package.json"
 replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""

.cargo/config.toml

@@ -31,6 +31,9 @@ rustflags = [
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 
+[target.x86_64-unknown-linux-musl]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"]
+
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
 

.github/workflows/docs.yml

@@ -72,9 +72,9 @@ jobs:
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.github/workflows/nodejs.yml

@@ -104,7 +104,6 @@ jobs:
         OPENAI_BASE_URL: http://0.0.0.0:8000
       run: |
         python ci/mock_openai.py &
-        ss -ltnp | grep :8000
         cd nodejs/examples
         npm test
   macos:

.github/workflows/npm-publish.yml

@@ -101,7 +101,7 @@ jobs:
           path: |
             nodejs/dist/*.node
 
-  node-linux:
+  node-linux-gnu:
     name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
@@ -133,15 +133,67 @@ jobs:
           free -h
       - name: Build Linux Artifacts
         run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: node-native-linux-${{ matrix.config.arch }}
+          name: node-native-linux-${{ matrix.config.arch }}-gnu
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
-  nodejs-linux:
+  node-linux-musl:
+    name: vectordb (${{ matrix.config.arch}}-unknown-linux-musl)
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+          - arch: aarch64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install common dependencies
+        run: |
+          apk add protobuf-dev curl clang mold grep npm bash
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          echo "source $HOME/.cargo/env" >> saved_env
+          echo "export CC=clang" >> saved_env
+          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          source "$HOME/.cargo/env"
+          rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
+          crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
+          sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
+          apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
+          curl -sSf $apk_url > apk_list
+          for pkg in gcc libgcc musl; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done
+          mkdir -p $sysroot_lib
+          echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so
+          cp usr/lib/libgcc_s.so.1 $sysroot_lib
+          cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib
+          cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so
+          echo '!<arch>' > $sysroot_lib/libdl.a
+          (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib)
+          echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env
+          echo "export RUSTFLAGS='-Ctarget-cpu=apple-m1 -Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env
+      - name: Build Linux Artifacts
+        run: |
+          source ./saved_env
+          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-musl
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-linux-${{ matrix.config.arch }}-musl
+          path: |
+            node/dist/lancedb-vectordb-linux*.tgz
+
+  nodejs-linux-gnu:
     name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
@@ -178,7 +230,7 @@ jobs:
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}
+          name: nodejs-native-linux-${{ matrix.config.arch }}-gnu
           path: |
             nodejs/dist/*.node
       # The generic files are the same in all distros so we just pick
@@ -192,6 +244,62 @@ jobs:
             nodejs/dist/*
             !nodejs/dist/*.node
 
+  nodejs-linux-musl:
+    name: lancedb (${{ matrix.config.arch}}-unknown-linux-musl
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+          - arch: aarch64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install common dependencies
+        run: |
+          apk add protobuf-dev curl clang mold grep npm bash openssl-dev openssl-libs-static
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          echo "source $HOME/.cargo/env" >> saved_env
+          echo "export CC=clang" >> saved_env
+          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
+          echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=/usr/include" >> saved_env
+          echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=/usr/lib" >> saved_env
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          source "$HOME/.cargo/env"
+          rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
+          crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
+          sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
+          apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
+          curl -sSf $apk_url > apk_list
+          for pkg in gcc libgcc musl openssl-dev openssl-libs-static; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done
+          mkdir -p $sysroot_lib
+          echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so
+          cp usr/lib/libgcc_s.so.1 $sysroot_lib
+          cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib
+          cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so
+          echo '!<arch>' > $sysroot_lib/libdl.a
+          (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib)
+          echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env
+          echo "export RUSTFLAGS='-Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env
+          echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=$(realpath usr/include)" >> saved_env
+          echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=$(realpath usr/lib)" >> saved_env
+      - name: Build Linux Artifacts
+        run: |
+          source ./saved_env
+          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-linux-${{ matrix.config.arch }}-musl
+          path: |
+            nodejs/dist/*.node
+
   node-windows:
     name: vectordb ${{ matrix.target }}
     runs-on: windows-2022
@@ -227,107 +335,49 @@ jobs:
             node/dist/lancedb-vectordb-win32*.tgz
 
   node-windows-arm64:
-    name: vectordb win32-arm64-msvc
-    runs-on: windows-4x-arm
+    name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
     if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          # - arch: x86_64
+          - arch: aarch64
     steps:
-      - uses: actions/checkout@v4
-      - name: Install Git
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install dependencies
         run: |
-          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-        shell: powershell
-      - name: Add Git to PATH
+          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          echo "source $HOME/.cargo/env" >> saved_env
+          echo "export CC=clang" >> saved_env
+          echo "export AR=llvm-ar" >> saved_env
+          source "$HOME/.cargo/env"
+          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc --toolchain 1.80.0
+          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
+          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
+          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
+      - name: Configure x86_64 build
+        if: ${{ matrix.config.arch == 'x86_64' }}
         run: |
-          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-        shell: powershell
-      - name: Configure Git symlinks
-        run: git config --global core.symlinks true
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-      - name: Install Visual Studio Build Tools
+          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
         run: |
-          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-            "--installPath", "C:\BuildTools", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-        shell: powershell
-      - name: Add Visual Studio Build Tools to PATH
+          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
+      - name: Build Windows Artifacts
         run: |
-          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
-
-          # Add MSVC runtime libraries to LIB
-          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" + 
-                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
-                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
-
-          # Add INCLUDE paths
-          $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
-                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
-          Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
-        shell: powershell
-      - name: Install Rust
-        run: |
-          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-        shell: powershell
-      - name: Add Rust to PATH
-        run: |
-          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-        shell: powershell
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install 7-Zip ARM
-        run: |
-          New-Item -Path 'C:\7zip' -ItemType Directory
-          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-        shell: powershell
-      - name: Add 7-Zip to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-        shell: powershell
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          if (Test-Path 'C:\protoc') {
-              Write-Host "Protoc directory exists, skipping installation"
-              return
-          }
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-        shell: powershell
-      - name: Add Protoc to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts.ps1 aarch64-pc-windows-msvc
-      - name: Upload Windows ARM64 Artifacts
+          source ./saved_env
+          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-pc-windows-msvc
+      - name: Upload Windows Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: node-native-windows-arm64
+          name: node-native-windows-${{ matrix.config.arch }}
           path: |
-            node/dist/*.node
+            node/dist/lancedb-vectordb-win32*.tgz
 
   nodejs-windows:
     name: lancedb ${{ matrix.target }}
@@ -364,101 +414,56 @@ jobs:
             nodejs/dist/*.node
 
   nodejs-windows-arm64:
-    name: lancedb win32-arm64-msvc
-    runs-on: windows-4x-arm
+    name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
+    # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          # - arch: x86_64
+          - arch: aarch64
     steps:
-      - uses: actions/checkout@v4
-      - name: Install Git
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install dependencies
         run: |
-          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-        shell: powershell
-      - name: Add Git to PATH
+          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          echo "source $HOME/.cargo/env" >> saved_env
+          echo "export CC=clang" >> saved_env
+          echo "export AR=llvm-ar" >> saved_env
+          source "$HOME/.cargo/env"
+          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc --toolchain 1.80.0
+          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
+          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
+          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
+          printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin
+          chmod u+x $HOME/.cargo/bin/cargo-xwin
+      - name: Configure x86_64 build
+        if: ${{ matrix.config.arch == 'x86_64' }}
         run: |
-          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-        shell: powershell
-      - name: Configure Git symlinks
-        run: git config --global core.symlinks true
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-      - name: Install Visual Studio Build Tools
+          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
         run: |
-          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-            "--installPath", "C:\BuildTools", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-        shell: powershell
-      - name: Add Visual Studio Build Tools to PATH
+          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
+      - name: Build Windows Artifacts
         run: |
-          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
-
-          $env:LIB = ""
-          Add-Content $env:GITHUB_ENV "LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-        shell: powershell
-      - name: Install Rust
-        run: |
-          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-        shell: powershell
-      - name: Add Rust to PATH
-        run: |
-          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-        shell: powershell
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install 7-Zip ARM
-        run: |
-          New-Item -Path 'C:\7zip' -ItemType Directory
-          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-        shell: powershell
-      - name: Add 7-Zip to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-        shell: powershell
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          if (Test-Path 'C:\protoc') {
-              Write-Host "Protoc directory exists, skipping installation"
-              return
-          }
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-        shell: powershell
-      - name: Add Protoc to PATH
-        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts_nodejs.ps1 aarch64-pc-windows-msvc
-      - name: Upload Windows ARM64 Artifacts
+          source ./saved_env
+          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
+      - name: Upload Windows Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: nodejs-native-windows-arm64
+          name: nodejs-native-windows-${{ matrix.config.arch }}
           path: |
             nodejs/dist/*.node
 
   release:
     name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux, node-windows, node-windows-arm64]
+    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows, node-windows-arm64]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -476,7 +481,7 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
-          # Tag beta as "preview" instead of default "latest". See lancedb 
+          # Tag beta as "preview" instead of default "latest". See lancedb
           # npm publish step for more info.
           if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
             PUBLISH_ARGS="--tag preview"
@@ -498,7 +503,7 @@ jobs:
 
   release-nodejs:
     name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows, nodejs-windows-arm64]
+    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows, nodejs-windows-arm64]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -556,6 +561,7 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   update-package-lock:
+    if: startsWith(github.ref, 'refs/tags/v')
     needs: [release]
     runs-on: ubuntu-latest
     permissions:
@@ -573,6 +579,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   update-package-lock-nodejs:
+    if: startsWith(github.ref, 'refs/tags/v')
     needs: [release-nodejs]
     runs-on: ubuntu-latest
     permissions:
@@ -590,6 +597,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   gh-release:
+    if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/pypi-publish.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - uses: ./.github/workflows/build_windows_wheel
         with:
           python-minor-version: 8

.github/workflows/upload_wheel/action.yml

@@ -17,6 +17,7 @@ runs:
     run: |
       python -m pip install --upgrade pip
       pip install twine
+      python3 -m pip install --upgrade pkginfo
   - name: Choose repo
     shell: bash
     id: choose_repo

Cargo.toml

@@ -18,31 +18,32 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.80.0" # TODO: lower this once we upgrade Lance again.
+rust-version = "1.80.0"                                                     # TODO: lower this once we upgrade Lance again.
 
 [workspace.dependencies]
-lance = { "version" = "=0.19.2", "features" = [
+lance = { "version" = "=0.20.0", "features" = [
     "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
-lance-index = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
-lance-linalg = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
-lance-table = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
-lance-testing = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
-lance-datafusion = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
-lance-encoding = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2" }
+] }
+lance-io = "0.20.0"
+lance-index = "0.20.0"
+lance-linalg = "0.20.0"
+lance-table = "0.20.0"
+lance-testing = "0.20.0"
+lance-datafusion = "0.20.0"
+lance-encoding = "0.20.0"
 # Note that this one does not include pyarrow
-arrow = { version = "52.2", optional = false }
-arrow-array = "52.2"
-arrow-data = "52.2"
-arrow-ipc = "52.2"
-arrow-ord = "52.2"
-arrow-schema = "52.2"
-arrow-arith = "52.2"
-arrow-cast = "52.2"
+arrow = { version = "53.2", optional = false }
+arrow-array = "53.2"
+arrow-data = "53.2"
+arrow-ipc = "53.2"
+arrow-ord = "53.2"
+arrow-schema = "53.2"
+arrow-arith = "53.2"
+arrow-cast = "53.2"
 async-trait = "0"
 chrono = "0.4.35"
-datafusion-common = "41.0"
-datafusion-physical-plan = "41.0"
+datafusion-common = "42.0"
+datafusion-physical-plan = "42.0"
 env_logger = "0.10"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",

ci/build_linux_artifacts.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 set -e
 ARCH=${1:-x86_64}
+TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
-# We pass down the current user so that when we later mount the local files 
+# We pass down the current user so that when we later mount the local files
 # into the container, the files are accessible by the current user.
 pushd ci/manylinux_node
 docker build \
@@ -18,4 +19,4 @@ docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
     lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH
+    bash ci/manylinux_node/build_vectordb.sh $ARCH $TARGET_TRIPLE

ci/manylinux_node/build_lancedb.sh

@@ -11,7 +11,8 @@ fi
 export OPENSSL_STATIC=1
 export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
 
-source $HOME/.bashrc
+#Alpine doesn't have .bashrc
+FILE=$HOME/.bashrc && test -f $FILE && source $FILE
 
 cd nodejs
 npm ci

ci/manylinux_node/build_vectordb.sh

@@ -2,18 +2,20 @@
 # Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
 set -e
 ARCH=${1:-x86_64}
+TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}
 
 if [ "$ARCH" = "x86_64" ]; then
     export OPENSSL_LIB_DIR=/usr/local/lib64/
-else 
+else
     export OPENSSL_LIB_DIR=/usr/local/lib/
 fi
 export OPENSSL_STATIC=1
 export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
 
-source $HOME/.bashrc
+#Alpine doesn't have .bashrc
+FILE=$HOME/.bashrc && test -f $FILE && source $FILE
 
 cd node
 npm ci
 npm run build-release
-npm run pack-build
+npm run pack-build -- -t $TARGET_TRIPLE

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+
+# fwpuclnt.lib arm64rt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
+
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
+mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# ARM intrinsics
+# original dir: MSVC/
+
+# '__n128x4' redefined in arm_neon.h
+# "arm64_neon.h" included from intrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
+
+# .lib
+
+# _Interlocked intrinsics
+# must always link with arm64rt.lib
+# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
+# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
+
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib

ci/sysroot-x86_64-pc-windows-msvc.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
+# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
+
+# function dl() {
+# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
+# }
+
+# [[.h]]
+
+# "id": "Win11SDK_10.0.26100"
+# "version": "10.0.26100.7"
+
+# libucrt.lib
+
+# example: <assert.h>
+# dir: ucrt/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
+
+# example: <windows.h>
+# dir: um/
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
+
+# example: <winapifamily.h>
+# dir: /shared
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
+
+
+# "id": "Microsoft.VisualC.14.16.CRT.Headers"
+# "version": "14.16.27045"
+
+# example: <vcruntime.h>
+# dir: MSVC/
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
+
+# [[.lib]]
+
+# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
+
+# dbghelp.lib fwpuclnt.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
+
+# libcmt.lib libvcruntime.lib
+curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+
+msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
+msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
+msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
+unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
+unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
+
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
+mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
+
+# lowercase folder/file names
+echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
+
+# .h
+(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
+
+# lowercase #include "" and #include <>
+find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
+
+# x86 intrinsics
+# original dir: MSVC/
+
+# '_mm_movemask_epi8' defined in emmintrin.h
+# '__v4sf' defined in xmmintrin.h
+# '__v2si' defined in mmintrin.h
+# '__m128d' redefined in immintrin.h
+# '__m128i' redefined in intrin.h
+# '_mm_comlt_epu8' defined in ammintrin.h
+
+(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
+
+# .lib
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
+
+cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib

docs/mkdocs.yml

@@ -55,6 +55,9 @@ plugins:
             show_signature_annotations: true
             show_root_heading: true
             members_order: source
+            docstring_section_style: list
+            signature_crossrefs: true
+            separate_signature: true
           import:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
@@ -138,6 +141,7 @@ nav:
               - Jina Reranker: reranking/jina.md
               - OpenAI Reranker: reranking/openai.md
               - AnswerDotAi Rerankers: reranking/answerdotai.md
+              - Voyage AI Rerankers: reranking/voyageai.md
               - Building Custom Rerankers: reranking/custom_reranker.md
               - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
@@ -165,6 +169,7 @@ nav:
                   - Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
                   - AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
                   - IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
+                  - Voyage AI Embeddings: embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
               - Multimodal Embedding Functions:
                   - OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
                   - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md

docs/package-lock.json

@@ -19,7 +19,7 @@
     },
     "../node": {
       "name": "vectordb",
-      "version": "0.4.6",
+      "version": "0.12.0",
       "cpu": [
         "x64",
         "arm64"
@@ -31,9 +31,7 @@
         "win32"
       ],
       "dependencies": {
-        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -46,6 +44,7 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -62,15 +61,19 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "*",
+        "typescript": "^5.1.0",
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.6",
-        "@lancedb/vectordb-darwin-x64": "0.4.6",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
+        "@lancedb/vectordb-darwin-arm64": "0.12.0",
+        "@lancedb/vectordb-darwin-x64": "0.12.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.12.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.12.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.12.0"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
       }
     },
     "../node/node_modules/apache-arrow": {

docs/src/ann_indexes.md

@@ -277,7 +277,15 @@ Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` t
 Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
 On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
 
-`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
+`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. The number should be a factor of the vector dimension. Because
 PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
-less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
-more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+
+!!! note
+    if `num_sub_vectors` is set to be greater than the vector dimension, you will see errors like `attempt to divide by zero`
+
+### How to choose `m` and `ef_construction` for `IVF_HNSW_*` index?
+
+`m` determines the number of connections a new node establishes with its closest neighbors upon entering the graph. Typically, `m` falls within the range of 5 to 48. Lower `m` values are suitable for low-dimensional data or scenarios where recall is less critical. Conversely, higher `m` values are beneficial for high-dimensional data or when high recall is required. In essence, a larger `m` results in a denser graph with increased connectivity, but at the expense of higher memory consumption.
+
+`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase

docs/src/concepts/index_hnsw.md

@@ -57,6 +57,13 @@ Then the greedy search routine operates as follows:
 
 ## Usage
 
+There are three key parameters to set when constructing an HNSW index:
+
+* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `m`: The number of neighbors to select for each vector in the HNSW graph.
+* `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph.
+
+
 We can combine the above concepts to understand how to build and query an HNSW index in LanceDB.
 
 ### Construct index

docs/src/concepts/index_ivfpq.md

@@ -58,8 +58,10 @@ In Python, the index can be created as follows:
 # Make sure you have enough data in the table for an effective training step
 tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
 ```
+!!! note
+    `num_partitions`=256 and `num_sub_vectors`=96 does not work for every dataset. Those values needs to be adjusted for your particular dataset.
 
-The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See the [FAQs](#faq) below for best practices on choosing these parameters.
+The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See [here](../ann_indexes.md/#how-to-choose-num_partitions-and-num_sub_vectors-for-ivf_pq-index) for best practices on choosing these parameters.
 
 
 ### Query the index

docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md

@@ -6,6 +6,7 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
 | `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
+| `use_azure` | bool | `False` | Set true to use Azure OpenAPI SDK |
 
 
 ```python

docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md

@@ -20,7 +20,7 @@ Supported parameters (to be passed in `create` method) are:
 
 | Parameter | Type | Default Value | Description |
 |---|---|--------|---------|
-| `name` | `str` | `"voyage-3"` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
+| `name` | `str` | `None` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
 | `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. |
 | `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. |
 

docs/src/embeddings/default_embedding_functions.md

@@ -53,6 +53,7 @@ These functions are registered by default to handle text embeddings.
 | [**Jina Embeddings**](available_embedding_models/text_embedding_functions/jina_embedding.md "jina") | 🔗 World-class embedding models to improve your search and RAG systems. You will need **jina api key**. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="Jina Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/jina_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 | [ **AWS Bedrock Functions**](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md "bedrock-text") | ☁️ AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/aws_bedrock.png" alt="AWS Bedrock Icon" width="120" height="35">](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 | [**IBM Watsonx.ai**](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md "watsonx") | 💡 Generate text embeddings using IBM's watsonx.ai platform. **Note**: watsonx.ai library is an optional dependency. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/watsonx.png" alt="Watsonx Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md) |
+| [**VoyageAI Embeddings**](available_embedding_models/text_embedding_functions/voyageai_embedding.md "voyageai") | 🌕 Voyage AI provides cutting-edge embedding and rerankers. This will help you get started with **VoyageAI** embedding models using LanceDB. Using voyageai API requires voyageai package. Install it via `pip`. | [<img src="https://www.voyageai.com/logo.svg" alt="VoyageAI Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/voyageai_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 
 
 
@@ -66,6 +67,7 @@ These functions are registered by default to handle text embeddings.
 [jina-key]: "jina"
 [aws-key]: "bedrock-text"
 [watsonx-key]: "watsonx"
+[voyageai-key]: "voyageai"
 
 
 ## Multi-modal Embedding Functions🖼️ 

docs/src/fts.md

@@ -114,12 +114,45 @@ table.create_fts_index("text",
 
 LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported.
 
-This can be invoked via the familiar `where` syntax:
-
+This can be invoked via the familiar `where` syntax.
+ 
+With pre-filtering:
 === "Python"
 
     ```python
-    table.search("puppy").limit(10).where("meta='foo'").to_list()
+    table.search("puppy").limit(10).where("meta='foo'", prefilte=True).to_list()
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl
+    .search("puppy")
+    .select(["id", "doc"])
+    .limit(10)
+    .where("meta='foo'")
+    .prefilter(true)
+    .toArray();
+    ```
+
+=== "Rust"
+
+    ```rust
+    table
+        .query()
+        .full_text_search(FullTextSearchQuery::new("puppy".to_owned()))
+        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
+        .limit(10)
+        .only_if("meta='foo'")
+        .execute()
+        .await?;
+    ```
+
+With post-filtering:
+=== "Python"
+
+    ```python
+    table.search("puppy").limit(10).where("meta='foo'", prefilte=False).to_list()
     ```
 
 === "TypeScript"
@@ -130,6 +163,7 @@ This can be invoked via the familiar `where` syntax:
     .select(["id", "doc"])
     .limit(10)
     .where("meta='foo'")
+    .prefilter(false)
     .toArray();
     ```
 
@@ -140,6 +174,7 @@ This can be invoked via the familiar `where` syntax:
         .query()
         .full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
         .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
+        .postfilter()
         .limit(10)
         .only_if("meta='foo'")
         .execute()
@@ -160,3 +195,35 @@ To search for a phrase, the index must be created with `with_position=True`:
 table.create_fts_index("text", use_tantivy=False, with_position=True)
 ```
 This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
+
+
+## Incremental indexing
+
+LanceDB supports incremental indexing, which means you can add new records to the table without reindexing the entire table.
+
+This can make the query more efficient, especially when the table is large and the new records are relatively small.
+
+=== "Python"
+
+    ```python
+    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
+    table.optimize()
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl.add([{ vector: [3.1, 4.1], text: "Frodo was a happy puppy" }]);
+    await tbl.optimize();
+    ```
+
+=== "Rust"
+
+    ```rust
+    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    tbl.add(more_data).execute().await?;
+    tbl.optimize(OptimizeAction::All).execute().await?;
+    ```
+!!! note
+
+    New data added after creating the FTS index will appear in search results while incremental index is still progress, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 

docs/src/fts_tantivy.md

@@ -153,9 +153,7 @@ table.create_fts_index(["title", "content"], use_tantivy=True, writer_heap_size=
 
 ## Current limitations
 
-1. Currently we do not yet support incremental writes.
-   If you add data after FTS index creation, it won't be reflected
-   in search results until you do a full reindex.
+1. New data added after creating the FTS index will appear in search results, but with increased latency due to a flat search on the unindexed portion. Re-indexing with `create_fts_index` will reduce latency. LanceDB Cloud automates this merging process, minimizing the impact on search speed. 
 
 2. We currently only support local filesystem paths for the FTS index.
    This is a tantivy limitation. We've implemented an object store plugin

docs/src/guides/scalar_index.md

@@ -1,23 +1,35 @@
-# Building Scalar Index
+# Building a Scalar Index
 
-Similar to many SQL databases, LanceDB supports several types of Scalar indices to accelerate search
+Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
+
+Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
 over scalar columns.
 
-- `BTREE`: The most common type is BTREE. This index is inspired by the btree data structure
-  although only the first few layers of the btree are cached in memory.
-  It will perform well on columns with a large number of unique values and few rows per value.
-- `BITMAP`: this index stores a bitmap for each unique value in the column.
-  This index is useful for columns with a finite number of unique values and many rows per value.
-  For example, columns that represent "categories", "labels", or "tags"
-- `LABEL_LIST`: a special index that is used to index list columns whose values have a finite set of possibilities.
+- `BTREE`: The most common type is BTREE. The index stores a copy of the
+  column in sorted order. This sorted copy allows a binary search to be used to
+  satisfy queries.
+- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
+  uses a series of bits to indicate whether a value is present in a row of a table
+- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
+  support queries with `array_contains_all` and `array_contains_any`
+  using an underlying bitmap index.
   For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.
 
+!!! tips "How to choose the right scalar index type"
+
+    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
+    
+    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
+    
+    `LABEL_LIST`: This index should be used for columns containing list-type data.
+
 | Data Type                                                       | Filter                                    | Index Type   |
 | --------------------------------------------------------------- | ----------------------------------------- | ------------ |
 | Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
 | Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
 | List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |
 
+### Create a scalar index
 === "Python"
 
     ```python
@@ -46,7 +58,7 @@ over scalar columns.
         await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
         ```
 
-For example, the following scan will be faster if the column `my_col` has a scalar index:
+The following scan will be faster if the column `book_id` has a scalar index:
 
 === "Python"
 
@@ -106,3 +118,30 @@ Scalar indices can also speed up scans containing a vector search or full text s
           .limit(10)
           .toArray();
         ```
+### Update a scalar index
+Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
+=== "Python"
+
+    ```python
+    table.add([{"vector": [7, 8], "book_id": 4}])
+    table.optimize()
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
+    await tbl.optimize();
+    ```
+
+=== "Rust"
+
+    ```rust
+    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    tbl.add(more_data).execute().await?;
+    tbl.optimize(OptimizeAction::All).execute().await?;
+    ```
+
+!!! note
+
+    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.

docs/src/guides/storage.md

@@ -27,10 +27,13 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 
     Azure Blob Storage:
 
+    <!-- skip-test -->
     ```python
     import lancedb
     db = lancedb.connect("az://bucket/path")
     ```
+    Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
+
 
 === "TypeScript"
 
@@ -87,11 +90,6 @@ In most cases, when running in the respective cloud and permissions are set up c
 export TIMEOUT=60s
 ```
 
-!!! note "`storage_options` availability"
-
-    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
-    It is not yet supported in the Python synchronous API.
-
 If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
 
 === "Python"

docs/src/guides/tables.md

@@ -274,7 +274,7 @@ table = db.create_table(table_name, schema=Content)
 
 Sometimes your data model may contain nested objects.
 For example, you may want to store the document string
-and the document soure name as a nested Document object:
+and the document source name as a nested Document object:
 
 ```python
 class Document(BaseModel):
@@ -466,7 +466,7 @@ You can create an empty table for scenarios where you want to add data to the ta
 
 ## Adding to a table
 
-After a table has been created, you can always add more data to it usind the `add` method
+After a table has been created, you can always add more data to it using the `add` method
 
 === "Python"
     You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or `Iterator[pa.RecordBatch]`. Below are some examples.
@@ -535,7 +535,7 @@ After a table has been created, you can always add more data to it usind the `ad
     ```
 
     ??? "Ingesting Pydantic models with LanceDB embedding API"
-        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` feild as None to allow LanceDB to automatically vectorize the data.
+        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` field as None to allow LanceDB to automatically vectorize the data.
 
         ```python
         import lancedb
@@ -790,6 +790,122 @@ Use the `drop_table()` method on the database to remove a table.
       This permanently removes the table and is not recoverable, unlike deleting rows.
       If the table does not exist an exception is raised.
 
+## Changing schemas
+
+While tables must have a schema specified when they are created, you can
+change the schema over time. There's three methods to alter the schema of
+a table:
+
+* `add_columns`: Add new columns to the table
+* `alter_columns`: Alter the name, nullability, or data type of a column
+* `drop_columns`: Drop columns from the table
+
+### Adding new columns
+
+You can add new columns to the table with the `add_columns` method. New columns
+are filled with values based on a SQL expression. For example, you can add a new
+column `y` to the table and fill it with the value of `x + 1`.
+
+=== "Python"
+
+    ```python
+    table.add_columns({"double_price": "price * 2"})
+    ```
+    **API Reference:** [lancedb.table.Table.add_columns][]
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/basic.test.ts:add_columns"
+    ```
+    **API Reference:** [lancedb.Table.addColumns](../js/classes/Table.md/#addcolumns)
+
+If you want to fill it with null, you can use `cast(NULL as <data_type>)` as
+the SQL expression to fill the column with nulls, while controlling the data
+type of the column. Available data types are base on the
+[DataFusion data types](https://datafusion.apache.org/user-guide/sql/data_types.html).
+You can use any of the SQL types, such as `BIGINT`:
+
+```sql
+cast(NULL as BIGINT)
+```
+
+Using Arrow data types and the `arrow_typeof` function is not yet supported.
+
+<!-- TODO: we could provide a better formula for filling with nulls:
+   https://github.com/lancedb/lance/issues/3175
+-->
+
+### Altering existing columns
+
+You can alter the name, nullability, or data type of a column with the `alter_columns`
+method.
+
+Changing the name or nullability of a column just updates the metadata. Because
+of this, it's a fast operation. Changing the data type of a column requires
+rewriting the column, which can be a heavy operation.
+
+=== "Python"
+
+    ```python
+    import pyarrow as pa
+    table.alter_column({"path": "double_price", "rename": "dbl_price",
+                        "data_type": pa.float32(), "nullable": False})
+    ```
+    **API Reference:** [lancedb.table.Table.alter_columns][]
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/basic.test.ts:alter_columns"
+    ```
+    **API Reference:** [lancedb.Table.alterColumns](../js/classes/Table.md/#altercolumns)
+
+### Dropping columns
+
+You can drop columns from the table with the `drop_columns` method. This will
+will remove the column from the schema.
+
+<!-- TODO: Provide guidance on how to reduce disk usage once optimize helps here
+    waiting on: https://github.com/lancedb/lance/issues/3177
+-->
+
+=== "Python"
+
+    ```python
+    table.drop_columns(["dbl_price"])
+    ```
+    **API Reference:** [lancedb.table.Table.drop_columns][]
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "nodejs/examples/basic.test.ts:drop_columns"
+    ```
+    **API Reference:** [lancedb.Table.dropColumns](../js/classes/Table.md/#altercolumns)
+
+
+## Handling bad vectors
+
+In LanceDB Python, you can use the `on_bad_vectors` parameter to choose how
+invalid vector values are handled. Invalid vectors are vectors that are not valid
+because:
+
+1. They are the wrong dimension
+2. They contain NaN values
+3. They are null but are on a non-nullable field
+
+By default, LanceDB will raise an error if it encounters a bad vector. You can
+also choose one of the following options:
+
+* `drop`: Ignore rows with bad vectors
+* `fill`: Replace bad values (NaNs) or missing values (too few dimensions) with
+    the fill value specified in the `fill_value` parameter. An input like
+    `[1.0, NaN, 3.0]` will be replaced with `[1.0, 0.0, 3.0]` if `fill_value=0.0`.
+* `null`: Replace bad vectors with null (only works if the column is nullable).
+    A bad vector `[1.0, NaN, 3.0]` will be replaced with `null` if the column is
+    nullable. If the vector column is non-nullable, then bad vectors will cause an
+    error
 
 ## Consistency
 
@@ -859,4 +975,4 @@ There are three possible settings for `read_consistency_interval`:
 
 Learn the best practices on creating an ANN index and getting the most out of it.
 
-[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](../migration.md) for more information.

docs/src/python/python.md

@@ -1,6 +1,16 @@
 # Python API Reference
 
-This section contains the API reference for the OSS Python API.
+This section contains the API reference for the Python API. There is a
+synchronous and an asynchronous API client.
+
+The general flow of using the API is:
+
+1. Use [lancedb.connect][] or [lancedb.connect_async][] to connect to a database.
+2. Use the returned [lancedb.DBConnection][] or [lancedb.AsyncConnection][] to
+   create or open tables.
+3. Use the returned [lancedb.table.Table][] or [lancedb.AsyncTable][] to query
+   or modify tables.
+
 
 ## Installation
 

docs/src/reranking/cohere.md

@@ -6,6 +6,9 @@ This re-ranker uses the [Cohere](https://cohere.ai/) API to rerank the search re
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
+```shell
+pip install cohere
+```
 
 ```python
 import numpy

docs/src/reranking/index.md

@@ -9,6 +9,7 @@ LanceDB comes with some built-in rerankers. Some of the rerankers that are avail
 | `CrossEncoderReranker` | Uses a cross-encoder model to rerank search results | Vector, FTS, Hybrid |
 | `ColbertReranker` | Uses a colbert model to rerank search results | Vector, FTS, Hybrid |
 | `OpenaiReranker`(Experimental) | Uses OpenAI's chat model to rerank search results | Vector, FTS, Hybrid |
+| `VoyageAIReranker` | Uses voyageai Reranker API to rerank results | Vector, FTS, Hybrid |
 
 
 ## Using a Reranker
@@ -73,6 +74,7 @@ LanceDB comes with some built-in rerankers. Here are some of the rerankers that
 - [Jina Reranker](./jina.md)
 - [AnswerDotAI Rerankers](./answerdotai.md)
 - [Reciprocal Rank Fusion Reranker](./rrf.md)
+- [VoyageAI Reranker](./voyageai.md)
 
 ## Creating Custom Rerankers
 

docs/src/sql.md

@@ -7,6 +7,10 @@ performed on the top-k results returned by the vector search. However, pre-filte
 option that performs the filter prior to vector search. This can be useful to narrow down on
 the search space on a very large dataset to reduce query latency.
 
+Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to set `use_index=false` if you know that the filter is highly selective.
+
+Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain.
+
 <!-- Setup Code
 ```python
 import lancedb
@@ -57,6 +61,9 @@ const tbl = await db.createTable('myVectors', data)
         ```ts
         --8<-- "docs/src/sql_legacy.ts:search"
         ```
+!!! note
+
+    Creating a [scalar index](guides/scalar_index.md) accelerates filtering
 
 ## SQL filters
 

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.13.0-beta.1</version>
+        <version>0.14.0-final.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.13.0-beta.1</version>
+    <version>0.14.0-final.0</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.13.0-beta.1",
+  "version": "0.14.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.13.0-beta.1",
+      "version": "0.14.0",
       "cpu": [
         "x64",
         "arm64"
@@ -52,12 +52,14 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
-        "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
-        "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.13.0-beta.1",
-        "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1"
+        "@lancedb/vectordb-darwin-arm64": "0.14.0",
+        "@lancedb/vectordb-darwin-x64": "0.14.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.14.0",
+        "@lancedb/vectordb-linux-arm64-musl": "0.14.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.14.0",
+        "@lancedb/vectordb-linux-x64-musl": "0.14.0",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.14.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.14.0"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -328,9 +330,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.13.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.13.0-beta.1.tgz",
-      "integrity": "sha512-beOrf6selCzzhLgDG8Nibma4nO/CSnA1wUKRmlJHEPtGcg7PW18z6MP/nfwQMpMR/FLRfTo8pPTbpzss47MiQQ==",
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.14.0.tgz",
+      "integrity": "sha512-C8wp+eJQY3RMLIRfxDnOm8bYg458OI3Cz7Jh7ws6ibquBdJDCiTdwFfcUXrkoaQ9Wv4nHZOEqupj3FBMsks1hw==",
       "cpu": [
         "arm64"
       ],
@@ -340,9 +342,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.13.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.13.0-beta.1.tgz",
-      "integrity": "sha512-YdraGRF/RbJRkKh0v3xT03LUhq47T2GtCvJ5gZp8wKlh4pHa8LuhLU0DIdvmG/DT5vuQA+td8HDkBm/e3EOdNg==",
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.14.0.tgz",
+      "integrity": "sha512-5jkQuEVGaPViFb4dOjncUqVCbvEiT8XYFZoprE0yv7HUUCdt5v15GTNxey72yw+aaX2mdb2CeFIs+4ySZqy/MA==",
       "cpu": [
         "x64"
       ],
@@ -352,9 +354,21 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.13.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.13.0-beta.1.tgz",
-      "integrity": "sha512-Pp0O/uhEqof1oLaWrNbv+Ym+q8kBkiCqaA5+2eAZ6a3e9U+Ozkvb0FQrHuyi9adJ5wKQ4NabyQE9BMf2bYpOnQ==",
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.14.0.tgz",
+      "integrity": "sha512-YLboFJLQyFzsYWi2iW1nr2SGaZTaj4gERIufyTSnX+VXlEYKHke3cMFLF+EamH8eejv2HwXdJpidPaP6aSzujw==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-musl": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-musl/-/vectordb-linux-arm64-musl-0.14.0.tgz",
+      "integrity": "sha512-rel/SaxGRtx5GdAkFH1IknBr0V/tbrN4jYT6FixmSvgc9kgxrMGlBUHSRAO5atdRXZ8jT7XWuOqW1QdgsmPi0g==",
       "cpu": [
         "arm64"
       ],
@@ -364,9 +378,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.13.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.13.0-beta.1.tgz",
-      "integrity": "sha512-y8nxOye4egfWF5FGED9EfkmZ1O5HnRLU4a61B8m5JSpkivO9v2epTcbYN0yt/7ZFCgtqMfJ8VW4Mi7qQcz3KDA==",
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.14.0.tgz",
+      "integrity": "sha512-N29n8OO2JqSPaSVd5gmyh6r4x6LX0qpcCHrhkEaRoKKIXYdHQ8sAHOqHNt3xhMDLwDJfjGmzAwd977cOYM5MBw==",
       "cpu": [
         "x64"
       ],
@@ -375,10 +389,34 @@
         "linux"
       ]
     },
+    "node_modules/@lancedb/vectordb-linux-x64-musl": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-musl/-/vectordb-linux-x64-musl-0.14.0.tgz",
+      "integrity": "sha512-36Ewl9M6IsYgxBIaThgqaSlQ++8YsSnZB85DOnuIds+sRbBfNkknvwBRFO1/FGN8RSBydFPy1irNFmCOnrlTZg==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-win32-arm64-msvc": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-arm64-msvc/-/vectordb-win32-arm64-msvc-0.14.0.tgz",
+      "integrity": "sha512-4qsna5yI7umGEA868/ifr1Np66d0dhFAOIGaJKS5Z+Zm4Zplr42BjVZiNWtwwKhndtsiPJnFCYVYRKfjTLZWdg==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.13.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.13.0-beta.1.tgz",
-      "integrity": "sha512-STMDP9dp0TBLkB3ro+16pKcGy6bmbhRuEZZZ1Tp5P75yTPeVh4zIgWkidMdU1qBbEYM7xacnsp9QAwgLnMU/Ow==",
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.14.0.tgz",
+      "integrity": "sha512-1u+J5WFClNc6mzgF5otevMnOxW3pj8yOHrPoIiZe9SrL8O2oVtdYfWJZYG/OST21cS0Mc4Z0upX86G0sA4kEfA==",
       "cpu": [
         "x64"
       ],
@@ -1501,9 +1539,9 @@
       "dev": true
     },
     "node_modules/cross-spawn": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
       "dev": true,
       "dependencies": {
         "path-key": "^3.1.0",

node/package.json

@@ -1,7 +1,8 @@
 {
   "name": "vectordb",
-  "version": "0.13.0-beta.1",
+  "version": "0.14.0",
   "description": " Serverless, low-latency vector database for AI applications",
+  "private": false,
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
@@ -84,16 +85,20 @@
       "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
       "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
       "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
+      "x86_64-unknown-linux-musl": "@lancedb/vectordb-linux-x64-musl",
+      "aarch64-unknown-linux-musl": "@lancedb/vectordb-linux-arm64-musl",
       "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc",
       "aarch64-pc-windows-msvc": "@lancedb/vectordb-win32-arm64-msvc"
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
-    "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
-    "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
-    "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.13.0-beta.1"
+    "@lancedb/vectordb-darwin-x64": "0.14.0",
+    "@lancedb/vectordb-darwin-arm64": "0.14.0",
+    "@lancedb/vectordb-linux-x64-gnu": "0.14.0",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.14.0",
+    "@lancedb/vectordb-linux-x64-musl": "0.14.0",
+    "@lancedb/vectordb-linux-arm64-musl": "0.14.0",
+    "@lancedb/vectordb-win32-x64-msvc": "0.14.0",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.14.0"
   }
 }

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.13.0-beta.1"
+version = "0.14.0"
 license.workspace = true
 description.workspace = true
 repository.workspace = true

nodejs/__test__/connection.test.ts

@@ -110,7 +110,10 @@ describe("given a connection", () => {
     let table = await db.createTable("test", data, { useLegacyFormat: true });
 
     const isV2 = async (table: Table) => {
-      const data = await table.query().toArrow({ maxBatchLength: 100000 });
+      const data = await table
+        .query()
+        .limit(10000)
+        .toArrow({ maxBatchLength: 100000 });
       console.log(data.batches.length);
       return data.batches.length < 5;
     };

nodejs/__test__/table.test.ts

@@ -187,6 +187,81 @@ describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
       },
     );
 
+    // TODO: https://github.com/lancedb/lancedb/issues/1832
+    it.skip("should be able to omit nullable fields", async () => {
+      const db = await connect(tmpDir.name);
+      const schema = new arrow.Schema([
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(
+            2,
+            new arrow.Field("item", new arrow.Float64()),
+          ),
+          true,
+        ),
+        new arrow.Field("item", new arrow.Utf8(), true),
+        new arrow.Field("price", new arrow.Float64(), false),
+      ]);
+      const table = await db.createEmptyTable("test", schema);
+
+      const data1 = { item: "foo", price: 10.0 };
+      await table.add([data1]);
+      const data2 = { vector: [3.1, 4.1], price: 2.0 };
+      await table.add([data2]);
+      const data3 = { vector: [5.9, 26.5], item: "bar", price: 3.0 };
+      await table.add([data3]);
+
+      let res = await table.query().limit(10).toArray();
+      const resVector = res.map((r) => r.get("vector").toArray());
+      expect(resVector).toEqual([null, data2.vector, data3.vector]);
+      const resItem = res.map((r) => r.get("item").toArray());
+      expect(resItem).toEqual(["foo", null, "bar"]);
+      const resPrice = res.map((r) => r.get("price").toArray());
+      expect(resPrice).toEqual([10.0, 2.0, 3.0]);
+
+      const data4 = { item: "foo" };
+      // We can't omit a column if it's not nullable
+      await expect(table.add([data4])).rejects.toThrow("Invalid user input");
+
+      // But we can alter columns to make them nullable
+      await table.alterColumns([{ path: "price", nullable: true }]);
+      await table.add([data4]);
+
+      res = (await table.query().limit(10).toArray()).map((r) => r.toJSON());
+      expect(res).toEqual([data1, data2, data3, data4]);
+    });
+
+    it("should be able to insert nullable data for non-nullable fields", async () => {
+      const db = await connect(tmpDir.name);
+      const schema = new arrow.Schema([
+        new arrow.Field("x", new arrow.Float64(), false),
+        new arrow.Field("id", new arrow.Utf8(), false),
+      ]);
+      const table = await db.createEmptyTable("test", schema);
+
+      const data1 = { x: 4.1, id: "foo" };
+      await table.add([data1]);
+      const res = (await table.query().toArray())[0];
+      expect(res.x).toEqual(data1.x);
+      expect(res.id).toEqual(data1.id);
+
+      const data2 = { x: null, id: "bar" };
+      await expect(table.add([data2])).rejects.toThrow(
+        "declared as non-nullable but contains null values",
+      );
+
+      // But we can alter columns to make them nullable
+      await table.alterColumns([{ path: "x", nullable: true }]);
+      await table.add([data2]);
+
+      const res2 = await table.query().toArray();
+      expect(res2.length).toBe(2);
+      expect(res2[0].x).toEqual(data1.x);
+      expect(res2[0].id).toEqual(data1.id);
+      expect(res2[1].x).toBeNull();
+      expect(res2[1].id).toEqual(data2.id);
+    });
+
     it("should return the table as an instance of an arrow table", async () => {
       const arrowTbl = await table.toArrow();
       expect(arrowTbl).toBeInstanceOf(ArrowTable);
@@ -402,6 +477,54 @@ describe("When creating an index", () => {
     expect(rst.numRows).toBe(1);
   });
 
+  it("should create and search IVF_HNSW indices", async () => {
+    await tbl.createIndex("vec", {
+      config: Index.hnswSq(),
+    });
+
+    // check index directory
+    const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
+    expect(fs.readdirSync(indexDir)).toHaveLength(1);
+    const indices = await tbl.listIndices();
+    expect(indices.length).toBe(1);
+    expect(indices[0]).toEqual({
+      name: "vec_idx",
+      indexType: "IvfHnswSq",
+      columns: ["vec"],
+    });
+
+    // Search without specifying the column
+    let rst = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(queryVec)
+      .distanceType("dot")
+      .toArrow();
+    expect(rst.numRows).toBe(2);
+
+    // Search using `vectorSearch`
+    rst = await tbl.vectorSearch(queryVec).limit(2).toArrow();
+    expect(rst.numRows).toBe(2);
+
+    // Search with specifying the column
+    const rst2 = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(queryVec)
+      .column("vec")
+      .toArrow();
+    expect(rst2.numRows).toBe(2);
+    expect(rst.toString()).toEqual(rst2.toString());
+
+    // test offset
+    rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
+    expect(rst.numRows).toBe(1);
+
+    // test ef
+    rst = await tbl.query().limit(2).nearestTo(queryVec).ef(100).toArrow();
+    expect(rst.numRows).toBe(2);
+  });
+
   it("should be able to query unindexed data", async () => {
     await tbl.createIndex("vec");
     await tbl.add([
@@ -462,11 +585,11 @@ describe("When creating an index", () => {
     expect(fs.readdirSync(indexDir)).toHaveLength(1);
 
     for await (const r of tbl.query().where("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(298);
+      expect(r.numRows).toBe(10);
     }
     // should also work with 'filter' alias
     for await (const r of tbl.query().filter("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(298);
+      expect(r.numRows).toBe(10);
     }
   });
 
@@ -702,6 +825,18 @@ describe("schema evolution", function () {
       new Field("price", new Float64(), true),
     ]);
     expect(await table.schema()).toEqual(expectedSchema);
+
+    await table.alterColumns([{ path: "new_id", dataType: "int32" }]);
+    const expectedSchema2 = new Schema([
+      new Field("new_id", new Int32(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32(), true)),
+        true,
+      ),
+      new Field("price", new Float64(), true),
+    ]);
+    expect(await table.schema()).toEqual(expectedSchema2);
   });
 
   it("can drop a column from the schema", async function () {

nodejs/examples/basic.test.ts

@@ -116,6 +116,26 @@ test("basic table examples", async () => {
       await tbl.add(data);
       // --8<-- [end:add_data]
     }
+
+    {
+      // --8<-- [start:add_columns]
+      await tbl.addColumns([{ name: "double_price", valueSql: "price * 2" }]);
+      // --8<-- [end:add_columns]
+      // --8<-- [start:alter_columns]
+      await tbl.alterColumns([
+        {
+          path: "double_price",
+          rename: "dbl_price",
+          dataType: "float",
+          nullable: true,
+        },
+      ]);
+      // --8<-- [end:alter_columns]
+      // --8<-- [start:drop_columns]
+      await tbl.dropColumns(["dbl_price"]);
+      // --8<-- [end:drop_columns]
+    }
+
     {
       // --8<-- [start:vector_search]
       const res = await tbl.search([100, 100]).limit(2).toArray();

nodejs/examples/sentence-transformers.test.ts

@@ -6,12 +6,16 @@ import { withTempDirectory } from "./util.ts";
 import * as lancedb from "@lancedb/lancedb";
 import "@lancedb/lancedb/embedding/transformers";
 import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
 import { Utf8 } from "apache-arrow";
 
 test("full text search", async () => {
   await withTempDirectory(async (databaseDir) => {
     const db = await lancedb.connect(databaseDir);
-    const func = await getRegistry().get("huggingface").create();
+    console.log(getRegistry());
+    const func = (await getRegistry()
+      .get("huggingface")
+      ?.create()) as EmbeddingFunction;
 
     const facts = [
       "Albert Einstein was a theoretical physicist.",
@@ -56,4 +60,4 @@ test("full text search", async () => {
 
     expect(actual[0]["text"]).toBe("The human body has 206 bones.");
   });
-});
+}, 100_000);

nodejs/lancedb/embedding/index.ts

@@ -19,9 +19,6 @@ import { EmbeddingFunctionConfig, getRegistry } from "./registry";
 
 export { EmbeddingFunction, TextEmbeddingFunction } from "./embedding_function";
 
-// We need to explicitly export '*' so that the `register` decorator actually registers the class.
-export * from "./openai";
-export * from "./transformers";
 export * from "./registry";
 
 /**

nodejs/lancedb/embedding/registry.ts

@@ -17,8 +17,6 @@ import {
   type EmbeddingFunctionConstructor,
 } from "./embedding_function";
 import "reflect-metadata";
-import { OpenAIEmbeddingFunction } from "./openai";
-import { TransformersEmbeddingFunction } from "./transformers";
 
 type CreateReturnType<T> = T extends { init: () => Promise<void> }
   ? Promise<T>
@@ -73,10 +71,6 @@ export class EmbeddingFunctionRegistry {
     };
   }
 
-  get(name: "openai"): EmbeddingFunctionCreate<OpenAIEmbeddingFunction>;
-  get(
-    name: "huggingface",
-  ): EmbeddingFunctionCreate<TransformersEmbeddingFunction>;
   get<T extends EmbeddingFunction<unknown>>(
     name: string,
   ): EmbeddingFunctionCreate<T> | undefined;

nodejs/lancedb/query.ts

@@ -385,6 +385,20 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
     return this;
   }
 
+  /**
+   * Set the number of candidates to consider during the search
+   *
+   * This argument is only used when the vector column has an HNSW index.
+   * If there is no index then this value is ignored.
+   *
+   * Increasing this value will increase the recall of your query but will
+   * also increase the latency of your query. The default value is 1.5*limit.
+   */
+  ef(ef: number): VectorQuery {
+    super.doCall((inner) => inner.ef(ef));
+    return this;
+  }
+
   /**
    * Set the vector column to query
    *

nodejs/lancedb/table.ts

@@ -87,6 +87,12 @@ export interface OptimizeOptions {
   deleteUnverified: boolean;
 }
 
+export interface Version {
+  version: number;
+  timestamp: Date;
+  metadata: Record<string, string>;
+}
+
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -360,6 +366,11 @@ export abstract class Table {
    */
   abstract checkoutLatest(): Promise<void>;
 
+  /**
+   * List all the versions of the table
+   */
+  abstract listVersions(): Promise<Version[]>;
+
   /**
    * Restore the table to the currently checked out version
    *
@@ -659,6 +670,14 @@ export class LocalTable extends Table {
     await this.inner.checkoutLatest();
   }
 
+  async listVersions(): Promise<Version[]> {
+    return (await this.inner.listVersions()).map((version) => ({
+      version: version.version,
+      timestamp: new Date(version.timestamp / 1000),
+      metadata: version.metadata,
+    }));
+  }
+
   async restore(): Promise<void> {
     await this.inner.restore();
   }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.13.0-beta.1",
+	"version": "0.14.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.13.0-beta.1",
+	"version": "0.14.0",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.13.0-beta.1",
+	"version": "0.14.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/README.md

@@ -0,0 +1,3 @@
+# `@lancedb/lancedb-linux-arm64-musl`
+
+This is the **aarch64-unknown-linux-musl** binary for `@lancedb/lancedb`

nodejs/npm/linux-arm64-musl/package.json

@@ -0,0 +1,13 @@
+{
+	"name": "@lancedb/lancedb-linux-arm64-musl",
+	"version": "0.14.0",
+	"os": ["linux"],
+	"cpu": ["arm64"],
+	"main": "lancedb.linux-arm64-musl.node",
+	"files": ["lancedb.linux-arm64-musl.node"],
+	"license": "Apache 2.0",
+	"engines": {
+		"node": ">= 18"
+	},
+	"libc": ["musl"]
+}

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.13.0-beta.1",
+	"version": "0.14.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/README.md

@@ -0,0 +1,3 @@
+# `@lancedb/lancedb-linux-x64-musl`
+
+This is the **x86_64-unknown-linux-musl** binary for `@lancedb/lancedb`

nodejs/npm/linux-x64-musl/package.json

@@ -0,0 +1,13 @@
+{
+	"name": "@lancedb/lancedb-linux-x64-musl",
+	"version": "0.14.0",
+	"os": ["linux"],
+	"cpu": ["x64"],
+	"main": "lancedb.linux-x64-musl.node",
+	"files": ["lancedb.linux-x64-musl.node"],
+	"license": "Apache 2.0",
+	"engines": {
+		"node": ">= 18"
+	},
+	"libc": ["musl"]
+}

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.13.0-beta.1",
+  "version": "0.14.0",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.13.0-beta.1",
+	"version": "0.14.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.13.0-beta.1",
+  "version": "0.14.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.13.0-beta.1",
+      "version": "0.14.0",
       "cpu": [
         "x64",
         "arm64"
@@ -6052,9 +6052,9 @@
       }
     },
     "node_modules/cross-spawn": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
       "devOptional": true,
       "dependencies": {
         "path-key": "^3.1.0",

nodejs/package.json

@@ -10,11 +10,14 @@
     "vector database",
     "ann"
   ],
-  "version": "0.13.0-beta.1",
+  "private": false,
+  "version": "0.14.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
-    "./embedding": "./dist/embedding/index.js"
+    "./embedding": "./dist/embedding/index.js",
+    "./embedding/openai": "./dist/embedding/openai.js",
+    "./embedding/transformers": "./dist/embedding/transformers.js"
   },
   "types": "dist/index.d.ts",
   "napi": {
@@ -22,11 +25,14 @@
     "triples": {
       "defaults": false,
       "additional": [
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
         "x86_64-apple-darwin",
+        "aarch64-apple-darwin",
         "x86_64-unknown-linux-gnu",
-        "x86_64-pc-windows-msvc"
+        "aarch64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+        "aarch64-unknown-linux-musl",
+        "x86_64-pc-windows-msvc",
+        "aarch64-pc-windows-msvc"
       ]
     }
   },

nodejs/src/query.rs

@@ -167,6 +167,11 @@ impl VectorQuery {
         self.inner = self.inner.clone().nprobes(nprobe as usize);
     }
 
+    #[napi]
+    pub fn ef(&mut self, ef: u32) {
+        self.inner = self.inner.clone().ef(ef as usize);
+    }
+
     #[napi]
     pub fn bypass_vector_index(&mut self) {
         self.inner = self.inner.clone().bypass_vector_index()

nodejs/src/table.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+
 use arrow_ipc::writer::FileWriter;
 use lancedb::ipc::ipc_file_to_batches;
 use lancedb::table::{
@@ -176,16 +178,20 @@ impl Table {
     #[napi(catch_unwind)]
     pub async fn alter_columns(&self, alterations: Vec<ColumnAlteration>) -> napi::Result<()> {
         for alteration in &alterations {
-            if alteration.rename.is_none() && alteration.nullable.is_none() {
+            if alteration.rename.is_none()
+                && alteration.nullable.is_none()
+                && alteration.data_type.is_none()
+            {
                 return Err(napi::Error::from_reason(
-                    "Alteration must have a 'rename' or 'nullable' field.",
+                    "Alteration must have a 'rename', 'dataType', or 'nullable' field.",
                 ));
             }
         }
         let alterations = alterations
             .into_iter()
-            .map(LanceColumnAlteration::from)
-            .collect::<Vec<_>>();
+            .map(LanceColumnAlteration::try_from)
+            .collect::<std::result::Result<Vec<_>, String>>()
+            .map_err(napi::Error::from_reason)?;
 
         self.inner_ref()?
             .alter_columns(&alterations)
@@ -226,6 +232,28 @@ impl Table {
         self.inner_ref()?.checkout_latest().await.default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn list_versions(&self) -> napi::Result<Vec<Version>> {
+        self.inner_ref()?
+            .list_versions()
+            .await
+            .map(|versions| {
+                versions
+                    .iter()
+                    .map(|version| Version {
+                        version: version.version as i64,
+                        timestamp: version.timestamp.timestamp_micros(),
+                        metadata: version
+                            .metadata
+                            .iter()
+                            .map(|(k, v)| (k.clone(), v.clone()))
+                            .collect(),
+                    })
+                    .collect()
+            })
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn restore(&self) -> napi::Result<()> {
         self.inner_ref()?.restore().await.default_error()
@@ -409,24 +437,43 @@ pub struct ColumnAlteration {
     /// The new name of the column. If not provided then the name will not be changed.
     /// This must be distinct from the names of all other columns in the table.
     pub rename: Option<String>,
+    /// A new data type for the column. If not provided then the data type will not be changed.
+    /// Changing data types is limited to casting to the same general type. For example, these
+    /// changes are valid:
+    /// * `int32` -> `int64` (integers)
+    /// * `double` -> `float` (floats)
+    /// * `string` -> `large_string` (strings)
+    /// But these changes are not:
+    /// * `int32` -> `double` (mix integers and floats)
+    /// * `string` -> `int32` (mix strings and integers)
+    pub data_type: Option<String>,
     /// Set the new nullability. Note that a nullable column cannot be made non-nullable.
     pub nullable: Option<bool>,
 }
 
-impl From<ColumnAlteration> for LanceColumnAlteration {
-    fn from(js: ColumnAlteration) -> Self {
+impl TryFrom<ColumnAlteration> for LanceColumnAlteration {
+    type Error = String;
+    fn try_from(js: ColumnAlteration) -> std::result::Result<Self, Self::Error> {
         let ColumnAlteration {
             path,
             rename,
             nullable,
+            data_type,
         } = js;
-        Self {
+        let data_type = if let Some(data_type) = data_type {
+            Some(
+                lancedb::utils::string_to_datatype(&data_type)
+                    .ok_or_else(|| format!("Invalid data type: {}", data_type))?,
+            )
+        } else {
+            None
+        };
+        Ok(Self {
             path,
             rename,
             nullable,
-            // TODO: wire up this field
-            data_type: None,
-        }
+            data_type,
+        })
     }
 }
 
@@ -466,3 +513,10 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
         }
     }
 }
+
+#[napi(object)]
+pub struct Version {
+    pub version: i64,
+    pub timestamp: i64,
+    pub metadata: HashMap<String, String>,
+}

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.0-beta.1"
+current_version = "0.17.1-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.16.0-beta.1"
+version = "0.17.1-beta.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -14,25 +14,30 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "52.1", features = ["pyarrow"] }
-lancedb = { path = "../rust/lancedb" }
+arrow = { version = "53.2", features = ["pyarrow"] }
+lancedb = { path = "../rust/lancedb", default-features = false }
 env_logger.workspace = true
-pyo3 = { version = "0.21", features = ["extension-module", "abi3-py38", "gil-refs"] }
-# Using this fork for now: https://github.com/awestlake87/pyo3-asyncio/issues/119
-# pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
-pyo3-asyncio-0-21 = { version = "0.21.0", features = ["attributes", "tokio-runtime"] }
-
+pyo3 = { version = "0.22.2", features = [
+    "extension-module",
+    "abi3-py39",
+    "gil-refs"
+] }
+pyo3-async-runtimes = { version = "0.22", features = ["attributes", "tokio-runtime"] }
 pin-project = "1.1.5"
 futures.workspace = true
-tokio = { version = "1.36.0", features = ["sync"] }
+tokio = { version = "1.40", features = ["sync"] }
 
 [build-dependencies]
 pyo3-build-config = { version = "0.20.3", features = [
     "extension-module",
-    "abi3-py38",
+    "abi3-py39",
 ] }
 
 [features]
-default = ["remote"]
+default = ["default-tls", "remote"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]
+# TLS
+default-tls = ["lancedb/default-tls"]
+native-tls = ["lancedb/native-tls"]
+rustls-tls = ["lancedb/rustls-tls"]

python/pyproject.toml

@@ -3,8 +3,7 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
     "deprecation",
-    "nest-asyncio~=1.0",
-    "pylance==0.19.2",
+    "pylance==0.20.0",
     "tqdm>=4.27.0",
     "pydantic>=1.10",
     "packaging",
@@ -31,7 +30,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",

python/python/lancedb/__init__.py

@@ -36,6 +36,7 @@ def connect(
     read_consistency_interval: Optional[timedelta] = None,
     request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
     client_config: Union[ClientConfig, Dict[str, Any], None] = None,
+    storage_options: Optional[Dict[str, str]] = None,
     **kwargs: Any,
 ) -> DBConnection:
     """Connect to a LanceDB database.
@@ -67,6 +68,9 @@ def connect(
         Configuration options for the LanceDB Cloud HTTP client. If a dict, then
         the keys are the attributes of the ClientConfig class. If None, then the
         default configuration is used.
+    storage_options: dict, optional
+        Additional options for the storage backend. See available options at
+        https://lancedb.github.io/lancedb/guides/storage/
 
     Examples
     --------
@@ -106,12 +110,17 @@ def connect(
             # TODO: remove this (deprecation warning downstream)
             request_thread_pool=request_thread_pool,
             client_config=client_config,
+            storage_options=storage_options,
             **kwargs,
         )
 
     if kwargs:
         raise ValueError(f"Unknown keyword arguments: {kwargs}")
-    return LanceDBConnection(uri, read_consistency_interval=read_consistency_interval)
+    return LanceDBConnection(
+        uri,
+        read_consistency_interval=read_consistency_interval,
+        storage_options=storage_options,
+    )
 
 
 async def connect_async(

python/python/lancedb/_lancedb.pyi

@@ -79,9 +79,21 @@ class Query:
     def limit(self, limit: int): ...
     def offset(self, offset: int): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
-    def nearest_to_text(self, query: dict) -> Query: ...
+    def nearest_to_text(self, query: dict) -> FTSQuery: ...
     async def execute(self, max_batch_legnth: Optional[int]) -> RecordBatchStream: ...
 
+class FTSQuery:
+    def where(self, filter: str): ...
+    def select(self, columns: List[str]): ...
+    def limit(self, limit: int): ...
+    def offset(self, offset: int): ...
+    def fast_search(self): ...
+    def with_row_id(self): ...
+    def postfilter(self): ...
+    def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
+    async def execute(self, max_batch_length: Optional[int]) -> RecordBatchStream: ...
+    async def explain_plan(self) -> str: ...
+
 class VectorQuery:
     async def execute(self) -> RecordBatchStream: ...
     def where(self, filter: str): ...
@@ -95,6 +107,24 @@ class VectorQuery:
     def refine_factor(self, refine_factor: int): ...
     def nprobes(self, nprobes: int): ...
     def bypass_vector_index(self): ...
+    def nearest_to_text(self, query: dict) -> HybridQuery: ...
+
+class HybridQuery:
+    def where(self, filter: str): ...
+    def select(self, columns: List[str]): ...
+    def limit(self, limit: int): ...
+    def offset(self, offset: int): ...
+    def fast_search(self): ...
+    def with_row_id(self): ...
+    def postfilter(self): ...
+    def distance_type(self, distance_type: str): ...
+    def refine_factor(self, refine_factor: int): ...
+    def nprobes(self, nprobes: int): ...
+    def bypass_vector_index(self): ...
+    def to_vector_query(self) -> VectorQuery: ...
+    def to_fts_query(self) -> FTSQuery: ...
+    def get_limit(self) -> int: ...
+    def get_with_row_id(self) -> bool: ...
 
 class CompactionStats:
     fragments_removed: int

python/python/lancedb/background_loop.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import asyncio
+import threading
+
+
+class BackgroundEventLoop:
+    """
+    A background event loop that can run futures.
+
+    Used to bridge sync and async code, without messing with users event loops.
+    """
+
+    def __init__(self):
+        self.loop = asyncio.new_event_loop()
+        self.thread = threading.Thread(
+            target=self.loop.run_forever,
+            name="LanceDBBackgroundEventLoop",
+            daemon=True,
+        )
+        self.thread.start()
+
+    def run(self, future):
+        return asyncio.run_coroutine_threadsafe(future, self.loop).result()

python/python/lancedb/db.py

@@ -13,34 +13,29 @@
 
 from __future__ import annotations
 
-import asyncio
-import os
 from abc import abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Union
 
-import pyarrow as pa
 from overrides import EnforceOverrides, override
-from pyarrow import fs
 
-from lancedb.common import data_to_reader, validate_schema
+from lancedb.common import data_to_reader, sanitize_uri, validate_schema
+from lancedb.background_loop import BackgroundEventLoop
 
 from ._lancedb import connect as lancedb_connect
 from .table import (
     AsyncTable,
     LanceTable,
     Table,
-    _table_path,
     sanitize_create_table,
 )
 from .util import (
-    fs_from_uri,
-    get_uri_location,
     get_uri_scheme,
     validate_table_name,
 )
 
 if TYPE_CHECKING:
+    import pyarrow as pa
     from .pydantic import LanceModel
     from datetime import timedelta
 
@@ -48,6 +43,8 @@ if TYPE_CHECKING:
     from .common import DATA, URI
     from .embeddings import EmbeddingFunctionConfig
 
+LOOP = BackgroundEventLoop()
+
 
 class DBConnection(EnforceOverrides):
     """An active LanceDB connection interface."""
@@ -180,6 +177,7 @@ class DBConnection(EnforceOverrides):
         control over how data is saved, either provide the PyArrow schema to
         convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
 
+        >>> import pyarrow as pa
         >>> custom_schema = pa.schema([
         ...   pa.field("vector", pa.list_(pa.float32(), 2)),
         ...   pa.field("lat", pa.float32()),
@@ -327,7 +325,11 @@ class LanceDBConnection(DBConnection):
     """
 
     def __init__(
-        self, uri: URI, *, read_consistency_interval: Optional[timedelta] = None
+        self,
+        uri: URI,
+        *,
+        read_consistency_interval: Optional[timedelta] = None,
+        storage_options: Optional[Dict[str, str]] = None,
     ):
         if not isinstance(uri, Path):
             scheme = get_uri_scheme(uri)
@@ -338,9 +340,27 @@ class LanceDBConnection(DBConnection):
             uri = uri.expanduser().absolute()
             Path(uri).mkdir(parents=True, exist_ok=True)
         self._uri = str(uri)
-
         self._entered = False
         self.read_consistency_interval = read_consistency_interval
+        self.storage_options = storage_options
+
+        if read_consistency_interval is not None:
+            read_consistency_interval_secs = read_consistency_interval.total_seconds()
+        else:
+            read_consistency_interval_secs = None
+
+        async def do_connect():
+            return await lancedb_connect(
+                sanitize_uri(uri),
+                None,
+                None,
+                None,
+                read_consistency_interval_secs,
+                None,
+                storage_options,
+            )
+
+        self._conn = AsyncConnection(LOOP.run(do_connect()))
 
     def __repr__(self) -> str:
         val = f"{self.__class__.__name__}({self._uri}"
@@ -364,32 +384,7 @@ class LanceDBConnection(DBConnection):
         Iterator of str.
             A list of table names.
         """
-        try:
-            asyncio.get_running_loop()
-            # User application is async.  Soon we will just tell them to use the
-            # async version.  Until then fallback to the old sync implementation.
-            try:
-                filesystem = fs_from_uri(self.uri)[0]
-            except pa.ArrowInvalid:
-                raise NotImplementedError("Unsupported scheme: " + self.uri)
-
-            try:
-                loc = get_uri_location(self.uri)
-                paths = filesystem.get_file_info(fs.FileSelector(loc))
-            except FileNotFoundError:
-                # It is ok if the file does not exist since it will be created
-                paths = []
-            tables = [
-                os.path.splitext(file_info.base_name)[0]
-                for file_info in paths
-                if file_info.extension == "lance"
-            ]
-            tables.sort()
-            return tables
-        except RuntimeError:
-            # User application is sync.  It is safe to use the async implementation
-            # under the hood.
-            return asyncio.run(self._async_get_table_names(page_token, limit))
+        return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -461,19 +456,16 @@ class LanceDBConnection(DBConnection):
             If True, ignore if the table does not exist.
         """
         try:
-            table_uri = _table_path(self.uri, name)
-            filesystem, path = fs_from_uri(table_uri)
-            filesystem.delete_dir(path)
-        except FileNotFoundError:
+            LOOP.run(self._conn.drop_table(name))
+        except ValueError as e:
             if not ignore_missing:
-                raise
+                raise e
+            if f"Table '{name}' was not found" not in str(e):
+                raise e
 
     @override
     def drop_database(self):
-        dummy_table_uri = _table_path(self.uri, "dummy")
-        uri = dummy_table_uri.removesuffix("dummy.lance")
-        filesystem, path = fs_from_uri(uri)
-        filesystem.delete_dir(path)
+        LOOP.run(self._conn.drop_database())
 
 
 class AsyncConnection(object):
@@ -689,6 +681,7 @@ class AsyncConnection(object):
         control over how data is saved, either provide the PyArrow schema to
         convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
 
+        >>> import pyarrow as pa
         >>> custom_schema = pa.schema([
         ...   pa.field("vector", pa.list_(pa.float32(), 2)),
         ...   pa.field("lat", pa.float32()),

python/python/lancedb/embeddings/openai.py

@@ -48,6 +48,9 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
     organization: Optional[str] = None
     api_key: Optional[str] = None
 
+    # Set true to use Azure OpenAI API
+    use_azure: bool = False
+
     def ndims(self):
         return self._ndims
 
@@ -83,25 +86,33 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
         """
         openai = attempt_import_or_raise("openai")
 
+        valid_texts = []
+        valid_indices = []
+        for idx, text in enumerate(texts):
+            if text:
+                valid_texts.append(text)
+                valid_indices.append(idx)
+
         # TODO retry, rate limit, token limit
         try:
-            if self.name == "text-embedding-ada-002":
-                rs = self._openai_client.embeddings.create(input=texts, model=self.name)
-            else:
-                kwargs = {
-                    "input": texts,
-                    "model": self.name,
-                }
-                if self.dim:
-                    kwargs["dimensions"] = self.dim
-                rs = self._openai_client.embeddings.create(**kwargs)
+            kwargs = {
+                "input": valid_texts,
+                "model": self.name,
+            }
+            if self.name != "text-embedding-ada-002":
+                kwargs["dimensions"] = self.dim
+
+            rs = self._openai_client.embeddings.create(**kwargs)
+            valid_embeddings = {
+                idx: v.embedding for v, idx in zip(rs.data, valid_indices)
+            }
         except openai.BadRequestError:
             logging.exception("Bad request: %s", texts)
             return [None] * len(texts)
         except Exception:
             logging.exception("OpenAI embeddings error")
             raise
-        return [v.embedding for v in rs.data]
+        return [valid_embeddings.get(idx, None) for idx in range(len(texts))]
 
     @cached_property
     def _openai_client(self):
@@ -115,4 +126,8 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
             kwargs["organization"] = self.organization
         if self.api_key:
             kwargs["api_key"] = self.api_key
-        return openai.OpenAI(**kwargs)
+
+        if self.use_azure:
+            return openai.AzureOpenAI(**kwargs)
+        else:
+            return openai.OpenAI(**kwargs)

python/python/lancedb/embeddings/voyageai.py

@@ -12,18 +12,22 @@
 #  limitations under the License.
 
 import os
-from typing import ClassVar, List, Union
+from typing import ClassVar, TYPE_CHECKING, List, Union
 
 import numpy as np
+import pyarrow as pa
 
 from ..util import attempt_import_or_raise
-from .base import TextEmbeddingFunction
+from .base import EmbeddingFunction
 from .registry import register
-from .utils import api_key_not_found_help, TEXT
+from .utils import api_key_not_found_help, IMAGES
+
+if TYPE_CHECKING:
+    import PIL
 
 
 @register("voyageai")
-class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
+class VoyageAIEmbeddingFunction(EmbeddingFunction):
     """
     An embedding function that uses the VoyageAI API
 
@@ -36,6 +40,7 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
 
             * voyage-3
             * voyage-3-lite
+            * voyage-multimodal-3
             * voyage-finance-2
             * voyage-multilingual-2
             * voyage-law-2
@@ -54,7 +59,7 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
         .create(name="voyage-3")
 
     class TextModel(LanceModel):
-        text: str = voyageai.SourceField()
+        data: str = voyageai.SourceField()
         vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
 
     data = [ { "text": "hello world" },
@@ -77,6 +82,7 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
             return 1536
         elif self.name in [
             "voyage-3",
+            "voyage-multimodal-3",
             "voyage-finance-2",
             "voyage-multilingual-2",
             "voyage-law-2",
@@ -85,19 +91,19 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
         else:
             raise ValueError(f"Model {self.name} not supported")
 
-    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
-        return self.compute_source_embeddings(query, input_type="query")
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
 
-    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
-        texts = self.sanitize_input(texts)
-        input_type = (
-            kwargs.get("input_type") or "document"
-        )  # assume source input type if not passed by `compute_query_embeddings`
-        return self.generate_embeddings(texts, input_type=input_type)
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray], *args, **kwargs
-    ) -> List[np.array]:
+    def generate_text_embeddings(self, text: str, **kwargs) -> np.ndarray:
         """
         Get the embeddings for the given texts
 
@@ -109,15 +115,55 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
 
         truncation: Optional[bool]
         """
-        VoyageAIEmbeddingFunction._init_client()
-        rs = VoyageAIEmbeddingFunction.client.embed(
-            texts=texts, model=self.name, **kwargs
-        )
+        if self.name in ["voyage-multimodal-3"]:
+            rs = VoyageAIEmbeddingFunction._get_client().multimodal_embed(
+                inputs=[[text]], model=self.name, **kwargs
+            )
+        else:
+            rs = VoyageAIEmbeddingFunction._get_client().embed(
+                texts=[text], model=self.name, **kwargs
+            )
 
-        return [emb for emb in rs.embeddings]
+        return rs.embeddings[0]
+
+    def generate_image_embedding(
+        self, image: "PIL.Image.Image", **kwargs
+    ) -> np.ndarray:
+        rs = VoyageAIEmbeddingFunction._get_client().multimodal_embed(
+            inputs=[[image]], model=self.name, **kwargs
+        )
+        return rs.embeddings[0]
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query, input_type="query")]
+        else:
+            PIL = attempt_import_or_raise("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query, input_type="query")]
+            else:
+                raise TypeError("Only text PIL images supported as query")
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        images = self.sanitize_input(images)
+        return [
+            self.generate_image_embedding(img, input_type="document") for img in images
+        ]
 
     @staticmethod
-    def _init_client():
+    def _get_client():
         if VoyageAIEmbeddingFunction.client is None:
             voyageai = attempt_import_or_raise("voyageai")
             if os.environ.get("VOYAGE_API_KEY") is None:
@@ -125,3 +171,4 @@ class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
             VoyageAIEmbeddingFunction.client = voyageai.Client(
                 os.environ["VOYAGE_API_KEY"]
             )
+        return VoyageAIEmbeddingFunction.client

python/python/lancedb/index.py

@@ -110,7 +110,16 @@ class FTS:
         remove_stop_words: bool = False,
         ascii_folding: bool = False,
     ):
-        self._inner = LanceDbIndex.fts(with_position=with_position)
+        self._inner = LanceDbIndex.fts(
+            with_position=with_position,
+            base_tokenizer=base_tokenizer,
+            language=language,
+            max_token_length=max_token_length,
+            lower_case=lower_case,
+            stem=stem,
+            remove_stop_words=remove_stop_words,
+            ascii_folding=ascii_folding,
+        )
 
 
 class HnswPq:

python/python/lancedb/integrations/pyarrow.py

@@ -0,0 +1,248 @@
+import logging
+from typing import Any, List, Optional, Tuple, Union, Literal
+
+import pyarrow as pa
+
+from ..table import Table
+
+Filter = Union[str, pa.compute.Expression]
+Keys = Union[str, List[str]]
+JoinType = Literal[
+    "left semi",
+    "right semi",
+    "left anti",
+    "right anti",
+    "inner",
+    "left outer",
+    "right outer",
+    "full outer",
+]
+
+
+class PyarrowScannerAdapter(pa.dataset.Scanner):
+    def __init__(
+        self,
+        table: Table,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        self.table = table
+        self.columns = columns
+        self.filter = filter
+        self.batch_size = batch_size
+        if batch_readahead is not None:
+            logging.debug("ignoring batch_readahead which has no lance equivalent")
+        if fragment_readahead is not None:
+            logging.debug("ignoring fragment_readahead which has no lance equivalent")
+        if fragment_scan_options is not None:
+            raise NotImplementedError("fragment_scan_options not supported")
+        if use_threads is False:
+            raise NotImplementedError("use_threads=False not supported")
+        if memory_pool is not None:
+            raise NotImplementedError("memory_pool not supported")
+
+    def count_rows(self):
+        return self.table.count_rows(self.filter)
+
+    def from_batches(self, **kwargs):
+        raise NotImplementedError
+
+    def from_dataset(self, **kwargs):
+        raise NotImplementedError
+
+    def from_fragment(self, **kwargs):
+        raise NotImplementedError
+
+    def head(self, num_rows: int):
+        return self.to_reader(limit=num_rows).read_all()
+
+    @property
+    def projected_schema(self):
+        return self.head(1).schema
+
+    def scan_batches(self):
+        return self.to_reader()
+
+    def take(self, indices: List[int]):
+        raise NotImplementedError
+
+    def to_batches(self):
+        return self.to_reader()
+
+    def to_table(self):
+        return self.to_reader().read_all()
+
+    def to_reader(self, *, limit: Optional[int] = None):
+        query = self.table.search()
+        # Disable the builtin limit
+        if limit is None:
+            num_rows = self.count_rows()
+            query.limit(num_rows)
+        elif limit <= 0:
+            raise ValueError("limit must be positive")
+        else:
+            query.limit(limit)
+        if self.columns is not None:
+            query = query.select(self.columns)
+        if self.filter is not None:
+            query = query.where(self.filter, prefilter=True)
+        return query.to_batches(batch_size=self.batch_size)
+
+
+class PyarrowDatasetAdapter(pa.dataset.Dataset):
+    def __init__(self, table: Table):
+        self.table = table
+
+    def count_rows(self, filter: Optional[Filter] = None):
+        return self.table.count_rows(filter)
+
+    def get_fragments(self, filter: Optional[Filter] = None):
+        raise NotImplementedError
+
+    def head(
+        self,
+        num_rows: int,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return self.scanner(
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        ).head(num_rows)
+
+    def join(
+        self,
+        right_dataset: Any,
+        keys: Keys,
+        right_keys: Optional[Keys] = None,
+        join_type: Optional[JoinType] = None,
+        left_suffix: Optional[str] = None,
+        right_suffix: Optional[str] = None,
+        coalesce_keys: bool = True,
+        use_threads: bool = True,
+    ):
+        raise NotImplementedError
+
+    def join_asof(
+        self,
+        right_dataset: Any,
+        on: str,
+        by: Keys,
+        tolerance: int,
+        right_on: Optional[str] = None,
+        right_by: Optional[Keys] = None,
+    ):
+        raise NotImplementedError
+
+    @property
+    def partition_expression(self):
+        raise NotImplementedError
+
+    def replace_schema(self, schema: pa.Schema):
+        raise NotImplementedError
+
+    def scanner(
+        self,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return PyarrowScannerAdapter(
+            self.table,
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        )
+
+    @property
+    def schema(self):
+        return self.table.schema
+
+    def sort_by(self, sorting: Union[str, List[Tuple[str, bool]]]):
+        raise NotImplementedError
+
+    def take(
+        self,
+        indices: List[int],
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        raise NotImplementedError
+
+    def to_batches(
+        self,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return self.scanner(
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        ).to_batches()
+
+    def to_table(
+        self,
+        columns: Optional[List[str]] = None,
+        filter: Optional[Filter] = None,
+        batch_size: Optional[int] = None,
+        batch_readahead: Optional[int] = None,
+        fragment_readahead: Optional[int] = None,
+        fragment_scan_options: Optional[Any] = None,
+        use_threads: bool = True,
+        memory_pool: Optional[Any] = None,
+    ):
+        return self.scanner(
+            columns,
+            filter,
+            batch_size,
+            batch_readahead,
+            fragment_readahead,
+            fragment_scan_options,
+            use_threads,
+            memory_pool,
+        ).to_table()

python/python/lancedb/pydantic.py

@@ -1,15 +1,5 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 """Pydantic (v1 / v2) adapter for LanceDB"""
 
@@ -30,6 +20,7 @@ from typing import (
     Type,
     Union,
     _GenericAlias,
+    GenericAlias,
 )
 
 import numpy as np
@@ -75,7 +66,7 @@ def vector(dim: int, value_type: pa.DataType = pa.float32()):
 
 
 def Vector(
-    dim: int, value_type: pa.DataType = pa.float32()
+    dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True
 ) -> Type[FixedSizeListMixin]:
     """Pydantic Vector Type.
 
@@ -88,6 +79,8 @@ def Vector(
         The dimension of the vector.
     value_type : pyarrow.DataType, optional
         The value type of the vector, by default pa.float32()
+    nullable : bool, optional
+        Whether the vector is nullable, by default it is True.
 
     Examples
     --------
@@ -103,7 +96,7 @@ def Vector(
     >>> assert schema == pa.schema([
     ...     pa.field("id", pa.int64(), False),
     ...     pa.field("url", pa.utf8(), False),
-    ...     pa.field("embeddings", pa.list_(pa.float32(), 768), False)
+    ...     pa.field("embeddings", pa.list_(pa.float32(), 768))
     ... ])
     """
 
@@ -112,6 +105,10 @@ def Vector(
         def __repr__(self):
             return f"FixedSizeList(dim={dim})"
 
+        @staticmethod
+        def nullable() -> bool:
+            return nullable
+
         @staticmethod
         def dim() -> int:
             return dim
@@ -205,9 +202,7 @@ else:
 def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
     """Convert a Pydantic FieldInfo to Arrow DataType"""
 
-    if isinstance(field.annotation, _GenericAlias) or (
-        sys.version_info > (3, 9) and isinstance(field.annotation, types.GenericAlias)
-    ):
+    if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
         origin = field.annotation.__origin__
         args = field.annotation.__args__
         if origin is list:
@@ -235,7 +230,7 @@ def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
 
 def is_nullable(field: FieldInfo) -> bool:
     """Check if a Pydantic FieldInfo is nullable."""
-    if isinstance(field.annotation, _GenericAlias):
+    if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
         origin = field.annotation.__origin__
         args = field.annotation.__args__
         if origin == Union:
@@ -246,6 +241,10 @@ def is_nullable(field: FieldInfo) -> bool:
         for typ in args:
             if typ is type(None):
                 return True
+    elif inspect.isclass(field.annotation) and issubclass(
+        field.annotation, FixedSizeListMixin
+    ):
+        return field.annotation.nullable()
     return False
 
 

python/python/lancedb/query.py

@@ -26,6 +26,7 @@ from typing import (
     Union,
 )
 
+import asyncio
 import deprecation
 import numpy as np
 import pyarrow as pa
@@ -44,6 +45,8 @@ if TYPE_CHECKING:
     import polars as pl
 
     from ._lancedb import Query as LanceQuery
+    from ._lancedb import FTSQuery as LanceFTSQuery
+    from ._lancedb import HybridQuery as LanceHybridQuery
     from ._lancedb import VectorQuery as LanceVectorQuery
     from .common import VEC
     from .pydantic import LanceModel
@@ -131,6 +134,8 @@ class Query(pydantic.BaseModel):
 
     fast_search: bool = False
 
+    ef: Optional[int] = None
+
 
 class LanceQueryBuilder(ABC):
     """An abstract query builder. Subclasses are defined for vector search,
@@ -257,6 +262,7 @@ class LanceQueryBuilder(ABC):
         self._with_row_id = False
         self._vector = None
         self._text = None
+        self._ef = None
 
     @deprecation.deprecated(
         deprecated_in="0.3.1",
@@ -322,6 +328,14 @@ class LanceQueryBuilder(ABC):
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def to_batches(self, /, batch_size: Optional[int] = None) -> pa.Table:
+        """
+        Execute the query and return the results as a pyarrow
+        [RecordBatchReader](https://arrow.apache.org/docs/python/generated/pyarrow.RecordBatchReader.html)
+        """
+        raise NotImplementedError
+
     def to_list(self) -> List[dict]:
         """
         Execute the query and return the results as a list of dictionaries.
@@ -367,11 +381,13 @@ class LanceQueryBuilder(ABC):
         ----------
         limit: int
             The maximum number of results to return.
-            By default the query is limited to the first 10.
-            Call this method and pass 0, a negative value,
-            or None to remove the limit.
-            *WARNING* if you have a large dataset, removing
-            the limit can potentially result in reading a
+            The default query limit is 10 results.
+            For ANN/KNN queries, you must specify a limit.
+            Entering 0, a negative number, or None will reset
+            the limit to the default value of 10.
+            *WARNING* if you have a large dataset, setting
+            the limit to a large number, e.g. the table size,
+            can potentially result in reading a
             large amount of data into memory and cause
             out of memory issues.
 
@@ -638,6 +654,28 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         self._nprobes = nprobes
         return self
 
+    def ef(self, ef: int) -> LanceVectorQueryBuilder:
+        """Set the number of candidates to consider during search.
+
+        Higher values will yield better recall (more likely to find vectors if
+        they exist) at the expense of latency.
+
+        This only applies to the HNSW-related index.
+        The default value is 1.5 * limit.
+
+        Parameters
+        ----------
+        ef: int
+            The number of candidates to consider during search.
+
+        Returns
+        -------
+        LanceVectorQueryBuilder
+            The LanceQueryBuilder object.
+        """
+        self._ef = ef
+        return self
+
     def refine_factor(self, refine_factor: int) -> LanceVectorQueryBuilder:
         """Set the refine factor to use, increasing the number of vectors sampled.
 
@@ -700,6 +738,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
             with_row_id=self._with_row_id,
             offset=self._offset,
             fast_search=self._fast_search,
+            ef=self._ef,
         )
         result_set = self._table._execute_query(query, batch_size)
         if self._reranker is not None:
@@ -841,6 +880,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
             check_reranker_result(results)
         return results
 
+    def to_batches(self, /, batch_size: Optional[int] = None):
+        raise NotImplementedError("to_batches on an FTS query")
+
     def tantivy_to_arrow(self) -> pa.Table:
         try:
             import tantivy
@@ -943,6 +985,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
 
 class LanceEmptyQueryBuilder(LanceQueryBuilder):
     def to_arrow(self) -> pa.Table:
+        return self.to_batches().read_all()
+
+    def to_batches(self, /, batch_size: Optional[int] = None) -> pa.RecordBatchReader:
         query = Query(
             columns=self._columns,
             filter=self._where,
@@ -952,7 +997,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
             # not actually respected in remote query
             offset=self._offset or 0,
         )
-        return self._table._execute_query(query).read_all()
+        return self._table._execute_query(query)
 
     def rerank(self, reranker: Reranker) -> LanceEmptyQueryBuilder:
         """Rerank the results using the specified reranker.
@@ -1071,6 +1116,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             self._vector_query.nprobes(self._nprobes)
         if self._refine_factor:
             self._vector_query.refine_factor(self._refine_factor)
+        if self._ef:
+            self._vector_query.ef(self._ef)
 
         with ThreadPoolExecutor() as executor:
             fts_future = executor.submit(self._fts_query.with_row_id(True).to_arrow)
@@ -1080,32 +1127,55 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             fts_results = fts_future.result()
             vector_results = vector_future.result()
 
-        # convert to ranks first if needed
-        if self._norm == "rank":
-            vector_results = self._rank(vector_results, "_distance")
-            fts_results = self._rank(fts_results, "_score")
+        return self._combine_hybrid_results(
+            fts_results=fts_results,
+            vector_results=vector_results,
+            norm=self._norm,
+            fts_query=self._fts_query._query,
+            reranker=self._reranker,
+            limit=self._limit,
+            with_row_ids=self._with_row_id,
+        )
+
+    @staticmethod
+    def _combine_hybrid_results(
+        fts_results: pa.Table,
+        vector_results: pa.Table,
+        norm: str,
+        fts_query: str,
+        reranker,
+        limit: int,
+        with_row_ids: bool,
+    ) -> pa.Table:
+        if norm == "rank":
+            vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
+            fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
 
         # normalize the scores to be between 0 and 1, 0 being most relevant
-        vector_results = self._normalize_scores(vector_results, "_distance")
+        vector_results = LanceHybridQueryBuilder._normalize_scores(
+            vector_results, "_distance"
+        )
 
         # In fts higher scores represent relevance. Not inverting them here as
         # rerankers might need to preserve this score to support `return_score="all"`
-        fts_results = self._normalize_scores(fts_results, "_score")
+        fts_results = LanceHybridQueryBuilder._normalize_scores(fts_results, "_score")
 
-        results = self._reranker.rerank_hybrid(
-            self._fts_query._query, vector_results, fts_results
-        )
+        results = reranker.rerank_hybrid(fts_query, vector_results, fts_results)
 
         check_reranker_result(results)
 
-        # apply limit after reranking
-        results = results.slice(length=self._limit)
+        results = results.slice(length=limit)
 
-        if not self._with_row_id:
+        if not with_row_ids:
             results = results.drop(["_rowid"])
+
         return results
 
-    def _rank(self, results: pa.Table, column: str, ascending: bool = True):
+    def to_batches(self):
+        raise NotImplementedError("to_batches not yet supported on a hybrid query")
+
+    @staticmethod
+    def _rank(results: pa.Table, column: str, ascending: bool = True):
         if len(results) == 0:
             return results
         # Get the _score column from results
@@ -1122,7 +1192,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         )
         return results
 
-    def _normalize_scores(self, results: pa.Table, column: str, invert=False):
+    @staticmethod
+    def _normalize_scores(results: pa.Table, column: str, invert=False):
         if len(results) == 0:
             return results
         # Get the _score column from results
@@ -1197,6 +1268,29 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         self._nprobes = nprobes
         return self
 
+    def ef(self, ef: int) -> LanceHybridQueryBuilder:
+        """
+        Set the number of candidates to consider during search.
+
+        Higher values will yield better recall (more likely to find vectors if
+        they exist) at the expense of latency.
+
+        This only applies to the HNSW-related index.
+        The default value is 1.5 * limit.
+
+        Parameters
+        ----------
+        ef: int
+            The number of candidates to consider during search.
+
+        Returns
+        -------
+        LanceHybridQueryBuilder
+            The LanceHybridQueryBuilder object.
+        """
+        self._ef = ef
+        return self
+
     def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceHybridQueryBuilder:
         """Set the distance metric to use.
 
@@ -1449,10 +1543,11 @@ class AsyncQueryBase(object):
         ...     print(plan)
         >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
-          FilterExec: _distance@2 IS NOT NULL
-            SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
-              KNNVectorDistance: metric=l2
-                LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
+          GlobalLimitExec: skip=0, fetch=10
+            FilterExec: _distance@2 IS NOT NULL
+              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+                KNNVectorDistance: metric=l2
+                  LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
 
         Parameters
         ----------
@@ -1495,7 +1590,8 @@ class AsyncQuery(AsyncQueryBase):
         return pa.array(vec)
 
     def nearest_to(
-        self, query_vector: Optional[Union[VEC, Tuple, List[VEC]]] = None
+        self,
+        query_vector: Union[VEC, Tuple, List[VEC]],
     ) -> AsyncVectorQuery:
         """
         Find the nearest vectors to the given query vector.
@@ -1542,6 +1638,9 @@ class AsyncQuery(AsyncQueryBase):
         will be added to the results.  This column will contain the index of the
         query vector that the result is nearest to.
         """
+        if query_vector is None:
+            raise ValueError("query_vector can not be None")
+
         if (
             isinstance(query_vector, list)
             and len(query_vector) > 0
@@ -1560,7 +1659,7 @@ class AsyncQuery(AsyncQueryBase):
 
     def nearest_to_text(
         self, query: str, columns: Union[str, List[str]] = []
-    ) -> AsyncQuery:
+    ) -> AsyncFTSQuery:
         """
         Find the documents that are most relevant to the given text query.
 
@@ -1583,8 +1682,90 @@ class AsyncQuery(AsyncQueryBase):
         """
         if isinstance(columns, str):
             columns = [columns]
-        self._inner.nearest_to_text({"query": query, "columns": columns})
-        return self
+        return AsyncFTSQuery(
+            self._inner.nearest_to_text({"query": query, "columns": columns})
+        )
+
+
+class AsyncFTSQuery(AsyncQueryBase):
+    """A query for full text search for LanceDB."""
+
+    def __init__(self, inner: LanceFTSQuery):
+        super().__init__(inner)
+        self._inner = inner
+
+    def get_query(self):
+        self._inner.get_query()
+
+    def nearest_to(
+        self,
+        query_vector: Union[VEC, Tuple, List[VEC]],
+    ) -> AsyncHybridQuery:
+        """
+        In addition doing text search on the LanceDB Table, also
+        find the nearest vectors to the given query vector.
+
+        This converts the query from a FTS Query to a Hybrid query. Results
+        from the vector search will be combined with results from the FTS query.
+
+        This method will attempt to convert the input to the query vector
+        expected by the embedding model.  If the input cannot be converted
+        then an error will be thrown.
+
+        By default, there is no embedding model, and the input should be
+        something that can be converted to a pyarrow array of floats.  This
+        includes lists, numpy arrays, and tuples.
+
+        If there is only one vector column (a column whose data type is a
+        fixed size list of floats) then the column does not need to be specified.
+        If there is more than one vector column you must use
+        [AsyncVectorQuery.column][lancedb.query.AsyncVectorQuery.column] to specify
+        which column you would like to compare with.
+
+        If no index has been created on the vector column then a vector query
+        will perform a distance comparison between the query vector and every
+        vector in the database and then sort the results.  This is sometimes
+        called a "flat search"
+
+        For small databases, with tens of thousands of vectors or less, this can
+        be reasonably fast.  In larger databases you should create a vector index
+        on the column.  If there is a vector index then an "approximate" nearest
+        neighbor search (frequently called an ANN search) will be performed.  This
+        search is much faster, but the results will be approximate.
+
+        The query can be further parameterized using the returned builder.  There
+        are various ANN search parameters that will let you fine tune your recall
+        accuracy vs search latency.
+
+        Hybrid searches always have a [limit][].  If `limit` has not been called then
+        a default `limit` of 10 will be used.
+
+        Typically, a single vector is passed in as the query. However, you can also
+        pass in multiple vectors.  This can be useful if you want to find the nearest
+        vectors to multiple query vectors. This is not expected to be faster than
+        making multiple queries concurrently; it is just a convenience method.
+        If multiple vectors are passed in then an additional column `query_index`
+        will be added to the results.  This column will contain the index of the
+        query vector that the result is nearest to.
+        """
+        if query_vector is None:
+            raise ValueError("query_vector can not be None")
+
+        if (
+            isinstance(query_vector, list)
+            and len(query_vector) > 0
+            and not isinstance(query_vector[0], (float, int))
+        ):
+            # multiple have been passed
+            query_vectors = [AsyncQuery._query_vec_to_array(v) for v in query_vector]
+            new_self = self._inner.nearest_to(query_vectors[0])
+            for v in query_vectors[1:]:
+                new_self.add_query_vector(v)
+            return AsyncHybridQuery(new_self)
+        else:
+            return AsyncHybridQuery(
+                self._inner.nearest_to(AsyncQuery._query_vec_to_array(query_vector))
+            )
 
 
 class AsyncVectorQuery(AsyncQueryBase):
@@ -1618,7 +1799,7 @@ class AsyncVectorQuery(AsyncQueryBase):
         """
         Set the number of partitions to search (probe)
 
-        This argument is only used when the vector column has an IVF PQ index.
+        This argument is only used when the vector column has an IVF-based index.
         If there is no index then this value is ignored.
 
         The IVF stage of IVF PQ divides the input into partitions (clusters) of
@@ -1640,6 +1821,21 @@ class AsyncVectorQuery(AsyncQueryBase):
         self._inner.nprobes(nprobes)
         return self
 
+    def ef(self, ef: int) -> AsyncVectorQuery:
+        """
+        Set the number of candidates to consider during search
+
+        This argument is only used when the vector column has an HNSW index.
+        If there is no index then this value is ignored.
+
+        Increasing this value will increase the recall of your query but will also
+        increase the latency of your query.  The default value is 1.5 * limit.  This
+        default is good for many cases but the best value to use will depend on your
+        data and the recall that you need to achieve.
+        """
+        self._inner.ef(ef)
+        return self
+
     def refine_factor(self, refine_factor: int) -> AsyncVectorQuery:
         """
         A multiplier to control how many additional rows are taken during the refine
@@ -1706,3 +1902,160 @@ class AsyncVectorQuery(AsyncQueryBase):
         """
         self._inner.bypass_vector_index()
         return self
+
+    def nearest_to_text(
+        self, query: str, columns: Union[str, List[str]] = []
+    ) -> AsyncHybridQuery:
+        """
+        Find the documents that are most relevant to the given text query,
+        in addition to vector search.
+
+        This converts the vector query into a hybrid query.
+
+        This search will perform a full text search on the table and return
+        the most relevant documents, combined with the vector query results.
+        The text relevance is determined by BM25.
+
+        The columns to search must be with native FTS index
+        (Tantivy-based can't work with this method).
+
+        By default, all indexed columns are searched,
+        now only one column can be searched at a time.
+
+        Parameters
+        ----------
+        query: str
+            The text query to search for.
+        columns: str or list of str, default None
+            The columns to search in. If None, all indexed columns are searched.
+            For now only one column can be searched at a time.
+        """
+        if isinstance(columns, str):
+            columns = [columns]
+        return AsyncHybridQuery(
+            self._inner.nearest_to_text({"query": query, "columns": columns})
+        )
+
+
+class AsyncHybridQuery(AsyncQueryBase):
+    """
+    A query builder that performs hybrid vector and full text search.
+    Results are combined and reranked based on the specified reranker.
+    By default, the results are reranked using the RRFReranker, which
+    uses reciprocal rank fusion score for reranking.
+
+    To make the vector and fts results comparable, the scores are normalized.
+    Instead of normalizing scores, the `normalize` parameter can be set to "rank"
+    in the `rerank` method to convert the scores to ranks and then normalize them.
+    """
+
+    def __init__(self, inner: LanceHybridQuery):
+        super().__init__(inner)
+        self._inner = inner
+        self._norm = "score"
+        self._reranker = RRFReranker()
+
+    def rerank(
+        self, reranker: Reranker = RRFReranker(), normalize: str = "score"
+    ) -> AsyncHybridQuery:
+        """
+        Rerank the hybrid search results using the specified reranker. The reranker
+        must be an instance of Reranker class.
+
+        Parameters
+        ----------
+        reranker: Reranker, default RRFReranker()
+            The reranker to use. Must be an instance of Reranker class.
+        normalize: str, default "score"
+            The method to normalize the scores. Can be "rank" or "score". If "rank",
+            the scores are converted to ranks and then normalized. If "score", the
+            scores are normalized directly.
+        Returns
+        -------
+        AsyncHybridQuery
+            The AsyncHybridQuery object.
+        """
+        if normalize not in ["rank", "score"]:
+            raise ValueError("normalize must be 'rank' or 'score'.")
+        if reranker and not isinstance(reranker, Reranker):
+            raise ValueError("reranker must be an instance of Reranker class.")
+
+        self._norm = normalize
+        self._reranker = reranker
+
+        return self
+
+    async def to_batches(self):
+        raise NotImplementedError("to_batches not yet supported on a hybrid query")
+
+    async def to_arrow(self) -> pa.Table:
+        fts_query = AsyncFTSQuery(self._inner.to_fts_query())
+        vec_query = AsyncVectorQuery(self._inner.to_vector_query())
+
+        # save the row ID choice that was made on the query builder and force it
+        # to actually fetch the row ids because we need this for reranking
+        with_row_ids = self._inner.get_with_row_id()
+        fts_query.with_row_id()
+        vec_query.with_row_id()
+
+        fts_results, vector_results = await asyncio.gather(
+            fts_query.to_arrow(),
+            vec_query.to_arrow(),
+        )
+
+        return LanceHybridQueryBuilder._combine_hybrid_results(
+            fts_results=fts_results,
+            vector_results=vector_results,
+            norm=self._norm,
+            fts_query=fts_query.get_query(),
+            reranker=self._reranker,
+            limit=self._inner.get_limit(),
+            with_row_ids=with_row_ids,
+        )
+
+    async def explain_plan(self, verbose: Optional[bool] = False):
+        """Return the execution plan for this query.
+
+        The output includes both the vector and FTS search plans.
+
+        Examples
+        --------
+        >>> import asyncio
+        >>> from lancedb import connect_async
+        >>> from lancedb.index import FTS
+        >>> async def doctest_example():
+        ...     conn = await connect_async("./.lancedb")
+        ...     table = await conn.create_table("my_table", [{"vector": [99, 99], "text": "hello world"}])
+        ...     await table.create_index("text", config=FTS(with_position=False))
+        ...     query = [100, 100]
+        ...     plan = await table.query().nearest_to([1, 2]).nearest_to_text("hello").explain_plan(True)
+        ...     print(plan)
+        >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+        Vector Search Plan:
+        ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance]
+            Take: columns="vector, _rowid, _distance, (text)"
+                CoalesceBatchesExec: target_batch_size=1024
+                GlobalLimitExec: skip=0, fetch=10
+                    FilterExec: _distance@2 IS NOT NULL
+                    SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+                        KNNVectorDistance: metric=l2
+                        LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
+        FTS Search Plan:
+        LanceScan: uri=..., projection=[vector, text], row_id=false, row_addr=false, ordered=true
+
+        Parameters
+        ----------
+        verbose : bool, default False
+            Use a verbose output format.
+
+        Returns
+        -------
+        plan
+        """  # noqa: E501
+
+        results = ["Vector Search Plan:"]
+        results.append(await self._inner.to_vector_query().explain_plan(verbose))
+        results.append("FTS Search Plan:")
+        results.append(await self._inner.to_fts_query().explain_plan(verbose))
+
+        return "\n".join(results)

python/python/lancedb/remote/db.py

@@ -11,7 +11,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import asyncio
 from datetime import timedelta
 import logging
 from concurrent.futures import ThreadPoolExecutor
@@ -25,7 +24,7 @@ import pyarrow as pa
 from overrides import override
 
 from ..common import DATA
-from ..db import DBConnection
+from ..db import DBConnection, LOOP
 from ..embeddings import EmbeddingFunctionConfig
 from ..pydantic import LanceModel
 from ..table import Table
@@ -45,9 +44,9 @@ class RemoteDBConnection(DBConnection):
         client_config: Union[ClientConfig, Dict[str, Any], None] = None,
         connection_timeout: Optional[float] = None,
         read_timeout: Optional[float] = None,
+        storage_options: Optional[Dict[str, str]] = None,
     ):
         """Connect to a remote LanceDB database."""
-
         if isinstance(client_config, dict):
             client_config = ClientConfig(**client_config)
         elif client_config is None:
@@ -86,24 +85,16 @@ class RemoteDBConnection(DBConnection):
             raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
         self.db_name = parsed.netloc
 
-        import nest_asyncio
-
-        nest_asyncio.apply()
-        try:
-            self._loop = asyncio.get_running_loop()
-        except RuntimeError:
-            self._loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(self._loop)
-
         self.client_config = client_config
 
-        self._conn = self._loop.run_until_complete(
+        self._conn = LOOP.run(
             connect_async(
                 db_url,
                 api_key=api_key,
                 region=region,
                 host_override=host_override,
                 client_config=client_config,
+                storage_options=storage_options,
             )
         )
 
@@ -127,9 +118,7 @@ class RemoteDBConnection(DBConnection):
         -------
         An iterator of table names.
         """
-        return self._loop.run_until_complete(
-            self._conn.table_names(start_after=page_token, limit=limit)
-        )
+        return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
 
     @override
     def open_table(self, name: str, *, index_cache_size: Optional[int] = None) -> Table:
@@ -152,8 +141,8 @@ class RemoteDBConnection(DBConnection):
                 " (there is no local cache to configure)"
             )
 
-        table = self._loop.run_until_complete(self._conn.open_table(name))
-        return RemoteTable(table, self.db_name, self._loop)
+        table = LOOP.run(self._conn.open_table(name))
+        return RemoteTable(table, self.db_name)
 
     @override
     def create_table(
@@ -268,7 +257,7 @@ class RemoteDBConnection(DBConnection):
 
         from .table import RemoteTable
 
-        table = self._loop.run_until_complete(
+        table = LOOP.run(
             self._conn.create_table(
                 name,
                 data,
@@ -278,7 +267,7 @@ class RemoteDBConnection(DBConnection):
                 fill_value=fill_value,
             )
         )
-        return RemoteTable(table, self.db_name, self._loop)
+        return RemoteTable(table, self.db_name)
 
     @override
     def drop_table(self, name: str):
@@ -289,7 +278,7 @@ class RemoteDBConnection(DBConnection):
         name: str
             The name of the table.
         """
-        self._loop.run_until_complete(self._conn.drop_table(name))
+        LOOP.run(self._conn.drop_table(name))
 
     @override
     def rename_table(self, cur_name: str, new_name: str):
@@ -302,7 +291,7 @@ class RemoteDBConnection(DBConnection):
         new_name: str
             The new name of the table.
         """
-        self._loop.run_until_complete(self._conn.rename_table(cur_name, new_name))
+        LOOP.run(self._conn.rename_table(cur_name, new_name))
 
     async def close(self):
         """Close the connection to the database."""

python/python/lancedb/remote/table.py

@@ -12,12 +12,12 @@
 #  limitations under the License.
 
 from datetime import timedelta
-import asyncio
 import logging
 from functools import cached_property
 from typing import Dict, Iterable, List, Optional, Union, Literal
 
 from lancedb.index import FTS, BTree, Bitmap, HnswPq, HnswSq, IvfPq, LabelList
+from lancedb.remote.db import LOOP
 import pyarrow as pa
 
 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
@@ -33,9 +33,7 @@ class RemoteTable(Table):
         self,
         table: AsyncTable,
         db_name: str,
-        loop: Optional[asyncio.AbstractEventLoop] = None,
     ):
-        self._loop = loop
         self._table = table
         self.db_name = db_name
 
@@ -56,12 +54,12 @@ class RemoteTable(Table):
         of this Table
 
         """
-        return self._loop.run_until_complete(self._table.schema())
+        return LOOP.run(self._table.schema())
 
     @property
     def version(self) -> int:
         """Get the current version of the table"""
-        return self._loop.run_until_complete(self._table.version())
+        return LOOP.run(self._table.version())
 
     @cached_property
     def embedding_functions(self) -> dict:
@@ -78,6 +76,10 @@ class RemoteTable(Table):
             self.schema.metadata
         )
 
+    def list_versions(self):
+        """List all versions of the table"""
+        return LOOP.run(self._table.list_versions())
+
     def to_arrow(self) -> pa.Table:
         """to_arrow() is not yet supported on LanceDB cloud."""
         raise NotImplementedError("to_arrow() is not yet supported on LanceDB cloud.")
@@ -86,13 +88,19 @@ class RemoteTable(Table):
         """to_pandas() is not yet supported on LanceDB cloud."""
         return NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
 
+    def checkout(self, version):
+        return LOOP.run(self._table.checkout(version))
+
+    def checkout_latest(self):
+        return LOOP.run(self._table.checkout_latest())
+
     def list_indices(self):
         """List all the indices on the table"""
-        return self._loop.run_until_complete(self._table.list_indices())
+        return LOOP.run(self._table.list_indices())
 
     def index_stats(self, index_uuid: str):
         """List all the stats of a specified index"""
-        return self._loop.run_until_complete(self._table.index_stats(index_uuid))
+        return LOOP.run(self._table.index_stats(index_uuid))
 
     def create_scalar_index(
         self,
@@ -122,9 +130,7 @@ class RemoteTable(Table):
         else:
             raise ValueError(f"Unknown index type: {index_type}")
 
-        self._loop.run_until_complete(
-            self._table.create_index(column, config=config, replace=replace)
-        )
+        LOOP.run(self._table.create_index(column, config=config, replace=replace))
 
     def create_fts_index(
         self,
@@ -132,11 +138,26 @@ class RemoteTable(Table):
         *,
         replace: bool = False,
         with_position: bool = True,
+        # tokenizer configs:
+        base_tokenizer: str = "simple",
+        language: str = "English",
+        max_token_length: Optional[int] = 40,
+        lower_case: bool = True,
+        stem: bool = False,
+        remove_stop_words: bool = False,
+        ascii_folding: bool = False,
     ):
-        config = FTS(with_position=with_position)
-        self._loop.run_until_complete(
-            self._table.create_index(column, config=config, replace=replace)
+        config = FTS(
+            with_position=with_position,
+            base_tokenizer=base_tokenizer,
+            language=language,
+            max_token_length=max_token_length,
+            lower_case=lower_case,
+            stem=stem,
+            remove_stop_words=remove_stop_words,
+            ascii_folding=ascii_folding,
         )
+        LOOP.run(self._table.create_index(column, config=config, replace=replace))
 
     def create_index(
         self,
@@ -217,9 +238,7 @@ class RemoteTable(Table):
                 " 'IVF_PQ', 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
             )
 
-        self._loop.run_until_complete(
-            self._table.create_index(vector_column_name, config=config)
-        )
+        LOOP.run(self._table.create_index(vector_column_name, config=config))
 
     def add(
         self,
@@ -251,7 +270,7 @@ class RemoteTable(Table):
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
         """
-        self._loop.run_until_complete(
+        LOOP.run(
             self._table.add(
                 data, mode=mode, on_bad_vectors=on_bad_vectors, fill_value=fill_value
             )
@@ -339,9 +358,7 @@ class RemoteTable(Table):
     def _execute_query(
         self, query: Query, batch_size: Optional[int] = None
     ) -> pa.RecordBatchReader:
-        return self._loop.run_until_complete(
-            self._table._execute_query(query, batch_size=batch_size)
-        )
+        return LOOP.run(self._table._execute_query(query, batch_size=batch_size))
 
     def merge_insert(self, on: Union[str, Iterable[str]]) -> LanceMergeInsertBuilder:
         """Returns a [`LanceMergeInsertBuilder`][lancedb.merge.LanceMergeInsertBuilder]
@@ -358,9 +375,7 @@ class RemoteTable(Table):
         on_bad_vectors: str,
         fill_value: float,
     ):
-        self._loop.run_until_complete(
-            self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
-        )
+        LOOP.run(self._table._do_merge(merge, new_data, on_bad_vectors, fill_value))
 
     def delete(self, predicate: str):
         """Delete rows from the table.
@@ -409,7 +424,7 @@ class RemoteTable(Table):
            x      vector  _distance # doctest: +SKIP
         0  2  [3.0, 4.0]       85.0 # doctest: +SKIP
         """
-        self._loop.run_until_complete(self._table.delete(predicate))
+        LOOP.run(self._table.delete(predicate))
 
     def update(
         self,
@@ -459,7 +474,7 @@ class RemoteTable(Table):
         2  2  [10.0, 10.0] # doctest: +SKIP
 
         """
-        self._loop.run_until_complete(
+        LOOP.run(
             self._table.update(where=where, updates=values, updates_sql=values_sql)
         )
 
@@ -489,22 +504,16 @@ class RemoteTable(Table):
         )
 
     def count_rows(self, filter: Optional[str] = None) -> int:
-        return self._loop.run_until_complete(self._table.count_rows(filter))
+        return LOOP.run(self._table.count_rows(filter))
 
     def add_columns(self, transforms: Dict[str, str]):
-        raise NotImplementedError(
-            "add_columns() is not yet supported on the LanceDB cloud"
-        )
+        return LOOP.run(self._table.add_columns(transforms))
 
-    def alter_columns(self, alterations: Iterable[Dict[str, str]]):
-        raise NotImplementedError(
-            "alter_columns() is not yet supported on the LanceDB cloud"
-        )
+    def alter_columns(self, *alterations: Iterable[Dict[str, str]]):
+        return LOOP.run(self._table.alter_columns(*alterations))
 
     def drop_columns(self, columns: Iterable[str]):
-        raise NotImplementedError(
-            "drop_columns() is not yet supported on the LanceDB cloud"
-        )
+        return LOOP.run(self._table.drop_columns(columns))
 
 
 def add_index(tbl: pa.Table, i: int) -> pa.Table:

python/python/lancedb/rerankers/cohere.py

@@ -41,7 +41,7 @@ class CohereReranker(Reranker):
 
     def __init__(
         self,
-        model_name: str = "rerank-english-v2.0",
+        model_name: str = "rerank-english-v3.0",
         column: str = "text",
         top_n: Union[int, None] = None,
         return_score="relevance",

python/python/lancedb/table.py

@@ -8,7 +8,7 @@ import inspect
 import time
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from datetime import timedelta
+from datetime import datetime, timedelta
 from functools import cached_property
 from typing import (
     TYPE_CHECKING,
@@ -967,8 +967,6 @@ class Table(ABC):
         """
         Add new columns with defined values.
 
-        This is not yet available in LanceDB Cloud.
-
         Parameters
         ----------
         transforms: Dict[str, str]
@@ -978,20 +976,21 @@ class Table(ABC):
         """
 
     @abstractmethod
-    def alter_columns(self, alterations: Iterable[Dict[str, str]]):
+    def alter_columns(self, *alterations: Iterable[Dict[str, str]]):
         """
         Alter column names and nullability.
 
-        This is not yet available in LanceDB Cloud.
-
         alterations : Iterable[Dict[str, Any]]
             A sequence of dictionaries, each with the following keys:
             - "path": str
                 The column path to alter. For a top-level column, this is the name.
                 For a nested column, this is the dot-separated path, e.g. "a.b.c".
-            - "name": str, optional
+            - "rename": str, optional
                 The new name of the column. If not specified, the column name is
                 not changed.
+            - "data_type": pyarrow.DataType, optional
+               The new data type of the column. Existing values will be casted
+               to this type. If not specified, the column data type is not changed.
             - "nullable": bool, optional
                 Whether the column should be nullable. If not specified, the column
                 nullability is not changed. Only non-nullable columns can be changed
@@ -1004,14 +1003,45 @@ class Table(ABC):
         """
         Drop columns from the table.
 
-        This is not yet available in LanceDB Cloud.
-
         Parameters
         ----------
         columns : Iterable[str]
             The names of the columns to drop.
         """
 
+    @abstractmethod
+    def checkout(self):
+        """
+        Checks out a specific version of the Table
+
+        Any read operation on the table will now access the data at the checked out
+        version. As a consequence, calling this method will disable any read consistency
+        interval that was previously set.
+
+        This is a read-only operation that turns the table into a sort of "view"
+        or "detached head".  Other table instances will not be affected.  To make the
+        change permanent you can use the `[Self::restore]` method.
+
+        Any operation that modifies the table will fail while the table is in a checked
+        out state.
+
+        To return the table to a normal state use `[Self::checkout_latest]`
+        """
+
+    @abstractmethod
+    def checkout_latest(self):
+        """
+        Ensures the table is pointing at the latest version
+
+        This can be used to manually update a table when the read_consistency_interval
+        is None
+        It can also be used to undo a `[Self::checkout]` operation
+        """
+
+    @abstractmethod
+    def list_versions(self):
+        """List all versions of the table"""
+
     @cached_property
     def _dataset_uri(self) -> str:
         return _table_uri(self._conn.uri, self.name)
@@ -1047,13 +1077,16 @@ class _LanceLatestDatasetRef(_LanceDatasetRef):
     index_cache_size: Optional[int] = None
     read_consistency_interval: Optional[timedelta] = None
     last_consistency_check: Optional[float] = None
+    storage_options: Optional[Dict[str, str]] = None
     _dataset: Optional[LanceDataset] = None
 
     @property
     def dataset(self) -> LanceDataset:
         if not self._dataset:
             self._dataset = lance.dataset(
-                self.uri, index_cache_size=self.index_cache_size
+                self.uri,
+                index_cache_size=self.index_cache_size,
+                storage_options=self.storage_options,
             )
             self.last_consistency_check = time.monotonic()
         elif self.read_consistency_interval is not None:
@@ -1084,13 +1117,17 @@ class _LanceTimeTravelRef(_LanceDatasetRef):
     uri: str
     version: int
     index_cache_size: Optional[int] = None
+    storage_options: Optional[Dict[str, str]] = None
     _dataset: Optional[LanceDataset] = None
 
     @property
     def dataset(self) -> LanceDataset:
         if not self._dataset:
             self._dataset = lance.dataset(
-                self.uri, version=self.version, index_cache_size=self.index_cache_size
+                self.uri,
+                version=self.version,
+                index_cache_size=self.index_cache_size,
+                storage_options=self.storage_options,
             )
         return self._dataset
 
@@ -1139,24 +1176,27 @@ class LanceTable(Table):
                 uri=self._dataset_uri,
                 version=version,
                 index_cache_size=index_cache_size,
+                storage_options=connection.storage_options,
             )
         else:
             self._ref = _LanceLatestDatasetRef(
                 uri=self._dataset_uri,
                 read_consistency_interval=connection.read_consistency_interval,
                 index_cache_size=index_cache_size,
+                storage_options=connection.storage_options,
             )
 
     @classmethod
     def open(cls, db, name, **kwargs):
         tbl = cls(db, name, **kwargs)
-        fs, path = fs_from_uri(tbl._dataset_path)
-        file_info = fs.get_file_info(path)
-        if file_info.type != pa.fs.FileType.Directory:
-            raise FileNotFoundError(
-                f"Table {name} does not exist."
-                f"Please first call db.create_table({name}, data)"
-            )
+
+        # check the dataset exists
+        try:
+            tbl.version
+        except ValueError as e:
+            if "Not found:" in str(e):
+                raise FileNotFoundError(f"Table {name} does not exist")
+            raise e
 
         return tbl
 
@@ -1567,7 +1607,7 @@ class LanceTable(Table):
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill".
+            One of "error", "drop", "fill", "null".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
@@ -1584,11 +1624,7 @@ class LanceTable(Table):
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
         )
-        # Access the dataset_mut property to ensure that the dataset is mutable.
-        self._ref.dataset_mut
-        self._ref.dataset = lance.write_dataset(
-            data, self._dataset_uri, schema=self.schema, mode=mode
-        )
+        self._ref.dataset_mut.insert(data, mode=mode, schema=self.schema)
 
     def merge(
         self,
@@ -1851,7 +1887,7 @@ class LanceTable(Table):
             data but will validate against any schema that's specified.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill".
+            One of "error", "drop", "fill", "null".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         embedding_functions: list of EmbeddingFunctionModel, default None
@@ -1872,7 +1908,13 @@ class LanceTable(Table):
 
         empty = pa.Table.from_batches([], schema=schema)
         try:
-            lance.write_dataset(empty, tbl._dataset_uri, schema=schema, mode=mode)
+            lance.write_dataset(
+                empty,
+                tbl._dataset_uri,
+                schema=schema,
+                mode=mode,
+                storage_options=db.storage_options,
+            )
         except OSError as err:
             if "Dataset already exists" in str(err) and exist_ok:
                 if tbl.schema != schema:
@@ -1959,6 +2001,7 @@ class LanceTable(Table):
                 "metric": query.metric,
                 "nprobes": query.nprobes,
                 "refine_factor": query.refine_factor,
+                "ef": query.ef,
             }
         return ds.scanner(
             columns=query.columns,
@@ -2151,13 +2194,11 @@ def _sanitize_schema(
         vector column to fixed_size_list(float32) if necessary.
     on_bad_vectors: str, default "error"
         What to do if any of the vectors are not the same size or contains NaNs.
-        One of "error", "drop", "fill".
+        One of "error", "drop", "fill", "null".
     fill_value: float, default 0.
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     if schema is not None:
-        if data.schema == schema:
-            return data
         # cast the columns to the expected types
         data = data.combine_chunks()
         for field in schema:
@@ -2177,6 +2218,7 @@ def _sanitize_schema(
                     vector_column_name=field.name,
                     on_bad_vectors=on_bad_vectors,
                     fill_value=fill_value,
+                    table_schema=schema,
                 )
         return pa.Table.from_arrays(
             [data[name] for name in schema.names], schema=schema
@@ -2197,6 +2239,7 @@ def _sanitize_schema(
 def _sanitize_vector_column(
     data: pa.Table,
     vector_column_name: str,
+    table_schema: Optional[pa.Schema] = None,
     on_bad_vectors: str = "error",
     fill_value: float = 0.0,
 ) -> pa.Table:
@@ -2211,12 +2254,16 @@ def _sanitize_vector_column(
         The name of the vector column.
     on_bad_vectors: str, default "error"
         What to do if any of the vectors are not the same size or contains NaNs.
-        One of "error", "drop", "fill".
+        One of "error", "drop", "fill", "null".
     fill_value: float, default 0.0
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     # ChunkedArray is annoying to work with, so we combine chunks here
     vec_arr = data[vector_column_name].combine_chunks()
+    if table_schema is not None:
+        field = table_schema.field(vector_column_name)
+    else:
+        field = None
     typ = data[vector_column_name].type
     if pa.types.is_list(typ) or pa.types.is_large_list(typ):
         # if it's a variable size list array,
@@ -2243,7 +2290,11 @@ def _sanitize_vector_column(
                 data, fill_value, on_bad_vectors, vec_arr, vector_column_name
             )
     else:
-        if pc.any(pc.is_null(vec_arr.values, nan_is_null=True)).as_py():
+        if (
+            field is not None
+            and not field.nullable
+            and pc.any(pc.is_null(vec_arr.values)).as_py()
+        ) or (pc.any(pc.is_nan(vec_arr.values)).as_py()):
             data = _sanitize_nans(
                 data, fill_value, on_bad_vectors, vec_arr, vector_column_name
             )
@@ -2287,6 +2338,12 @@ def _sanitize_jagged(data, fill_value, on_bad_vectors, vec_arr, vector_column_na
         )
     elif on_bad_vectors == "drop":
         data = data.filter(correct_ndims)
+    elif on_bad_vectors == "null":
+        data = data.set_column(
+            data.column_names.index(vector_column_name),
+            vector_column_name,
+            pc.if_else(correct_ndims, vec_arr, pa.scalar(None)),
+        )
     return data
 
 
@@ -2303,7 +2360,8 @@ def _sanitize_nans(
         raise ValueError(
             f"Vector column {vector_column_name} has NaNs. "
             "Set on_bad_vectors='drop' to remove them, or "
-            "set on_bad_vectors='fill' and fill_value=<value> to replace them."
+            "set on_bad_vectors='fill' and fill_value=<value> to replace them. "
+            "Or set on_bad_vectors='null' to replace them with null."
         )
     elif on_bad_vectors == "fill":
         if fill_value is None:
@@ -2323,6 +2381,17 @@ def _sanitize_nans(
         np_arr = np_arr.reshape(-1, vec_arr.type.list_size)
         not_nulls = np.any(np_arr, axis=1)
         data = data.filter(~not_nulls)
+    elif on_bad_vectors == "null":
+        # null = pa.nulls(len(vec_arr)).cast(vec_arr.type)
+        # values = pc.if_else(pc.is_nan(vec_arr.values), fill_value, vec_arr.values)
+        np_arr = np.isnan(vec_arr.values.to_numpy(zero_copy_only=False))
+        np_arr = np_arr.reshape(-1, vec_arr.type.list_size)
+        no_nans = np.any(np_arr, axis=1)
+        data = data.set_column(
+            data.column_names.index(vector_column_name),
+            vector_column_name,
+            pc.if_else(no_nans, vec_arr, pa.scalar(None)),
+        )
     return data
 
 
@@ -2588,7 +2657,7 @@ class AsyncTable:
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill".
+            One of "error", "drop", "fill", "null".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
@@ -2671,7 +2740,7 @@ class AsyncTable:
 
     def vector_search(
         self,
-        query_vector: Optional[Union[VEC, Tuple]] = None,
+        query_vector: Union[VEC, Tuple],
     ) -> AsyncVectorQuery:
         """
         Search the table with a given query vector.
@@ -2710,6 +2779,8 @@ class AsyncTable:
                 async_query = async_query.refine_factor(query.refine_factor)
             if query.vector_column:
                 async_query = async_query.column(query.vector_column)
+            if query.ef:
+                async_query = async_query.ef(query.ef)
 
         if not query.prefilter:
             async_query = async_query.postfilter()
@@ -2861,6 +2932,53 @@ class AsyncTable:
 
         return await self._inner.update(updates_sql, where)
 
+    async def add_columns(self, transforms: Dict[str, str]):
+        """
+        Add new columns with defined values.
+
+        Parameters
+        ----------
+        transforms: Dict[str, str]
+            A map of column name to a SQL expression to use to calculate the
+            value of the new column. These expressions will be evaluated for
+            each row in the table, and can reference existing columns.
+        """
+        await self._inner.add_columns(list(transforms.items()))
+
+    async def alter_columns(self, *alterations: Iterable[Dict[str, str]]):
+        """
+        Alter column names and nullability.
+
+        alterations : Iterable[Dict[str, Any]]
+            A sequence of dictionaries, each with the following keys:
+            - "path": str
+                The column path to alter. For a top-level column, this is the name.
+                For a nested column, this is the dot-separated path, e.g. "a.b.c".
+            - "rename": str, optional
+                The new name of the column. If not specified, the column name is
+                not changed.
+            - "data_type": pyarrow.DataType, optional
+               The new data type of the column. Existing values will be casted
+               to this type. If not specified, the column data type is not changed.
+            - "nullable": bool, optional
+                Whether the column should be nullable. If not specified, the column
+                nullability is not changed. Only non-nullable columns can be changed
+                to nullable. Currently, you cannot change a nullable column to
+                non-nullable.
+        """
+        await self._inner.alter_columns(alterations)
+
+    async def drop_columns(self, columns: Iterable[str]):
+        """
+        Drop columns from the table.
+
+        Parameters
+        ----------
+        columns : Iterable[str]
+            The names of the columns to drop.
+        """
+        await self._inner.drop_columns(columns)
+
     async def version(self) -> int:
         """
         Retrieve the version of the table
@@ -2873,6 +2991,19 @@ class AsyncTable:
         """
         return await self._inner.version()
 
+    async def list_versions(self):
+        """
+        List all versions of the table
+        """
+        versions = await self._inner.list_versions()
+        for v in versions:
+            ts_nanos = v["timestamp"]
+            v["timestamp"] = datetime.fromtimestamp(ts_nanos // 1e9) + timedelta(
+                microseconds=(ts_nanos % 1e9) // 1e3
+            )
+
+        return versions
+
     async def checkout(self, version):
         """
         Checks out a specific version of the Table

python/python/tests/test_db.py

@@ -599,7 +599,9 @@ async def test_create_in_v2_mode(tmp_path):
     )
 
     async def is_in_v2_mode(tbl):
-        batches = await tbl.query().to_batches(max_batch_length=1024 * 10)
+        batches = (
+            await tbl.query().limit(10 * 1024).to_batches(max_batch_length=1024 * 10)
+        )
         num_batches = 0
         async for batch in batches:
             num_batches += 1

python/python/tests/test_duckdb.py

@@ -0,0 +1,21 @@
+import duckdb
+import pyarrow as pa
+
+import lancedb
+from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
+
+
+def test_basic_query(tmp_path):
+    data = pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]})
+    conn = lancedb.connect(tmp_path)
+    tbl = conn.create_table("test", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)  # noqa: F841
+
+    duck_conn = duckdb.connect()
+
+    results = duck_conn.sql("SELECT SUM(x) FROM adapter").fetchall()
+    assert results[0][0] == 10
+
+    results = duck_conn.sql("SELECT SUM(y) FROM adapter").fetchall()
+    assert results[0][0] == 26

python/python/tests/test_embeddings.py

@@ -81,28 +81,36 @@ def test_embedding_function(tmp_path):
 
 
 def test_embedding_with_bad_results(tmp_path):
-    @register("mock-embedding")
-    class MockEmbeddingFunction(TextEmbeddingFunction):
+    @register("null-embedding")
+    class NullEmbeddingFunction(TextEmbeddingFunction):
         def ndims(self):
             return 128
 
         def generate_embeddings(
             self, texts: Union[List[str], np.ndarray]
         ) -> list[Union[np.array, None]]:
-            return [
-                None if i % 2 == 0 else np.random.randn(self.ndims())
+            # Return None, which is bad if field is non-nullable
+            a = [
+                np.full(self.ndims(), np.nan)
+                if i % 2 == 0
+                else np.random.randn(self.ndims())
                 for i in range(len(texts))
             ]
+            return a
 
     db = lancedb.connect(tmp_path)
     registry = EmbeddingFunctionRegistry.get_instance()
-    model = registry.get("mock-embedding").create()
+    model = registry.get("null-embedding").create()
 
     class Schema(LanceModel):
         text: str = model.SourceField()
         vector: Vector(model.ndims()) = model.VectorField()
 
     table = db.create_table("test", schema=Schema, mode="overwrite")
+    with pytest.raises(ValueError):
+        # Default on_bad_vectors is "error"
+        table.add([{"text": "hello world"}])
+
     table.add(
         [{"text": "hello world"}, {"text": "bar"}],
         on_bad_vectors="drop",
@@ -112,13 +120,33 @@ def test_embedding_with_bad_results(tmp_path):
     assert len(table) == 1
     assert df.iloc[0]["text"] == "bar"
 
-    # table = db.create_table("test2", schema=Schema, mode="overwrite")
-    # table.add(
-    #     [{"text": "hello world"}, {"text": "bar"}],
-    # )
-    # assert len(table) == 2
-    # tbl = table.to_arrow()
-    # assert tbl["vector"].null_count == 1
+    @register("nan-embedding")
+    class NanEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> list[Union[np.array, None]]:
+            # Return NaN to produce bad vectors
+            return [
+                [np.NAN] * 128 if i % 2 == 0 else np.random.randn(self.ndims())
+                for i in range(len(texts))
+            ]
+
+    db = lancedb.connect(tmp_path)
+    registry = EmbeddingFunctionRegistry.get_instance()
+    model = registry.get("nan-embedding").create()
+
+    table = db.create_table("test2", schema=Schema, mode="overwrite")
+    table.alter_columns(dict(path="vector", nullable=True))
+    table.add(
+        [{"text": "hello world"}, {"text": "bar"}],
+        on_bad_vectors="null",
+    )
+    assert len(table) == 2
+    tbl = table.to_arrow()
+    assert tbl["vector"].null_count == 1
 
 
 def test_with_existing_vectors(tmp_path):

python/python/tests/test_embeddings_slow.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import importlib
 import io
 import os
@@ -17,6 +8,7 @@ import os
 import lancedb
 import numpy as np
 import pandas as pd
+import pyarrow as pa
 import pytest
 from lancedb.embeddings import get_registry
 from lancedb.pydantic import LanceModel, Vector
@@ -444,6 +436,30 @@ def test_watsonx_embedding(tmp_path):
         assert tbl.search("hello").limit(1).to_pandas()["text"][0] == "hello world"
 
 
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("OPENAI_API_KEY") is None, reason="OPENAI_API_KEY not set"
+)
+def test_openai_with_empty_strs(tmp_path):
+    model = get_registry().get("openai").create(max_retries=0)
+
+    class TextModel(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", ""]})
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(df, on_bad_vectors="skip")
+    tb = tbl.to_arrow()
+    assert tb.schema.field_by_name("vector").type == pa.list_(
+        pa.float32(), model.ndims()
+    )
+    assert len(tb) == 2
+    assert tb["vector"].is_null().to_pylist() == [False, True]
+
+
 @pytest.mark.slow
 @pytest.mark.skipif(
     importlib.util.find_spec("ollama") is None, reason="Ollama not installed"

python/python/tests/test_hybrid_query.py

@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+
+import pyarrow as pa
+import pytest
+import pytest_asyncio
+
+from lancedb.index import FTS
+from lancedb.table import AsyncTable
+
+
+@pytest_asyncio.fixture
+async def table(tmpdir_factory) -> AsyncTable:
+    tmp_path = str(tmpdir_factory.mktemp("data"))
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table(
+        {
+            "text": pa.array(["a", "b", "cat", "dog"]),
+            "vector": pa.array(
+                [[0.1, 0.1], [2, 2], [-0.1, -0.1], [0.5, -0.5]],
+                type=pa.list_(pa.float32(), list_size=2),
+            ),
+        }
+    )
+    table = await db.create_table("test", data)
+    await table.create_index("text", config=FTS(with_position=False))
+    return table
+
+
+@pytest.mark.asyncio
+async def test_async_hybrid_query(table: AsyncTable):
+    result = await (
+        table.query().nearest_to([0.0, 0.4]).nearest_to_text("dog").limit(2).to_arrow()
+    )
+    assert len(result) == 2
+    # ensure we get results that would match well for text and vector
+    assert result["text"].to_pylist() == ["a", "dog"]
+
+    # ensure there is no rowid by default
+    assert "_rowid" not in result
+
+
+@pytest.mark.asyncio
+async def test_async_hybrid_query_with_row_ids(table: AsyncTable):
+    result = await (
+        table.query()
+        .nearest_to([0.0, 0.4])
+        .nearest_to_text("dog")
+        .limit(2)
+        .with_row_id()
+        .to_arrow()
+    )
+    assert len(result) == 2
+    # ensure we get results that would match well for text and vector
+    assert result["text"].to_pylist() == ["a", "dog"]
+    assert result["_rowid"].to_pylist() == [0, 3]
+
+
+@pytest.mark.asyncio
+async def test_async_hybrid_query_filters(table: AsyncTable):
+    # test that query params are passed down from the regular builder to
+    # child vector/fts builders
+    result = await (
+        table.query()
+        .where("text not in ('a', 'dog')")
+        .nearest_to([0.3, 0.3])
+        .nearest_to_text("*a*")
+        .limit(2)
+        .to_arrow()
+    )
+    assert len(result) == 2
+    # ensure we get results that would match well for text and vector
+    assert result["text"].to_pylist() == ["cat", "b"]
+
+
+@pytest.mark.asyncio
+async def test_async_hybrid_query_default_limit(table: AsyncTable):
+    # add 10 new rows
+    new_rows = []
+    for i in range(100):
+        if i < 2:
+            new_rows.append({"text": "close_vec", "vector": [0.1, 0.1]})
+        else:
+            new_rows.append({"text": "far_vec", "vector": [5 * i, 5 * i]})
+    await table.add(new_rows)
+    result = await (
+        table.query().nearest_to_text("dog").nearest_to([0.1, 0.1]).to_arrow()
+    )
+
+    # assert we got the default limit of 10
+    assert len(result) == 10
+
+    # assert we got the closest vectors and the text searched for
+    texts = result["text"].to_pylist()
+    assert texts.count("close_vec") == 2
+    assert texts.count("dog") == 1
+    assert texts.count("a") == 1
+
+
+@pytest.mark.asyncio
+async def test_explain_plan(table: AsyncTable):
+    plan = await (
+        table.query().nearest_to_text("dog").nearest_to([0.1, 0.1]).explain_plan(True)
+    )
+
+    assert "Vector Search Plan" in plan
+    assert "KNNVectorDistance" in plan
+    assert "FTS Search Plan" in plan
+    assert "LanceScan" in plan

python/python/tests/test_pyarrow.py

@@ -0,0 +1,47 @@
+import pyarrow as pa
+
+import lancedb
+from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
+
+
+def test_dataset_adapter(tmp_path):
+    data = pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]})
+    conn = lancedb.connect(tmp_path)
+    tbl = conn.create_table("test", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)
+
+    assert adapter.count_rows() == 4
+    assert adapter.count_rows("x > 2") == 2
+    assert adapter.schema == data.schema
+    assert adapter.head(2) == data.slice(0, 2)
+    assert adapter.to_table() == data
+    assert adapter.to_batches().read_all() == data
+    assert adapter.scanner().to_table() == data
+    assert adapter.scanner().to_batches().read_all() == data
+
+    assert adapter.scanner().projected_schema == data.schema
+    assert adapter.scanner(columns=["x"]).projected_schema == pa.schema(
+        [data.schema.field("x")]
+    )
+    assert adapter.scanner(columns=["x"]).to_table() == pa.table({"x": [1, 2, 3, 4]})
+
+    # Make sure we bypass the limit
+    data = pa.table({"x": range(100)})
+    tbl = conn.create_table("test2", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)
+
+    assert adapter.count_rows() == 100
+    assert adapter.to_table().num_rows == 100
+    assert adapter.head(10).num_rows == 10
+
+    # Empty table
+    tbl = conn.create_table("test3", None, schema=pa.schema({"x": pa.int64()}))
+    adapter = PyarrowDatasetAdapter(tbl)
+
+    assert adapter.count_rows() == 0
+    assert adapter.to_table().num_rows == 0
+    assert adapter.head(10).num_rows == 0
+
+    assert adapter.scanner().projected_schema == pa.schema({"x": pa.int64()})

python/python/tests/test_pydantic.py

@@ -1,16 +1,5 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import json
 import sys
@@ -172,6 +161,26 @@ def test_pydantic_to_arrow_py38():
     assert schema == expect_schema
 
 
+def test_nullable_vector():
+    class NullableModel(pydantic.BaseModel):
+        vec: Vector(16, nullable=False)
+
+    schema = pydantic_to_schema(NullableModel)
+    assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), False)])
+
+    class DefaultModel(pydantic.BaseModel):
+        vec: Vector(16)
+
+    schema = pydantic_to_schema(DefaultModel)
+    assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])
+
+    class NotNullableModel(pydantic.BaseModel):
+        vec: Vector(16)
+
+    schema = pydantic_to_schema(NotNullableModel)
+    assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])
+
+
 def test_fixed_size_list_field():
     class TestModel(pydantic.BaseModel):
         vec: Vector(16)
@@ -192,7 +201,7 @@ def test_fixed_size_list_field():
     schema = pydantic_to_schema(TestModel)
     assert schema == pa.schema(
         [
-            pa.field("vec", pa.list_(pa.float32(), 16), False),
+            pa.field("vec", pa.list_(pa.float32(), 16)),
             pa.field("li", pa.list_(pa.int64()), False),
         ]
     )

python/python/tests/test_query.py

@@ -1,21 +1,9 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import unittest.mock as mock
 from datetime import timedelta
-from typing import Optional
 
-import lance
 import lancedb
 from lancedb.index import IvfPq
 import numpy as np
@@ -23,41 +11,15 @@ import pandas.testing as tm
 import pyarrow as pa
 import pytest
 import pytest_asyncio
-from lancedb.db import LanceDBConnection
 from lancedb.pydantic import LanceModel, Vector
 from lancedb.query import AsyncQueryBase, LanceVectorQueryBuilder, Query
 from lancedb.table import AsyncTable, LanceTable
 
 
-class MockTable:
-    def __init__(self, tmp_path):
-        self.uri = tmp_path
-        self._conn = LanceDBConnection(self.uri)
-
-    def to_lance(self):
-        return lance.dataset(self.uri)
-
-    def _execute_query(self, query, batch_size: Optional[int] = None):
-        ds = self.to_lance()
-        return ds.scanner(
-            columns=query.columns,
-            filter=query.filter,
-            prefilter=query.prefilter,
-            nearest={
-                "column": query.vector_column,
-                "q": query.vector,
-                "k": query.k,
-                "metric": query.metric,
-                "nprobes": query.nprobes,
-                "refine_factor": query.refine_factor,
-            },
-            batch_size=batch_size,
-            offset=query.offset,
-        ).to_reader()
-
-
-@pytest.fixture
-def table(tmp_path) -> MockTable:
+@pytest.fixture(scope="module")
+def table(tmpdir_factory) -> lancedb.table.Table:
+    tmp_path = str(tmpdir_factory.mktemp("data"))
+    db = lancedb.connect(tmp_path)
     df = pa.table(
         {
             "vector": pa.array(
@@ -68,8 +30,7 @@ def table(tmp_path) -> MockTable:
             "float_field": pa.array([1.0, 2.0]),
         }
     )
-    lance.write_dataset(df, tmp_path)
-    return MockTable(tmp_path)
+    return db.create_table("test", df)
 
 
 @pytest_asyncio.fixture
@@ -126,6 +87,12 @@ def test_query_builder(table):
     assert all(np.array(rs[0]["vector"]) == [1, 2])
 
 
+def test_with_row_id(table: lancedb.table.Table):
+    rs = table.search().with_row_id(True).to_arrow()
+    assert "_rowid" in rs.column_names
+    assert rs["_rowid"].to_pylist() == [0, 1]
+
+
 def test_vector_query_with_no_limit(table):
     with pytest.raises(ValueError):
         LanceVectorQueryBuilder(table, [0, 0], "vector").limit(0).select(
@@ -365,6 +332,12 @@ async def test_query_to_pandas_async(table_async: AsyncTable):
     assert df.shape == (0, 4)
 
 
+@pytest.mark.asyncio
+async def test_none_query(table_async: AsyncTable):
+    with pytest.raises(ValueError):
+        await table_async.query().nearest_to(None).to_arrow()
+
+
 @pytest.mark.asyncio
 async def test_fast_search_async(tmp_path):
     db = await lancedb.connect_async(tmp_path)

python/python/tests/test_remote_db.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+from concurrent.futures import ThreadPoolExecutor
 import contextlib
 from datetime import timedelta
 import http.server
@@ -103,6 +104,47 @@ async def test_async_remote_db():
         assert table_names == []
 
 
+@pytest.mark.asyncio
+async def test_async_checkout():
+    def handler(request):
+        if request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            response = json.dumps({"version": 42, "schema": {"fields": []}})
+            request.wfile.write(response.encode())
+            return
+
+        content_len = int(request.headers.get("Content-Length"))
+        body = request.rfile.read(content_len)
+        body = json.loads(body)
+
+        print("body is", body)
+
+        count = 0
+        if body["version"] == 1:
+            count = 100
+        elif body["version"] == 2:
+            count = 200
+        elif body["version"] is None:
+            count = 300
+
+        request.send_response(200)
+        request.send_header("Content-Type", "application/json")
+        request.end_headers()
+        request.wfile.write(json.dumps(count).encode())
+
+    async with mock_lancedb_connection_async(handler) as db:
+        table = await db.open_table("test")
+        assert await table.count_rows() == 300
+        await table.checkout(1)
+        assert await table.count_rows() == 100
+        await table.checkout(2)
+        assert await table.count_rows() == 200
+        await table.checkout_latest()
+        assert await table.count_rows() == 300
+
+
 @pytest.mark.asyncio
 async def test_http_error():
     request_id_holder = {"request_id": None}
@@ -146,6 +188,85 @@ async def test_retry_error():
         assert cause.status_code == 429
 
 
+def test_table_add_in_threadpool():
+    def handler(request):
+        if request.path == "/v1/table/test/insert/":
+            request.send_response(200)
+            request.end_headers()
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(
+                dict(
+                    version=1,
+                    schema=dict(
+                        fields=[
+                            dict(name="id", type={"type": "int64"}, nullable=False),
+                        ]
+                    ),
+                )
+            )
+            request.wfile.write(payload.encode())
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        with ThreadPoolExecutor(3) as executor:
+            futures = []
+            for _ in range(10):
+                future = executor.submit(table.add, [{"id": 1}])
+                futures.append(future)
+
+            for future in futures:
+                future.result()
+
+
+def test_table_create_indices():
+    def handler(request):
+        if request.path == "/v1/table/test/create_index/":
+            request.send_response(200)
+            request.end_headers()
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(
+                dict(
+                    version=1,
+                    schema=dict(
+                        fields=[
+                            dict(name="id", type={"type": "int64"}, nullable=False),
+                        ]
+                    ),
+                )
+            )
+            request.wfile.write(payload.encode())
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        # Parameters are well-tested through local and async tests.
+        # This is a smoke-test.
+        table = db.create_table("test", [{"id": 1}])
+        table.create_scalar_index("id")
+        table.create_fts_index("text")
+        table.create_scalar_index("vector")
+
+
 @contextlib.contextmanager
 def query_test_table(query_handler):
     def handler(request):
@@ -185,8 +306,10 @@ def test_query_sync_minimal():
             "k": 10,
             "prefilter": False,
             "refine_factor": None,
+            "ef": None,
             "vector": [1.0, 2.0, 3.0],
             "nprobes": 20,
+            "version": None,
         }
 
         return pa.table({"id": [1, 2, 3]})
@@ -204,6 +327,7 @@ def test_query_sync_empty_query():
             "filter": "true",
             "vector": [],
             "columns": ["id"],
+            "version": None,
         }
 
         return pa.table({"id": [1, 2, 3]})
@@ -223,11 +347,13 @@ def test_query_sync_maximal():
             "refine_factor": 10,
             "vector": [1.0, 2.0, 3.0],
             "nprobes": 5,
+            "ef": None,
             "filter": "id > 0",
             "columns": ["id", "name"],
             "vector_column": "vector2",
             "fast_search": True,
             "with_row_id": True,
+            "version": None,
         }
 
         return pa.table({"id": [1, 2, 3], "name": ["a", "b", "c"]})
@@ -266,6 +392,7 @@ def test_query_sync_fts():
             },
             "k": 10,
             "vector": [],
+            "version": None,
         }
 
         return pa.table({"id": [1, 2, 3]})
@@ -282,6 +409,7 @@ def test_query_sync_fts():
             "k": 42,
             "vector": [],
             "with_row_id": True,
+            "version": None,
         }
 
         return pa.table({"id": [1, 2, 3]})
@@ -307,6 +435,7 @@ def test_query_sync_hybrid():
                 "k": 42,
                 "vector": [],
                 "with_row_id": True,
+                "version": None,
             }
             return pa.table({"_rowid": [1, 2, 3], "_score": [0.1, 0.2, 0.3]})
         else:
@@ -318,7 +447,9 @@ def test_query_sync_hybrid():
                 "refine_factor": None,
                 "vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
                 "nprobes": 20,
+                "ef": None,
                 "with_row_id": True,
+                "version": None,
             }
             return pa.table({"_rowid": [1, 2, 3], "_distance": [0.1, 0.2, 0.3]})
 

python/python/tests/test_table.py

@@ -30,6 +30,7 @@ class MockDB:
     def __init__(self, uri: Path):
         self.uri = str(uri)
         self.read_consistency_interval = None
+        self.storage_options = None
 
     @functools.cached_property
     def is_managed_remote(self) -> bool:
@@ -240,6 +241,121 @@ def test_add(db):
     _add(table, schema)
 
 
+def test_add_subschema(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
+            pa.field("item", pa.string(), nullable=True),
+            pa.field("price", pa.float64(), nullable=False),
+        ]
+    )
+    table = db.create_table("test", schema=schema)
+
+    data = {"price": 10.0, "item": "foo"}
+    table.add([data])
+    data = {"price": 2.0, "vector": [3.1, 4.1]}
+    table.add([data])
+    data = {"price": 3.0, "vector": [5.9, 26.5], "item": "bar"}
+    table.add([data])
+
+    expected = pa.table(
+        {
+            "vector": [None, [3.1, 4.1], [5.9, 26.5]],
+            "item": ["foo", None, "bar"],
+            "price": [10.0, 2.0, 3.0],
+        },
+        schema=schema,
+    )
+    assert table.to_arrow() == expected
+
+    data = {"item": "foo"}
+    # We can't omit a column if it's not nullable
+    with pytest.raises(OSError, match="Invalid user input"):
+        table.add([data])
+
+    # We can add it if we make the column nullable
+    table.alter_columns(dict(path="price", nullable=True))
+    table.add([data])
+
+    expected_schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
+            pa.field("item", pa.string(), nullable=True),
+            pa.field("price", pa.float64(), nullable=True),
+        ]
+    )
+    expected = pa.table(
+        {
+            "vector": [None, [3.1, 4.1], [5.9, 26.5], None],
+            "item": ["foo", None, "bar", "foo"],
+            "price": [10.0, 2.0, 3.0, None],
+        },
+        schema=expected_schema,
+    )
+    assert table.to_arrow() == expected
+
+
+def test_add_nullability(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2), nullable=False),
+            pa.field("id", pa.string(), nullable=False),
+        ]
+    )
+    table = db.create_table("test", schema=schema)
+
+    nullable_schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
+            pa.field("id", pa.string(), nullable=True),
+        ]
+    )
+    data = pa.table(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5]],
+            "id": ["foo", "bar"],
+        },
+        schema=nullable_schema,
+    )
+    # We can add nullable schema if it doesn't actually contain nulls
+    table.add(data)
+
+    expected = data.cast(schema)
+    assert table.to_arrow() == expected
+
+    data = pa.table(
+        {
+            "vector": [None],
+            "id": ["baz"],
+        },
+        schema=nullable_schema,
+    )
+    # We can't add nullable schema if it contains nulls
+    with pytest.raises(Exception, match="Vector column vector has NaNs"):
+        table.add(data)
+
+    # But we can make it nullable
+    table.alter_columns(dict(path="vector", nullable=True))
+    table.add(data)
+
+    expected_schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
+            pa.field("id", pa.string(), nullable=False),
+        ]
+    )
+    expected = pa.table(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5], None],
+            "id": ["foo", "bar", "baz"],
+        },
+        schema=expected_schema,
+    )
+    assert table.to_arrow() == expected
+
+
 def test_add_pydantic_model(db):
     # https://github.com/lancedb/lancedb/issues/562
 
@@ -1177,6 +1293,19 @@ def test_add_columns(tmp_path):
     assert table.to_arrow().column_names == ["id", "new_col"]
     assert table.to_arrow()["new_col"].to_pylist() == [2, 3]
 
+    table.add_columns({"null_int": "cast(null as bigint)"})
+    assert table.schema.field("null_int").type == pa.int64()
+
+
+@pytest.mark.asyncio
+async def test_add_columns_async(db_async: AsyncConnection):
+    data = pa.table({"id": [0, 1]})
+    table = await db_async.create_table("my_table", data=data)
+    await table.add_columns({"new_col": "id + 2"})
+    data = await table.to_arrow()
+    assert data.column_names == ["id", "new_col"]
+    assert data["new_col"].to_pylist() == [2, 3]
+
 
 def test_alter_columns(tmp_path):
     db = lancedb.connect(tmp_path)
@@ -1186,6 +1315,18 @@ def test_alter_columns(tmp_path):
     assert table.to_arrow().column_names == ["new_id"]
 
 
+@pytest.mark.asyncio
+async def test_alter_columns_async(db_async: AsyncConnection):
+    data = pa.table({"id": [0, 1]})
+    table = await db_async.create_table("my_table", data=data)
+    await table.alter_columns({"path": "id", "rename": "new_id"})
+    assert (await table.to_arrow()).column_names == ["new_id"]
+    await table.alter_columns(dict(path="new_id", data_type=pa.int16(), nullable=True))
+    data = await table.to_arrow()
+    assert data.column(0).type == pa.int16()
+    assert data.schema.field(0).nullable
+
+
 def test_drop_columns(tmp_path):
     db = lancedb.connect(tmp_path)
     data = pa.table({"id": [0, 1], "category": ["a", "b"]})
@@ -1194,6 +1335,14 @@ def test_drop_columns(tmp_path):
     assert table.to_arrow().column_names == ["id"]
 
 
+@pytest.mark.asyncio
+async def test_drop_columns_async(db_async: AsyncConnection):
+    data = pa.table({"id": [0, 1], "category": ["a", "b"]})
+    table = await db_async.create_table("my_table", data=data)
+    await table.drop_columns(["category"])
+    assert (await table.to_arrow()).column_names == ["id"]
+
+
 @pytest.mark.asyncio
 async def test_time_travel(db_async: AsyncConnection):
     # Setup

python/src/arrow.rs

@@ -10,7 +10,7 @@ use arrow::{
 use futures::stream::StreamExt;
 use lancedb::arrow::SendableRecordBatchStream;
 use pyo3::{pyclass, pymethods, Bound, PyAny, PyObject, PyRef, PyResult, Python};
-use pyo3_asyncio_0_21::tokio::future_into_py;
+use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::error::PythonErrorExt;
 

python/src/connection.rs

@@ -9,7 +9,7 @@ use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods, Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
 };
-use pyo3_asyncio_0_21::tokio::future_into_py;
+use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::{error::PythonErrorExt, table::Table};
 
@@ -58,6 +58,7 @@ impl Connection {
         self.inner.take();
     }
 
+    #[pyo3(signature = (start_after=None, limit=None))]
     pub fn table_names(
         self_: PyRef<'_, Self>,
         start_after: Option<String>,
@@ -74,6 +75,7 @@ impl Connection {
         future_into_py(self_.py(), async move { op.execute().await.infer_error() })
     }
 
+    #[pyo3(signature = (name, mode, data, storage_options=None, data_storage_version=None, enable_v2_manifest_paths=None))]
     pub fn create_table<'a>(
         self_: PyRef<'a, Self>,
         name: String,
@@ -111,6 +113,7 @@ impl Connection {
         })
     }
 
+    #[pyo3(signature = (name, mode, schema, storage_options=None, data_storage_version=None, enable_v2_manifest_paths=None))]
     pub fn create_empty_table<'a>(
         self_: PyRef<'a, Self>,
         name: String,
@@ -198,6 +201,7 @@ impl Connection {
 }
 
 #[pyfunction]
+#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None))]
 #[allow(clippy::too_many_arguments)]
 pub fn connect(
     py: Python,

python/src/error.rs

@@ -138,7 +138,9 @@ fn http_from_rust_error(
     status_code: Option<u16>,
 ) -> PyResult<PyErr> {
     let message = err.to_string();
-    let http_err_cls = py.import("lancedb.remote.errors")?.getattr("HttpError")?;
+    let http_err_cls = py
+        .import_bound("lancedb.remote.errors")?
+        .getattr("HttpError")?;
     let py_err = http_err_cls.call1((message, request_id, status_code))?;
 
     // Reset the traceback since it doesn't provide additional information.
@@ -149,5 +151,5 @@ fn http_from_rust_error(
         py_err.setattr(intern!(py, "__cause__"), cause_err)?;
     }
 
-    Ok(PyErr::from_value(py_err))
+    Ok(PyErr::from_value_bound(py_err))
 }

python/src/index.rs

@@ -47,6 +47,7 @@ impl Index {
 
 #[pymethods]
 impl Index {
+    #[pyo3(signature = (distance_type=None, num_partitions=None, num_sub_vectors=None, max_iterations=None, sample_rate=None))]
     #[staticmethod]
     pub fn ivf_pq(
         distance_type: Option<String>,
@@ -106,6 +107,7 @@ impl Index {
         })
     }
 
+    #[pyo3(signature = (with_position=None, base_tokenizer=None, language=None, max_token_length=None, lower_case=None, stem=None, remove_stop_words=None, ascii_folding=None))]
     #[allow(clippy::too_many_arguments)]
     #[staticmethod]
     pub fn fts(
@@ -146,6 +148,7 @@ impl Index {
         }
     }
 
+    #[pyo3(signature = (distance_type=None, num_partitions=None, num_sub_vectors=None, max_iterations=None, sample_rate=None, m=None, ef_construction=None))]
     #[staticmethod]
     pub fn hnsw_pq(
         distance_type: Option<String>,
@@ -184,6 +187,7 @@ impl Index {
         })
     }
 
+    #[pyo3(signature = (distance_type=None, num_partitions=None, max_iterations=None, sample_rate=None, m=None, ef_construction=None))]
     #[staticmethod]
     pub fn hnsw_sq(
         distance_type: Option<String>,

python/src/lib.rs

@@ -16,7 +16,11 @@ use arrow::RecordBatchStream;
 use connection::{connect, Connection};
 use env_logger::Env;
 use index::{Index, IndexConfig};
-use pyo3::{pymodule, types::PyModule, wrap_pyfunction, PyResult, Python};
+use pyo3::{
+    pymodule,
+    types::{PyModule, PyModuleMethods},
+    wrap_pyfunction, Bound, PyResult, Python,
+};
 use query::{Query, VectorQuery};
 use table::Table;
 
@@ -29,7 +33,7 @@ pub mod table;
 pub mod util;
 
 #[pymodule]
-pub fn _lancedb(_py: Python, m: &PyModule) -> PyResult<()> {
+pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     let env = Env::new()
         .filter_or("LANCEDB_LOG", "warn")
         .write_style("LANCEDB_LOG_STYLE");

python/src/query.rs

@@ -18,7 +18,8 @@ use arrow::pyarrow::FromPyArrow;
 use lancedb::index::scalar::FullTextSearchQuery;
 use lancedb::query::QueryExecutionOptions;
 use lancedb::query::{
-    ExecutableQuery, Query as LanceDbQuery, QueryBase, Select, VectorQuery as LanceDbVectorQuery,
+    ExecutableQuery, HasQuery, Query as LanceDbQuery, QueryBase, Select,
+    VectorQuery as LanceDbVectorQuery,
 };
 use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::{PyAnyMethods, PyDictMethods};
@@ -29,7 +30,7 @@ use pyo3::PyAny;
 use pyo3::PyRef;
 use pyo3::PyResult;
 use pyo3::{pyclass, PyErr};
-use pyo3_asyncio_0_21::tokio::future_into_py;
+use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::arrow::RecordBatchStream;
 use crate::error::PythonErrorExt;
@@ -87,7 +88,7 @@ impl Query {
         Ok(VectorQuery { inner })
     }
 
-    pub fn nearest_to_text(&mut self, query: Bound<'_, PyDict>) -> PyResult<()> {
+    pub fn nearest_to_text(&mut self, query: Bound<'_, PyDict>) -> PyResult<FTSQuery> {
         let query_text = query
             .get_item("query")?
             .ok_or(PyErr::new::<PyRuntimeError, _>(
@@ -100,11 +101,14 @@ impl Query {
             .transpose()?;
 
         let fts_query = FullTextSearchQuery::new(query_text).columns(columns);
-        self.inner = self.inner.clone().full_text_search(fts_query);
 
-        Ok(())
+        Ok(FTSQuery {
+            fts_query,
+            inner: self.inner.clone(),
+        })
     }
 
+    #[pyo3(signature = (max_batch_length=None))]
     pub fn execute(
         self_: PyRef<'_, Self>,
         max_batch_length: Option<u32>,
@@ -132,6 +136,87 @@ impl Query {
 }
 
 #[pyclass]
+#[derive(Clone)]
+pub struct FTSQuery {
+    inner: LanceDbQuery,
+    fts_query: FullTextSearchQuery,
+}
+
+#[pymethods]
+impl FTSQuery {
+    pub fn r#where(&mut self, predicate: String) {
+        self.inner = self.inner.clone().only_if(predicate);
+    }
+
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner = self.inner.clone().select(Select::dynamic(&columns));
+    }
+
+    pub fn limit(&mut self, limit: u32) {
+        self.inner = self.inner.clone().limit(limit as usize);
+    }
+
+    pub fn offset(&mut self, offset: u32) {
+        self.inner = self.inner.clone().offset(offset as usize);
+    }
+
+    pub fn fast_search(&mut self) {
+        self.inner = self.inner.clone().fast_search();
+    }
+
+    pub fn with_row_id(&mut self) {
+        self.inner = self.inner.clone().with_row_id();
+    }
+
+    pub fn postfilter(&mut self) {
+        self.inner = self.inner.clone().postfilter();
+    }
+
+    #[pyo3(signature = (max_batch_length=None))]
+    pub fn execute(
+        self_: PyRef<'_, Self>,
+        max_batch_length: Option<u32>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_
+            .inner
+            .clone()
+            .full_text_search(self_.fts_query.clone());
+
+        future_into_py(self_.py(), async move {
+            let mut opts = QueryExecutionOptions::default();
+            if let Some(max_batch_length) = max_batch_length {
+                opts.max_batch_length = max_batch_length;
+            }
+            let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
+            Ok(RecordBatchStream::new(inner_stream))
+        })
+    }
+
+    pub fn nearest_to(&mut self, vector: Bound<'_, PyAny>) -> PyResult<HybridQuery> {
+        let vector_query = Query::new(self.inner.clone()).nearest_to(vector)?;
+        Ok(HybridQuery {
+            inner_fts: self.clone(),
+            inner_vec: vector_query,
+        })
+    }
+
+    pub fn explain_plan(self_: PyRef<'_, Self>, verbose: bool) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .explain_plan(verbose)
+                .await
+                .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+        })
+    }
+
+    pub fn get_query(&self) -> String {
+        self.fts_query.query.clone()
+    }
+}
+
+#[pyclass]
+#[derive(Clone)]
 pub struct VectorQuery {
     inner: LanceDbVectorQuery,
 }
@@ -195,10 +280,15 @@ impl VectorQuery {
         self.inner = self.inner.clone().nprobes(nprobe as usize);
     }
 
+    pub fn ef(&mut self, ef: u32) {
+        self.inner = self.inner.clone().ef(ef as usize);
+    }
+
     pub fn bypass_vector_index(&mut self) {
         self.inner = self.inner.clone().bypass_vector_index()
     }
 
+    #[pyo3(signature = (max_batch_length=None))]
     pub fn execute(
         self_: PyRef<'_, Self>,
         max_batch_length: Option<u32>,
@@ -223,4 +313,105 @@ impl VectorQuery {
                 .map_err(|e| PyRuntimeError::new_err(e.to_string()))
         })
     }
+
+    pub fn nearest_to_text(&mut self, query: Bound<'_, PyDict>) -> PyResult<HybridQuery> {
+        let fts_query = Query::new(self.inner.mut_query().clone()).nearest_to_text(query)?;
+        Ok(HybridQuery {
+            inner_vec: self.clone(),
+            inner_fts: fts_query,
+        })
+    }
+}
+
+#[pyclass]
+pub struct HybridQuery {
+    inner_vec: VectorQuery,
+    inner_fts: FTSQuery,
+}
+
+#[pymethods]
+impl HybridQuery {
+    pub fn r#where(&mut self, predicate: String) {
+        self.inner_vec.r#where(predicate.clone());
+        self.inner_fts.r#where(predicate);
+    }
+
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner_vec.select(columns.clone());
+        self.inner_fts.select(columns);
+    }
+
+    pub fn limit(&mut self, limit: u32) {
+        self.inner_vec.limit(limit);
+        self.inner_fts.limit(limit);
+    }
+
+    pub fn offset(&mut self, offset: u32) {
+        self.inner_vec.offset(offset);
+        self.inner_fts.offset(offset);
+    }
+
+    pub fn fast_search(&mut self) {
+        self.inner_vec.fast_search();
+        self.inner_fts.fast_search();
+    }
+
+    pub fn with_row_id(&mut self) {
+        self.inner_fts.with_row_id();
+        self.inner_vec.with_row_id();
+    }
+
+    pub fn postfilter(&mut self) {
+        self.inner_vec.postfilter();
+        self.inner_fts.postfilter();
+    }
+
+    pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> {
+        self.inner_vec.add_query_vector(vector)
+    }
+
+    pub fn column(&mut self, column: String) {
+        self.inner_vec.column(column);
+    }
+
+    pub fn distance_type(&mut self, distance_type: String) -> PyResult<()> {
+        self.inner_vec.distance_type(distance_type)
+    }
+
+    pub fn refine_factor(&mut self, refine_factor: u32) {
+        self.inner_vec.refine_factor(refine_factor);
+    }
+
+    pub fn nprobes(&mut self, nprobe: u32) {
+        self.inner_vec.nprobes(nprobe);
+    }
+
+    pub fn ef(&mut self, ef: u32) {
+        self.inner_vec.ef(ef);
+    }
+
+    pub fn bypass_vector_index(&mut self) {
+        self.inner_vec.bypass_vector_index();
+    }
+
+    pub fn to_vector_query(&mut self) -> PyResult<VectorQuery> {
+        Ok(VectorQuery {
+            inner: self.inner_vec.inner.clone(),
+        })
+    }
+
+    pub fn to_fts_query(&mut self) -> PyResult<FTSQuery> {
+        Ok(FTSQuery {
+            inner: self.inner_fts.inner.clone(),
+            fts_query: self.inner_fts.fts_query.clone(),
+        })
+    }
+
+    pub fn get_limit(&mut self) -> Option<u32> {
+        self.inner_fts.inner.limit.map(|i| i as u32)
+    }
+
+    pub fn get_with_row_id(&mut self) -> bool {
+        self.inner_fts.inner.with_row_id
+    }
 }

python/src/table.rs

@@ -1,17 +1,21 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
 use arrow::{
+    datatypes::DataType,
     ffi_stream::ArrowArrayStreamReader,
     pyarrow::{FromPyArrow, ToPyArrow},
 };
 use lancedb::table::{
-    AddDataMode, Duration, OptimizeAction, OptimizeOptions, Table as LanceDbTable,
+    AddDataMode, ColumnAlteration, Duration, NewColumnTransform, OptimizeAction, OptimizeOptions,
+    Table as LanceDbTable,
 };
 use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pymethods,
-    types::{PyDict, PyDictMethods, PyString},
+    types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods},
     Bound, FromPyObject, PyAny, PyRef, PyResult, Python, ToPyObject,
 };
-use pyo3_asyncio_0_21::tokio::future_into_py;
+use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::{
     error::PythonErrorExt,
@@ -137,9 +141,10 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (updates, r#where=None))]
     pub fn update<'a>(
         self_: PyRef<'a, Self>,
-        updates: &PyDict,
+        updates: &Bound<'_, PyDict>,
         r#where: Option<String>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let mut op = self_.inner_ref()?.update();
@@ -147,10 +152,8 @@ impl Table {
             op = op.only_if(only_if);
         }
         for (column_name, value) in updates.into_iter() {
-            let column_name: &PyString = column_name.downcast()?;
-            let column_name = column_name.to_str()?.to_string();
-            let value: &PyString = value.downcast()?;
-            let value = value.to_str()?.to_string();
+            let column_name: String = column_name.extract()?;
+            let value: String = value.extract()?;
             op = op.column(column_name, value);
         }
         future_into_py(self_.py(), async move {
@@ -159,6 +162,7 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (filter=None))]
     pub fn count_rows(
         self_: PyRef<'_, Self>,
         filter: Option<String>,
@@ -169,6 +173,7 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (column, index=None, replace=None))]
     pub fn create_index<'a>(
         self_: PyRef<'a, Self>,
         column: String,
@@ -246,6 +251,34 @@ impl Table {
         )
     }
 
+    pub fn list_versions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let versions = inner.list_versions().await.infer_error()?;
+            let versions_as_dict = Python::with_gil(|py| {
+                versions
+                    .iter()
+                    .map(|v| {
+                        let dict = PyDict::new_bound(py);
+                        dict.set_item("version", v.version).unwrap();
+                        dict.set_item(
+                            "timestamp",
+                            v.timestamp.timestamp_nanos_opt().unwrap_or_default(),
+                        )
+                        .unwrap();
+
+                        let tup: Vec<(&String, &String)> = v.metadata.iter().collect();
+                        dict.set_item("metadata", tup.into_py_dict_bound(py))
+                            .unwrap();
+                        dict.to_object(py)
+                    })
+                    .collect::<Vec<_>>()
+            });
+
+            Ok(versions_as_dict)
+        })
+    }
+
     pub fn checkout(self_: PyRef<'_, Self>, version: u64) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
@@ -272,6 +305,7 @@ impl Table {
         Query::new(self.inner_ref().unwrap().query())
     }
 
+    #[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None))]
     pub fn optimize(
         self_: PyRef<'_, Self>,
         cleanup_since_ms: Option<u64>,
@@ -379,6 +413,72 @@ impl Table {
                 .infer_error()
         })
     }
+
+    pub fn add_columns(
+        self_: PyRef<'_, Self>,
+        definitions: Vec<(String, String)>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let definitions = NewColumnTransform::SqlExpressions(definitions);
+
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.add_columns(definitions, None).await.infer_error()?;
+            Ok(())
+        })
+    }
+
+    pub fn alter_columns<'a>(
+        self_: PyRef<'a, Self>,
+        alterations: Vec<Bound<PyDict>>,
+    ) -> PyResult<Bound<'a, PyAny>> {
+        let alterations = alterations
+            .iter()
+            .map(|alteration| {
+                let path = alteration
+                    .get_item("path")?
+                    .ok_or_else(|| PyValueError::new_err("Missing path"))?
+                    .extract()?;
+                let rename = {
+                    // We prefer rename, but support name for backwards compatibility
+                    let rename = if let Ok(Some(rename)) = alteration.get_item("rename") {
+                        Some(rename)
+                    } else {
+                        alteration.get_item("name")?
+                    };
+                    rename.map(|name| name.extract()).transpose()?
+                };
+                let nullable = alteration
+                    .get_item("nullable")?
+                    .map(|val| val.extract())
+                    .transpose()?;
+                let data_type = alteration
+                    .get_item("data_type")?
+                    .map(|val| DataType::from_pyarrow_bound(&val))
+                    .transpose()?;
+                Ok(ColumnAlteration {
+                    path,
+                    rename,
+                    nullable,
+                    data_type,
+                })
+            })
+            .collect::<PyResult<Vec<_>>>()?;
+
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.alter_columns(&alterations).await.infer_error()?;
+            Ok(())
+        })
+    }
+
+    pub fn drop_columns(self_: PyRef<Self>, columns: Vec<String>) -> PyResult<Bound<PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let column_refs = columns.iter().map(String::as_str).collect::<Vec<&str>>();
+            inner.drop_columns(&column_refs).await.infer_error()?;
+            Ok(())
+        })
+    }
 }
 
 #[derive(FromPyObject)]

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.13.0-beta.1"
+version = "0.14.0"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.13.0-beta.1"
+version = "0.14.0"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -27,6 +27,7 @@ half = { workspace = true }
 lazy_static.workspace = true
 lance = { workspace = true }
 lance-datafusion.workspace = true
+lance-io = { workspace = true }
 lance-index = { workspace = true }
 lance-table = { workspace = true }
 lance-linalg = { workspace = true }
@@ -46,10 +47,18 @@ serde = { version = "^1" }
 serde_json = { version = "1" }
 async-openai = { version = "0.20.0", optional = true }
 serde_with = { version = "3.8.1" }
+aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
 # For remote feature
-reqwest = { version = "0.12.0", features = ["gzip", "json", "stream"], optional = true }
-rand = { version = "0.8.3", features = ["small_rng"], optional = true}
-http = { version = "1",  optional = true } # Matching what is in reqwest
+reqwest = { version = "0.12.0", default-features = false, features = [
+    "charset",
+    "gzip",
+    "http2",
+    "json",
+    "macos-system-configuration",
+    "stream",
+], optional = true }
+rand = { version = "0.8.3", features = ["small_rng"], optional = true }
+http = { version = "1", optional = true } # Matching what is in reqwest
 uuid = { version = "1.7.0", features = ["v4"], optional = true }
 polars-arrow = { version = ">=0.37,<0.40.0", optional = true }
 polars = { version = ">=0.37,<0.40.0", optional = true }
@@ -72,11 +81,13 @@ aws-config = { version = "1.0" }
 aws-smithy-runtime = { version = "1.3" }
 http-body = "1" # Matching reqwest
 
+
 [features]
-default = []
+default = ["default-tls"]
 remote = ["dep:reqwest", "dep:http", "dep:rand", "dep:uuid"]
 fp16kernels = ["lance-linalg/fp16kernels"]
 s3-test = []
+bedrock = ["dep:aws-sdk-bedrockruntime"]
 openai = ["dep:async-openai", "dep:reqwest"]
 polars = ["dep:polars-arrow", "dep:polars"]
 sentence-transformers = [
@@ -87,6 +98,11 @@ sentence-transformers = [
     "dep:tokenizers"
 ]
 
+# TLS
+default-tls = ["reqwest?/default-tls"]
+native-tls = ["reqwest?/native-tls"]
+rustls-tls = ["reqwest?/rustls-tls"]
+
 [[example]]
 name = "openai"
 required-features = ["openai"]
@@ -94,3 +110,7 @@ required-features = ["openai"]
 [[example]]
 name = "sentence_transformers"
 required-features = ["sentence-transformers"]
+
+[[example]]
+name = "bedrock"
+required-features = ["bedrock"]

rust/lancedb/examples/bedrock.rs

@@ -0,0 +1,89 @@
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use aws_config::Region;
+use aws_sdk_bedrockruntime::Client;
+use futures::StreamExt;
+use lancedb::{
+    arrow::IntoArrow,
+    connect,
+    embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let tempdir = tempfile::tempdir().unwrap();
+    let tempdir = tempdir.path().to_str().unwrap();
+
+    // create Bedrock embedding function
+    let region: String = "us-east-1".to_string();
+    let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
+        .region(Region::new(region))
+        .load()
+        .await;
+
+    let embedding = Arc::new(BedrockEmbeddingFunction::new(
+        Client::new(&config), // AWS Region
+    ));
+
+    let db = connect(tempdir).execute().await?;
+    db.embedding_registry()
+        .register("bedrock", embedding.clone())?;
+
+    let table = db
+        .create_table("vectors", make_data())
+        .add_embedding(EmbeddingDefinition::new(
+            "text",
+            "bedrock",
+            Some("embeddings"),
+        ))?
+        .execute()
+        .await?;
+
+    // execute vector search
+    let query = Arc::new(StringArray::from_iter_values(once("something warm")));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    let mut results = table
+        .vector_search(query_vector)?
+        .limit(1)
+        .execute()
+        .await?;
+
+    let rb = results.next().await.unwrap()?;
+    let out = rb
+        .column_by_name("text")
+        .unwrap()
+        .as_any()
+        .downcast_ref::<StringArray>()
+        .unwrap();
+    let text = out.iter().next().unwrap().unwrap();
+    println!("Closest match: {}", text);
+    Ok(())
+}
+
+fn make_data() -> impl IntoArrow {
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Int32, true),
+        Field::new("text", DataType::Utf8, false),
+        Field::new("price", DataType::Float64, false),
+    ]);
+
+    let id = Int32Array::from(vec![1, 2, 3, 4]);
+    let text = StringArray::from_iter_values(vec![
+        "Black T-Shirt",
+        "Leather Jacket",
+        "Winter Parka",
+        "Hooded Sweatshirt",
+    ]);
+    let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
+    let schema = Arc::new(schema);
+    let rb = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(id), Arc::new(text), Arc::new(price)],
+    )
+    .unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
+}

rust/lancedb/src/connection.rs

@@ -38,6 +38,9 @@ use crate::table::{NativeTable, TableDefinition, WriteOptions};
 use crate::utils::validate_table_name;
 use crate::Table;
 pub use lance_encoding::version::LanceFileVersion;
+#[cfg(feature = "remote")]
+use lance_io::object_store::StorageOptions;
+use lance_table::io::commit::commit_handler_from_url;
 
 pub const LANCE_FILE_EXTENSION: &str = "lance";
 
@@ -133,7 +136,7 @@ impl IntoArrow for NoData {
 
 /// A builder for configuring a [`Connection::create_table`] operation
 pub struct CreateTableBuilder<const HAS_DATA: bool, T: IntoArrow> {
-    parent: Arc<dyn ConnectionInternal>,
+    pub(crate) parent: Arc<dyn ConnectionInternal>,
     pub(crate) name: String,
     pub(crate) data: Option<T>,
     pub(crate) mode: CreateTableMode,
@@ -341,7 +344,7 @@ pub struct OpenTableBuilder {
 }
 
 impl OpenTableBuilder {
-    fn new(parent: Arc<dyn ConnectionInternal>, name: String) -> Self {
+    pub(crate) fn new(parent: Arc<dyn ConnectionInternal>, name: String) -> Self {
         Self {
             parent,
             name,
@@ -622,7 +625,7 @@ impl ConnectBuilder {
 
     /// Set the LanceDB Cloud client configuration.
     ///
-    /// ```
+    /// ```no_run
     /// # use lancedb::connect;
     /// # use lancedb::remote::*;
     /// connect("db://my_database")
@@ -717,12 +720,14 @@ impl ConnectBuilder {
             message: "An api_key is required when connecting to LanceDb Cloud".to_string(),
         })?;
 
+        let storage_options = StorageOptions(self.storage_options.clone());
         let internal = Arc::new(crate::remote::db::RemoteDatabase::try_new(
             &self.uri,
             &api_key,
             &region,
             self.host_override,
             self.client_config,
+            storage_options.into(),
         )?);
         Ok(Connection {
             internal,
@@ -855,7 +860,7 @@ impl Database {
                 let table_base_uri = if let Some(store) = engine {
                     static WARN_ONCE: std::sync::Once = std::sync::Once::new();
                     WARN_ONCE.call_once(|| {
-                        log::warn!("Specifing engine is not a publicly supported feature in lancedb yet. THE API WILL CHANGE");
+                        log::warn!("Specifying engine is not a publicly supported feature in lancedb yet. THE API WILL CHANGE");
                     });
                     let old_scheme = url.scheme().to_string();
                     let new_scheme = format!("{}+{}", old_scheme, store);
@@ -1036,6 +1041,7 @@ impl ConnectionInternal for Database {
         };
 
         let mut write_params = options.write_options.lance_write_params.unwrap_or_default();
+
         if matches!(&options.mode, CreateTableMode::Overwrite) {
             write_params.mode = WriteMode::Overwrite;
         }
@@ -1122,7 +1128,7 @@ impl ConnectionInternal for Database {
         let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
         let full_path = self.base_path.child(dir_name.clone());
         self.object_store
-            .remove_dir_all(full_path)
+            .remove_dir_all(full_path.clone())
             .await
             .map_err(|err| match err {
                 // this error is not lance::Error::DatasetNotFound,
@@ -1132,6 +1138,19 @@ impl ConnectionInternal for Database {
                 },
                 _ => Error::from(err),
             })?;
+
+        let object_store_params = ObjectStoreParams {
+            storage_options: Some(self.storage_options.clone()),
+            ..Default::default()
+        };
+        let mut uri = self.uri.clone();
+        if let Some(query_string) = &self.query_string {
+            uri.push_str(&format!("?{}", query_string));
+        }
+        let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params))
+            .await
+            .unwrap();
+        commit_handler.delete(&full_path).await.unwrap();
         Ok(())
     }
 
@@ -1169,6 +1188,7 @@ mod tests {
     use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
     use tempfile::tempdir;
 
+    use crate::query::QueryBase;
     use crate::query::{ExecutableQuery, QueryExecutionOptions};
 
     use super::*;
@@ -1296,6 +1316,7 @@ mod tests {
         // In v1 the row group size will trump max_batch_length
         let batches = tbl
             .query()
+            .limit(20000)
             .execute_with_options(QueryExecutionOptions {
                 max_batch_length: 50000,
                 ..Default::default()

rust/lancedb/src/embeddings.rs

@@ -17,6 +17,9 @@ pub mod openai;
 #[cfg(feature = "sentence-transformers")]
 pub mod sentence_transformers;
 
+#[cfg(feature = "bedrock")]
+pub mod bedrock;
+
 use lance::arrow::RecordBatchExt;
 use std::{
     borrow::Cow,

rust/lancedb/src/embeddings/bedrock.rs

@@ -0,0 +1,210 @@
+use aws_sdk_bedrockruntime::Client as BedrockClient;
+use std::{borrow::Cow, fmt::Formatter, str::FromStr, sync::Arc};
+
+use arrow::array::{AsArray, Float32Builder};
+use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array};
+use arrow_data::ArrayData;
+use arrow_schema::DataType;
+use serde_json::{json, Value};
+
+use super::EmbeddingFunction;
+use crate::{Error, Result};
+
+use tokio::runtime::Handle;
+use tokio::task::block_in_place;
+
+#[derive(Debug)]
+pub enum BedrockEmbeddingModel {
+    TitanEmbedding,
+    CohereLarge,
+}
+
+impl BedrockEmbeddingModel {
+    fn ndims(&self) -> usize {
+        match self {
+            Self::TitanEmbedding => 1536,
+            Self::CohereLarge => 1024,
+        }
+    }
+
+    fn model_id(&self) -> &str {
+        match self {
+            Self::TitanEmbedding => "amazon.titan-embed-text-v1",
+            Self::CohereLarge => "cohere.embed-english-v3",
+        }
+    }
+}
+
+impl FromStr for BedrockEmbeddingModel {
+    type Err = Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s {
+            "titan-embed-text-v1" => Ok(Self::TitanEmbedding),
+            "cohere-embed-english-v3" => Ok(Self::CohereLarge),
+            _ => Err(Error::InvalidInput {
+                message: "Invalid model. Available models are: 'titan-embed-text-v1', 'cohere-embed-english-v3'".to_string()
+            }),
+        }
+    }
+}
+
+pub struct BedrockEmbeddingFunction {
+    model: BedrockEmbeddingModel,
+    client: BedrockClient,
+}
+
+impl BedrockEmbeddingFunction {
+    pub fn new(client: BedrockClient) -> Self {
+        Self {
+            model: BedrockEmbeddingModel::TitanEmbedding,
+            client,
+        }
+    }
+
+    pub fn with_model(client: BedrockClient, model: BedrockEmbeddingModel) -> Self {
+        Self { model, client }
+    }
+}
+
+impl EmbeddingFunction for BedrockEmbeddingFunction {
+    fn name(&self) -> &str {
+        "bedrock"
+    }
+
+    fn source_type(&self) -> Result<Cow<DataType>> {
+        Ok(Cow::Owned(DataType::Utf8))
+    }
+
+    fn dest_type(&self) -> Result<Cow<DataType>> {
+        let n_dims = self.model.ndims();
+        Ok(Cow::Owned(DataType::new_fixed_size_list(
+            DataType::Float32,
+            n_dims as i32,
+            false,
+        )))
+    }
+
+    fn compute_source_embeddings(&self, source: ArrayRef) -> Result<ArrayRef> {
+        let len = source.len();
+        let n_dims = self.model.ndims();
+        let inner = self.compute_inner(source)?;
+
+        let fsl = DataType::new_fixed_size_list(DataType::Float32, n_dims as i32, false);
+
+        let array_data = ArrayData::builder(fsl)
+            .len(len)
+            .add_child_data(inner.into_data())
+            .build()?;
+
+        Ok(Arc::new(FixedSizeListArray::from(array_data)))
+    }
+
+    fn compute_query_embeddings(&self, input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        let arr = self.compute_inner(input)?;
+        Ok(Arc::new(arr))
+    }
+}
+
+impl std::fmt::Debug for BedrockEmbeddingFunction {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BedrockEmbeddingFunction")
+            .field("model", &self.model)
+            // Skip client field as it doesn't implement Debug
+            .finish()
+    }
+}
+
+impl BedrockEmbeddingFunction {
+    fn compute_inner(&self, source: Arc<dyn Array>) -> Result<Float32Array> {
+        if source.is_nullable() {
+            return Err(Error::InvalidInput {
+                message: "Expected non-nullable data type".to_string(),
+            });
+        }
+
+        if !matches!(source.data_type(), DataType::Utf8 | DataType::LargeUtf8) {
+            return Err(Error::InvalidInput {
+                message: "Expected Utf8 data type".to_string(),
+            });
+        }
+
+        let mut builder = Float32Builder::new();
+
+        let texts = match source.data_type() {
+            DataType::Utf8 => source
+                .as_string::<i32>()
+                .into_iter()
+                .map(|s| s.expect("array is non-nullable").to_string())
+                .collect::<Vec<String>>(),
+            DataType::LargeUtf8 => source
+                .as_string::<i64>()
+                .into_iter()
+                .map(|s| s.expect("array is non-nullable").to_string())
+                .collect::<Vec<String>>(),
+            _ => unreachable!(),
+        };
+
+        for text in texts {
+            let request_body = match self.model {
+                BedrockEmbeddingModel::TitanEmbedding => {
+                    json!({
+                        "inputText": text
+                    })
+                }
+                BedrockEmbeddingModel::CohereLarge => {
+                    json!({
+                        "texts": [text],
+                        "input_type": "search_document"
+                    })
+                }
+            };
+
+            let client = self.client.clone();
+            let model_id = self.model.model_id().to_string();
+            let request_body = request_body.clone();
+
+            let response = block_in_place(move || {
+                Handle::current().block_on(async move {
+                    client
+                        .invoke_model()
+                        .model_id(model_id)
+                        .body(aws_sdk_bedrockruntime::primitives::Blob::new(
+                            serde_json::to_vec(&request_body).unwrap(),
+                        ))
+                        .send()
+                        .await
+                })
+            })
+            .unwrap();
+
+            let response_json: Value =
+                serde_json::from_slice(response.body.as_ref()).map_err(|e| Error::Runtime {
+                    message: format!("Failed to parse response: {}", e),
+                })?;
+
+            let embedding = match self.model {
+                BedrockEmbeddingModel::TitanEmbedding => response_json["embedding"]
+                    .as_array()
+                    .ok_or_else(|| Error::Runtime {
+                        message: "Missing embedding in response".to_string(),
+                    })?
+                    .iter()
+                    .map(|v| v.as_f64().unwrap() as f32)
+                    .collect::<Vec<f32>>(),
+                BedrockEmbeddingModel::CohereLarge => response_json["embeddings"][0]
+                    .as_array()
+                    .ok_or_else(|| Error::Runtime {
+                        message: "Missing embeddings in response".to_string(),
+                    })?
+                    .iter()
+                    .map(|v| v.as_f64().unwrap() as f32)
+                    .collect::<Vec<f32>>(),
+            };
+
+            builder.append_slice(&embedding);
+        }
+
+        Ok(builder.finish())
+    }
+}

rust/lancedb/src/lib.rs

@@ -30,7 +30,7 @@
 //!
 //! LanceDB runs in process, to use it in your Rust project, put the following in your `Cargo.toml`:
 //!
-//! ```ignore
+//! ```shell
 //! cargo install lancedb
 //! ```
 //!

rust/lancedb/src/query.rs

@@ -348,7 +348,7 @@ pub trait QueryBase {
     ///
     /// The filter should be supplied as an SQL query string.  For example:
     ///
-    /// ```ignore
+    /// ```sql
     /// x > 10
     /// y > 0 AND y < 100
     /// x > 5 OR y = 'test'
@@ -364,8 +364,18 @@ pub trait QueryBase {
     ///
     /// This method is only valid on tables that have a full text search index.
     ///
-    /// ```ignore
-    /// query.full_text_search(FullTextSearchQuery::new("hello world"))
+    /// ```
+    /// use lance_index::scalar::FullTextSearchQuery;
+    /// use lancedb::query::{QueryBase, ExecutableQuery};
+    ///
+    /// # use lancedb::Table;
+    /// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let results = table.query()
+    ///     .full_text_search(FullTextSearchQuery::new("hello world".into()))
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
     /// ```
     fn full_text_search(self, query: FullTextSearchQuery) -> Self;
 
@@ -563,7 +573,7 @@ pub struct Query {
     parent: Arc<dyn TableInternal>,
 
     /// limit the number of rows to return.
-    pub(crate) limit: Option<usize>,
+    pub limit: Option<usize>,
 
     /// Offset of the query.
     pub(crate) offset: Option<usize>,
@@ -586,7 +596,7 @@ pub struct Query {
     /// If set to true, the query will return the `_rowid` meta column.
     ///
     /// By default, this is false.
-    pub(crate) with_row_id: bool,
+    pub with_row_id: bool,
 
     /// If set to false, the filter will be applied after the vector search.
     pub(crate) prefilter: bool,
@@ -596,7 +606,7 @@ impl Query {
     pub(crate) fn new(parent: Arc<dyn TableInternal>) -> Self {
         Self {
             parent,
-            limit: None,
+            limit: Some(DEFAULT_TOP_K),
             offset: None,
             filter: None,
             full_text_search: None,
@@ -704,6 +714,9 @@ pub struct VectorQuery {
     // IVF PQ - ANN search.
     pub(crate) query_vector: Vec<Arc<dyn Array>>,
     pub(crate) nprobes: usize,
+    // The number of candidates to return during the refine step for HNSW,
+    // defaults to 1.5 * limit.
+    pub(crate) ef: Option<usize>,
     pub(crate) refine_factor: Option<u32>,
     pub(crate) distance_type: Option<DistanceType>,
     /// Default is true. Set to false to enforce a brute force search.
@@ -717,6 +730,7 @@ impl VectorQuery {
             column: None,
             query_vector: Vec::new(),
             nprobes: 20,
+            ef: None,
             refine_factor: None,
             distance_type: None,
             use_index: true,
@@ -776,6 +790,18 @@ impl VectorQuery {
         self
     }
 
+    /// Set the number of candidates to return during the refine step for HNSW
+    ///
+    /// This argument is only used when the vector column has an HNSW index.
+    /// If there is no index then this value is ignored.
+    ///
+    /// Increasing this value will increase the recall of your query but will
+    /// also increase the latency of your query.  The default value is 1.5*limit.
+    pub fn ef(mut self, ef: usize) -> Self {
+        self.ef = Some(ef);
+        self
+    }
+
     /// A multiplier to control how many additional rows are taken during the refine step
     ///
     /// This argument is only used when the vector column has an IVF PQ index.