Merge branch 'main' of https://github.com/lancedb/lancedb into yang/relative-lance-dep

Signed-off-by: BubbleCal <bubble-cal@outlook.com>

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.13.0"
+current_version = "0.13.0-beta.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/docs.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Install dependecies needed for ubuntu
         run: |
           sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default
+          rustup update && rustup default        
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -41,8 +41,8 @@ jobs:
       - name: Build Python
         working-directory: python
         run: |
-          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .
-          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
+          python -m pip install -e .
+          python -m pip install -r ../docs/requirements.txt
       - name: Set up node
         uses: actions/setup-node@v3
         with:

.github/workflows/docs_test.yml

@@ -49,7 +49,7 @@ jobs:
     - name: Build Python
       working-directory: docs/test
       run:
-        python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r requirements.txt
+        python -m pip install -r requirements.txt
     - name: Create test files
       run: |
         cd docs/test

.github/workflows/nodejs.yml

@@ -53,9 +53,6 @@ jobs:
         cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint-ci
-    - name: Lint examples
-      working-directory: nodejs/examples
-      run: npm ci && npm run lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -94,18 +91,6 @@ jobs:
       env:
         S3_TEST: "1"
       run: npm run test
-    - name: Setup examples
-      working-directory: nodejs/examples
-      run: npm ci
-    - name: Test examples
-      working-directory: ./
-      env:
-        OPENAI_API_KEY: test
-        OPENAI_BASE_URL: http://0.0.0.0:8000
-      run: |
-        python ci/mock_openai.py &
-        cd nodejs/examples
-        npm test
   macos:
     timeout-minutes: 30
     runs-on: "macos-14"

.github/workflows/npm-publish.yml

@@ -226,109 +226,125 @@ jobs:
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
-  # TODO: re-enable once working https://github.com/lancedb/lancedb/pull/1831
-  # node-windows-arm64:
-  #   name: vectordb win32-arm64-msvc
-  #   runs-on: windows-4x-arm
-  #   if: startsWith(github.ref, 'refs/tags/v')
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install Git
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-  #         Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-  #       shell: powershell
-  #     - name: Add Git to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-  #         $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-  #       shell: powershell
-  #     - name: Configure Git symlinks
-  #       run: git config --global core.symlinks true
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-python@v5
-  #       with:
-  #         python-version: "3.13"
-  #     - name: Install Visual Studio Build Tools
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-  #         Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-  #           "--installPath", "C:\BuildTools", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-  #           "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-  #       shell: powershell
-  #     - name: Add Visual Studio Build Tools to PATH
-  #       run: |
-  #         $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-  #         $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+  node-windows-arm64:
+    name: vectordb win32-arm64-msvc
+    runs-on: windows-4x-arm
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache installations
+        id: cache-installs
+        uses: actions/cache@v4
+        with:
+          path: |
+            C:\Program Files\Git
+            C:\BuildTools
+            C:\Program Files (x86)\Windows Kits
+            C:\Program Files\7-Zip
+            C:\protoc
+          key: ${{ runner.os }}-arm64-installs-v1
+          restore-keys: |
+            ${{ runner.os }}-arm64-installs-
+      - name: Install Git
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+        shell: powershell
+      - name: Add Git to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+        shell: powershell
+      - name: Configure Git symlinks
+        run: git config --global core.symlinks true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install Visual Studio Build Tools
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+            "--installPath", "C:\BuildTools", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+        shell: powershell
+      - name: Add Visual Studio Build Tools to PATH
+        run: |
+          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
 
-  #         # Add MSVC runtime libraries to LIB
-  #         $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
-  #                    "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
-  #                    "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-  #         Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
+          # Add MSVC runtime libraries to LIB
+          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" + 
+                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
+                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
 
-  #         # Add INCLUDE paths
-  #         $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
-  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
-  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
-  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
-  #         Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
-  #       shell: powershell
-  #     - name: Install Rust
-  #       run: |
-  #         Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-  #         .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-  #       shell: powershell
-  #     - name: Add Rust to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-  #       shell: powershell
+          # Add INCLUDE paths
+          $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
+          Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
+        shell: powershell
+      - name: Install Rust
+        run: |
+          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+        shell: powershell
+      - name: Add Rust to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+        shell: powershell
 
-  #     - uses: Swatinem/rust-cache@v2
-  #       with:
-  #         workspaces: rust
-  #     - name: Install 7-Zip ARM
-  #       run: |
-  #         New-Item -Path 'C:\7zip' -ItemType Directory
-  #         Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-  #         Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-  #       shell: powershell
-  #     - name: Add 7-Zip to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-  #       shell: powershell
-  #     - name: Install Protoc v21.12
-  #       working-directory: C:\
-  #       run: |
-  #         if (Test-Path 'C:\protoc') {
-  #             Write-Host "Protoc directory exists, skipping installation"
-  #             return
-  #         }
-  #         New-Item -Path 'C:\protoc' -ItemType Directory
-  #         Set-Location C:\protoc
-  #         Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-  #         & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-  #       shell: powershell
-  #     - name: Add Protoc to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-  #       shell: powershell
-  #     - name: Build Windows native node modules
-  #       run: .\ci\build_windows_artifacts.ps1 aarch64-pc-windows-msvc
-  #     - name: Upload Windows ARM64 Artifacts
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: node-native-windows-arm64
-  #         path: |
-  #           node/dist/*.node
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install 7-Zip ARM
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          New-Item -Path 'C:\7zip' -ItemType Directory
+          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+        shell: powershell
+      - name: Add 7-Zip to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+        shell: powershell
+      - name: Install Protoc v21.12
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        working-directory: C:\
+        run: |
+          if (Test-Path 'C:\protoc') {
+              Write-Host "Protoc directory exists, skipping installation"
+              return
+          }
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+        shell: powershell
+      - name: Add Protoc to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 aarch64-pc-windows-msvc
+      - name: Upload Windows ARM64 Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-windows-arm64
+          path: |
+            node/dist/*.node
 
   nodejs-windows:
     name: lancedb ${{ matrix.target }}
@@ -364,103 +380,119 @@ jobs:
           path: |
             nodejs/dist/*.node
 
-  # TODO: re-enable once working https://github.com/lancedb/lancedb/pull/1831
-  # nodejs-windows-arm64:
-  #   name: lancedb win32-arm64-msvc
-  #   runs-on: windows-4x-arm
-  #   if: startsWith(github.ref, 'refs/tags/v')
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install Git
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
-  #         Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
-  #       shell: powershell
-  #     - name: Add Git to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
-  #         $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
-  #       shell: powershell
-  #     - name: Configure Git symlinks
-  #       run: git config --global core.symlinks true
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-python@v5
-  #       with:
-  #         python-version: "3.13"
-  #     - name: Install Visual Studio Build Tools
-  #       run: |
-  #         Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
-  #         Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
-  #           "--installPath", "C:\BuildTools", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
-  #           "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
-  #           "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
-  #       shell: powershell
-  #     - name: Add Visual Studio Build Tools to PATH
-  #       run: |
-  #         $vsPath = "C:\BuildTools\VC\Tools\MSVC"
-  #         $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
-  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
-  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+  nodejs-windows-arm64:
+    name: lancedb win32-arm64-msvc
+    runs-on: windows-4x-arm
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache installations
+        id: cache-installs
+        uses: actions/cache@v4
+        with:
+          path: |
+            C:\Program Files\Git
+            C:\BuildTools
+            C:\Program Files (x86)\Windows Kits
+            C:\Program Files\7-Zip
+            C:\protoc
+          key: ${{ runner.os }}-arm64-installs-v1
+          restore-keys: |
+            ${{ runner.os }}-arm64-installs-
+      - name: Install Git
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+        shell: powershell
+      - name: Add Git to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+        shell: powershell
+      - name: Configure Git symlinks
+        run: git config --global core.symlinks true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install Visual Studio Build Tools
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+            "--installPath", "C:\BuildTools", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+        shell: powershell
+      - name: Add Visual Studio Build Tools to PATH
+        run: |
+          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
 
-  #         $env:LIB = ""
-  #         Add-Content $env:GITHUB_ENV "LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
-  #       shell: powershell
-  #     - name: Install Rust
-  #       run: |
-  #         Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-  #         .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
-  #       shell: powershell
-  #     - name: Add Rust to PATH
-  #       run: |
-  #         Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
-  #       shell: powershell
+          $env:LIB = ""
+          Add-Content $env:GITHUB_ENV "LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+        shell: powershell
+      - name: Install Rust
+        run: |
+          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+        shell: powershell
+      - name: Add Rust to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+        shell: powershell
 
-  #     - uses: Swatinem/rust-cache@v2
-  #       with:
-  #         workspaces: rust
-  #     - name: Install 7-Zip ARM
-  #       run: |
-  #         New-Item -Path 'C:\7zip' -ItemType Directory
-  #         Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
-  #         Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
-  #       shell: powershell
-  #     - name: Add 7-Zip to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
-  #       shell: powershell
-  #     - name: Install Protoc v21.12
-  #       working-directory: C:\
-  #       run: |
-  #         if (Test-Path 'C:\protoc') {
-  #             Write-Host "Protoc directory exists, skipping installation"
-  #             return
-  #         }
-  #         New-Item -Path 'C:\protoc' -ItemType Directory
-  #         Set-Location C:\protoc
-  #         Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-  #         & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
-  #       shell: powershell
-  #     - name: Add Protoc to PATH
-  #       run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-  #       shell: powershell
-  #     - name: Build Windows native node modules
-  #       run: .\ci\build_windows_artifacts_nodejs.ps1 aarch64-pc-windows-msvc
-  #     - name: Upload Windows ARM64 Artifacts
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: nodejs-native-windows-arm64
-  #         path: |
-  #           nodejs/dist/*.node
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install 7-Zip ARM
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        run: |
+          New-Item -Path 'C:\7zip' -ItemType Directory
+          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+        shell: powershell
+      - name: Add 7-Zip to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+        shell: powershell
+      - name: Install Protoc v21.12
+        if: steps.cache-installs.outputs.cache-hit != 'true'
+        working-directory: C:\
+        run: |
+          if (Test-Path 'C:\protoc') {
+              Write-Host "Protoc directory exists, skipping installation"
+              return
+          }
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+        shell: powershell
+      - name: Add Protoc to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts_nodejs.ps1 aarch64-pc-windows-msvc
+      - name: Upload Windows ARM64 Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows-arm64
+          path: |
+            nodejs/dist/*.node
 
   release:
     name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux, node-windows]
+    needs: [node, node-macos, node-linux, node-windows, node-windows-arm64]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -478,7 +510,7 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
-          # Tag beta as "preview" instead of default "latest". See lancedb
+          # Tag beta as "preview" instead of default "latest". See lancedb 
           # npm publish step for more info.
           if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
             PUBLISH_ARGS="--tag preview"
@@ -500,7 +532,7 @@ jobs:
 
   release-nodejs:
     name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows, nodejs-windows-arm64]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')

.github/workflows/python.yml

@@ -138,7 +138,7 @@ jobs:
         run: rm -rf target/wheels
   windows:
     name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 60
+    timeout-minutes: 30
     strategy:
       matrix:
         config:

.github/workflows/rust.yml

@@ -50,7 +50,6 @@ jobs:
         run: cargo fmt --all -- --check
       - name: Run clippy
         run: cargo clippy --workspace --tests --all-features -- -D warnings
-
   linux:
     timeout-minutes: 30
     # To build all features, we need more disk space than is available
@@ -92,7 +91,6 @@ jobs:
         run: cargo test --all-features
       - name: Run examples
         run: cargo run --example simple
-
   macos:
     timeout-minutes: 30
     strategy:
@@ -120,7 +118,6 @@ jobs:
       - name: Run tests
         # Run with everything except the integration tests.
         run: cargo test --features remote,fp16kernels
-
   windows:
     runs-on: windows-2022
     steps:
@@ -142,11 +139,24 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
-
   windows-arm64:
     runs-on: windows-4x-arm
     steps:
+      - name: Cache installations
+        id: cache-installs
+        uses: actions/cache@v4
+        with:
+          path: |
+            C:\Program Files\Git
+            C:\BuildTools
+            C:\Program Files (x86)\Windows Kits
+            C:\Program Files\7-Zip
+            C:\protoc
+          key: ${{ runner.os }}-arm64-installs-v1
+          restore-keys: |
+            ${{ runner.os }}-arm64-installs-
       - name: Install Git
+        if: steps.cache-installs.outputs.cache-hit != 'true'
         run: |
           Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
           Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
@@ -163,6 +173,7 @@ jobs:
         with:
           python-version: "3.13"
       - name: Install Visual Studio Build Tools
+        if: steps.cache-installs.outputs.cache-hit != 'true'
         run: |
           Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
           Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
@@ -206,10 +217,12 @@ jobs:
         run: |
           Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
         shell: powershell
+
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
       - name: Install 7-Zip ARM
+        if: steps.cache-installs.outputs.cache-hit != 'true'
         run: |
           New-Item -Path 'C:\7zip' -ItemType Directory
           Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
@@ -219,11 +232,12 @@ jobs:
         run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
         shell: powershell
       - name: Install Protoc v21.12
+        if: steps.cache-installs.outputs.cache-hit != 'true'
         working-directory: C:\
         run: |
           if (Test-Path 'C:\protoc') {
-            Write-Host "Protoc directory exists, skipping installation"
-            return
+              Write-Host "Protoc directory exists, skipping installation"
+              return
           }
           New-Item -Path 'C:\protoc' -ItemType Directory
           Set-Location C:\protoc

Cargo.toml

@@ -18,18 +18,16 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.80.0"                                                     # TODO: lower this once we upgrade Lance again.
+rust-version = "1.80.0" # TODO: lower this once we upgrade Lance again.
 
 [workspace.dependencies]
-lance = { "version" = "=0.19.3", "features" = [
-    "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
-lance-index = { version = "=0.19.3", git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
-lance-linalg = { version = "=0.19.3", git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
-lance-table = { version = "=0.19.3", git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
-lance-testing = { version = "=0.19.3", git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
-lance-datafusion = { version = "=0.19.3", git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
-lance-encoding = { version = "=0.19.3", git = "https://github.com/lancedb/lance.git", tag = "v0.19.3-beta.1" }
+lance = { "version" = "=0.19.2", "features" = ["dynamodb"], path = "../lance/rust/lance"}
+lance-index = { "version" = "=0.19.2", path = "../lance/rust/lance-index"}
+lance-linalg = { "version" = "=0.19.2", path = "../lance/rust/lance-linalg"}
+lance-testing = { "version" = "=0.19.2", path = "../lance/rust/lance-testing"}
+lance-datafusion = { "version" = "=0.19.2", path = "../lance/rust/lance-datafusion"}
+lance-encoding = { "version" = "=0.19.2", path = "../lance/rust/lance-encoding"}
+lance-table = { "version" = "=0.19.2", path = "../lance/rust/lance-table"}
 # Note that this one does not include pyarrow
 arrow = { version = "52.2", optional = false }
 arrow-array = "52.2"

ci/mock_openai.py

@@ -1,57 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
-import argparse
-import json
-import http.server
-
-
-class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
-    def do_POST(self):
-        content_length = int(self.headers["Content-Length"])
-        post_data = self.rfile.read(content_length)
-        post_data = json.loads(post_data.decode("utf-8"))
-        # See: https://platform.openai.com/docs/api-reference/embeddings/create
-
-        if isinstance(post_data["input"], str):
-            num_inputs = 1
-        else:
-            num_inputs = len(post_data["input"])
-
-        model = post_data.get("model", "text-embedding-ada-002")
-
-        data = []
-        for i in range(num_inputs):
-            data.append({
-                "object": "embedding",
-                "embedding": [0.1] * 1536,
-                "index": i,
-            })
-
-        response = {
-            "object": "list",
-            "data": data,
-            "model": model,
-            "usage": {
-                "prompt_tokens": 0,
-                "total_tokens": 0,
-            }
-        }
-
-        self.send_response(200)
-        self.send_header("Content-type", "application/json")
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode("utf-8"))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
-    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
-    args = parser.parse_args()
-    port = args.port
-
-    print(f"server started on port {port}. Press Ctrl-C to stop.")
-    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
-
-    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
-        server.serve_forever()

docs/package-lock.json

@@ -19,7 +19,7 @@
     },
     "../node": {
       "name": "vectordb",
-      "version": "0.12.0",
+      "version": "0.4.6",
       "cpu": [
         "x64",
         "arm64"
@@ -31,7 +31,9 @@
         "win32"
       ],
       "dependencies": {
+        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -44,7 +46,6 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
-        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -61,19 +62,15 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "^5.1.0",
+        "typescript": "*",
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.12.0",
-        "@lancedb/vectordb-darwin-x64": "0.12.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.12.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.12.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.12.0"
-      },
-      "peerDependencies": {
-        "@apache-arrow/ts": "^14.0.2",
-        "apache-arrow": "^14.0.2"
+        "@lancedb/vectordb-darwin-arm64": "0.4.6",
+        "@lancedb/vectordb-darwin-x64": "0.4.6",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
       }
     },
     "../node/node_modules/apache-arrow": {

docs/src/ann_indexes.md

@@ -45,9 +45,9 @@ Lance supports `IVF_PQ` index type by default.
         Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
 
         ```typescript
-        --8<--- "nodejs/examples/ann_indexes.test.ts:import"
+        --8<--- "nodejs/examples/ann_indexes.ts:import"
 
-        --8<-- "nodejs/examples/ann_indexes.test.ts:ingest"
+        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
         ```
 
     === "vectordb (deprecated)"
@@ -140,15 +140,13 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 - **limit** (default: 10): The amount of results that will be returned
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
-    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
-  
+  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
+  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
+  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
   A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
-!!! note 
-    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
-
+  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
+  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
 
@@ -171,7 +169,7 @@ There are a couple of parameters that can be used to fine-tune the search:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search1"
+        --8<-- "nodejs/examples/ann_indexes.ts:search1"
         ```
 
     === "vectordb (deprecated)"
@@ -205,7 +203,7 @@ You can further filter the elements returned by a search using a where clause.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search2"
+        --8<-- "nodejs/examples/ann_indexes.ts:search2"
         ```
 
     === "vectordb (deprecated)"
@@ -237,7 +235,7 @@ You can select the columns returned by the query using a select clause.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/ann_indexes.test.ts:search3"
+        --8<-- "nodejs/examples/ann_indexes.ts:search3"
         ```
 
     === "vectordb (deprecated)"

docs/src/basic.md

@@ -157,7 +157,7 @@ recommend switching to stable releases.
         import * as lancedb from "@lancedb/lancedb";
         import * as arrow from "apache-arrow";
 
-        --8<-- "nodejs/examples/basic.test.ts:connect"
+        --8<-- "nodejs/examples/basic.ts:connect"
         ```
 
     === "vectordb (deprecated)"
@@ -212,7 +212,7 @@ table.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_table"
+        --8<-- "nodejs/examples/basic.ts:create_table"
         ```
 
     === "vectordb (deprecated)"
@@ -268,7 +268,7 @@ similar to a `CREATE TABLE` statement in SQL.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
         ```
 
     === "vectordb (deprecated)"
@@ -298,7 +298,7 @@ Once created, you can open a table as follows:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:open_table"
+        --8<-- "nodejs/examples/basic.ts:open_table"
         ```
 
     === "vectordb (deprecated)"
@@ -327,7 +327,7 @@ If you forget the name of your table, you can always get a listing of all table
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:table_names"
+        --8<-- "nodejs/examples/basic.ts:table_names"
         ```
 
     === "vectordb (deprecated)"
@@ -357,7 +357,7 @@ After a table has been created, you can always add more data to it as follows:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:add_data"
+        --8<-- "nodejs/examples/basic.ts:add_data"
         ```
 
     === "vectordb (deprecated)"
@@ -389,7 +389,7 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:vector_search"
+        --8<-- "nodejs/examples/basic.ts:vector_search"
         ```
 
     === "vectordb (deprecated)"
@@ -429,7 +429,7 @@ LanceDB allows you to create an ANN index on a table as follows:
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_index"
+        --8<-- "nodejs/examples/basic.ts:create_index"
         ```
 
     === "vectordb (deprecated)"
@@ -469,7 +469,7 @@ This can delete any number of rows that match the filter.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:delete_rows"
+        --8<-- "nodejs/examples/basic.ts:delete_rows"
         ```
 
     === "vectordb (deprecated)"
@@ -527,7 +527,7 @@ Use the `drop_table()` method on the database to remove a table.
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:drop_table"
+        --8<-- "nodejs/examples/basic.ts:drop_table"
         ```
 
     === "vectordb (deprecated)"
@@ -561,8 +561,8 @@ You can use the embedding API when working with embedding models. It automatical
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.test.ts:openai_embeddings"
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
         ```
 
 === "Rust"

docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md

@@ -20,7 +20,7 @@ Supported parameters (to be passed in `create` method) are:
 
 | Parameter | Type | Default Value | Description |
 |---|---|--------|---------|
-| `name` | `str` | `None` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
+| `name` | `str` | `"voyage-3"` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
 | `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. |
 | `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. |
 

docs/src/embeddings/custom_embedding_function.md

@@ -47,9 +47,9 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 === "TypeScript"
 
     ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:imports"
+    --8<--- "nodejs/examples/custom_embedding_function.ts:imports"
 
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:embedding_impl"
+    --8<--- "nodejs/examples/custom_embedding_function.ts:embedding_impl"
     ```
 
 
@@ -78,7 +78,7 @@ Now you can use this embedding function to create your table schema and that's i
 === "TypeScript"
 
     ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.test.ts:call_custom_function"
+    --8<--- "nodejs/examples/custom_embedding_function.ts:call_custom_function"
     ```
 
 !!! note

docs/src/embeddings/default_embedding_functions.md

@@ -53,7 +53,6 @@ These functions are registered by default to handle text embeddings.
 | [**Jina Embeddings**](available_embedding_models/text_embedding_functions/jina_embedding.md "jina") | 🔗 World-class embedding models to improve your search and RAG systems. You will need **jina api key**. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/jina.png" alt="Jina Icon" width="90" height="35">](available_embedding_models/text_embedding_functions/jina_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 | [ **AWS Bedrock Functions**](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md "bedrock-text") | ☁️ AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/aws_bedrock.png" alt="AWS Bedrock Icon" width="120" height="35">](available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 | [**IBM Watsonx.ai**](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md "watsonx") | 💡 Generate text embeddings using IBM's watsonx.ai platform. **Note**: watsonx.ai library is an optional dependency. | [<img src="https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/logos/watsonx.png" alt="Watsonx Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md) |
-| [**VoyageAI Embeddings**](available_embedding_models/text_embedding_functions/voyageai_embedding.md "voyageai") | 🌕 Voyage AI provides cutting-edge embedding and rerankers. This will help you get started with **VoyageAI** embedding models using LanceDB. Using voyageai API requires voyageai package. Install it via `pip`. | [<img src="https://www.voyageai.com/logo.svg" alt="VoyageAI Icon" width="140" height="35">](available_embedding_models/text_embedding_functions/voyageai_embedding.md) |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
 
 
 
@@ -67,7 +66,6 @@ These functions are registered by default to handle text embeddings.
 [jina-key]: "jina"
 [aws-key]: "bedrock-text"
 [watsonx-key]: "watsonx"
-[voyageai-key]: "voyageai"
 
 
 ## Multi-modal Embedding Functions🖼️ 

docs/src/embeddings/embedding_functions.md

@@ -94,8 +94,8 @@ the embeddings at all:
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/embedding.test.ts:imports"
-        --8<-- "nodejs/examples/embedding.test.ts:embedding_function"
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:embedding_function"
         ```
 
     === "vectordb (deprecated)"
@@ -150,7 +150,7 @@ need to worry about it when you query the table:
             .toArray()
         ```
 
-    === "vectordb (deprecated)"
+    === "vectordb (deprecated)
 
         ```ts
         const results = await table

docs/src/embeddings/index.md

@@ -51,8 +51,8 @@ LanceDB registers the OpenAI embeddings function in the registry as `openai`. Yo
 === "TypeScript"
 
     ```typescript
-    --8<--- "nodejs/examples/embedding.test.ts:imports"
-    --8<--- "nodejs/examples/embedding.test.ts:openai_embeddings"
+    --8<--- "nodejs/examples/embedding.ts:imports"
+    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
     ```
 
 === "Rust"
@@ -121,10 +121,12 @@ class Words(LanceModel):
     vector: Vector(func.ndims()) = func.VectorField()
 
 table = db.create_table("words", schema=Words)
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+    )
 
 query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]

docs/src/fts.md

@@ -160,32 +160,3 @@ To search for a phrase, the index must be created with `with_position=True`:
 table.create_fts_index("text", use_tantivy=False, with_position=True)
 ```
 This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
-
-
-## Incremental indexing
-
-LanceDB supports incremental indexing, which means you can add new records to the table without reindexing the entire table.
-
-This can make the query more efficient, especially when the table is large and the new records are relatively small.
-
-=== "Python"
-
-    ```python
-    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
-    table.optimize()
-    ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [3.1, 4.1], text: "Frodo was a happy puppy" }]);
-    await tbl.optimize();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
-    ```

docs/src/guides/tables.md

@@ -85,13 +85,13 @@ Initialize a LanceDB connection and create a table
 
 
         ```ts
-        --8<-- "nodejs/examples/basic.test.ts:create_table"
+        --8<-- "nodejs/examples/basic.ts:create_table"
         ```
 
         This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use `apache-arrow` to declare a schema
 
         ```ts
-        --8<-- "nodejs/examples/basic.test.ts:create_table_with_schema"
+        --8<-- "nodejs/examples/basic.ts:create_table_with_schema"
         ```
 
         !!! info "Note"
@@ -100,14 +100,14 @@ Initialize a LanceDB connection and create a table
             passed in will NOT be appended to the table in that case.
 
         ```ts
-        --8<-- "nodejs/examples/basic.test.ts:create_table_exists_ok"
+        --8<-- "nodejs/examples/basic.ts:create_table_exists_ok"
         ```
 
         Sometimes you want to make sure that you start fresh. If you want to
         overwrite the table, you can pass in mode: "overwrite" to the createTable function.
 
         ```ts
-        --8<-- "nodejs/examples/basic.test.ts:create_table_overwrite"
+        --8<-- "nodejs/examples/basic.ts:create_table_overwrite"
         ```
 
     === "vectordb (deprecated)"
@@ -227,7 +227,7 @@ LanceDB supports float16 data type!
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_f16_table"
+        --8<-- "nodejs/examples/basic.ts:create_f16_table"
         ```
 
     === "vectordb (deprecated)"
@@ -274,7 +274,7 @@ table = db.create_table(table_name, schema=Content)
 
 Sometimes your data model may contain nested objects.
 For example, you may want to store the document string
-and the document source name as a nested Document object:
+and the document soure name as a nested Document object:
 
 ```python
 class Document(BaseModel):
@@ -455,7 +455,7 @@ You can create an empty table for scenarios where you want to add data to the ta
     === "@lancedb/lancedb"
 
         ```typescript
-        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
         ```
 
     === "vectordb (deprecated)"
@@ -466,7 +466,7 @@ You can create an empty table for scenarios where you want to add data to the ta
 
 ## Adding to a table
 
-After a table has been created, you can always add more data to it using the `add` method
+After a table has been created, you can always add more data to it usind the `add` method
 
 === "Python"
     You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or `Iterator[pa.RecordBatch]`. Below are some examples.
@@ -535,7 +535,7 @@ After a table has been created, you can always add more data to it using the `ad
     ```
 
     ??? "Ingesting Pydantic models with LanceDB embedding API"
-        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` field as None to allow LanceDB to automatically vectorize the data.
+        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` feild as None to allow LanceDB to automatically vectorize the data.
 
         ```python
         import lancedb
@@ -790,27 +790,6 @@ Use the `drop_table()` method on the database to remove a table.
       This permanently removes the table and is not recoverable, unlike deleting rows.
       If the table does not exist an exception is raised.
 
-## Handling bad vectors
-
-In LanceDB Python, you can use the `on_bad_vectors` parameter to choose how
-invalid vector values are handled. Invalid vectors are vectors that are not valid
-because:
-
-1. They are the wrong dimension
-2. They contain NaN values
-3. They are null but are on a non-nullable field
-
-By default, LanceDB will raise an error if it encounters a bad vector. You can
-also choose one of the following options:
-
-* `drop`: Ignore rows with bad vectors
-* `fill`: Replace bad values (NaNs) or missing values (too few dimensions) with
-    the fill value specified in the `fill_value` parameter. An input like
-    `[1.0, NaN, 3.0]` will be replaced with `[1.0, 0.0, 3.0]` if `fill_value=0.0`.
-* `null`: Replace bad vectors with null (only works if the column is nullable).
-    A bad vector `[1.0, NaN, 3.0]` will be replaced with `null` if the column is
-    nullable. If the vector column is non-nullable, then bad vectors will cause an
-    error
 
 ## Consistency
 
@@ -880,4 +859,4 @@ There are three possible settings for `read_consistency_interval`:
 
 Learn the best practices on creating an ANN index and getting the most out of it.
 
-[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](../migration.md) for more information.
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/reranking/index.md

@@ -9,7 +9,6 @@ LanceDB comes with some built-in rerankers. Some of the rerankers that are avail
 | `CrossEncoderReranker` | Uses a cross-encoder model to rerank search results | Vector, FTS, Hybrid |
 | `ColbertReranker` | Uses a colbert model to rerank search results | Vector, FTS, Hybrid |
 | `OpenaiReranker`(Experimental) | Uses OpenAI's chat model to rerank search results | Vector, FTS, Hybrid |
-| `VoyageAIReranker` | Uses voyageai Reranker API to rerank results | Vector, FTS, Hybrid |
 
 
 ## Using a Reranker
@@ -74,7 +73,6 @@ LanceDB comes with some built-in rerankers. Here are some of the rerankers that
 - [Jina Reranker](./jina.md)
 - [AnswerDotAI Rerankers](./answerdotai.md)
 - [Reciprocal Rank Fusion Reranker](./rrf.md)
-- [VoyageAI Reranker](./voyageai.md)
 
 ## Creating Custom Rerankers
 

docs/src/search.md

@@ -58,9 +58,9 @@ db.create_table("my_vectors", data=data)
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/search.test.ts:import"
+        --8<-- "nodejs/examples/search.ts:import"
 
-        --8<-- "nodejs/examples/search.test.ts:search1"
+        --8<-- "nodejs/examples/search.ts:search1"
         ```
 
 
@@ -89,7 +89,7 @@ By default, `l2` will be used as metric type. You can specify the metric type as
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/search.test.ts:search2"
+        --8<-- "nodejs/examples/search.ts:search2"
         ```
 
     === "vectordb (deprecated)"

docs/src/sql.md

@@ -49,7 +49,7 @@ const tbl = await db.createTable('myVectors', data)
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/filtering.test.ts:search"
+        --8<-- "nodejs/examples/filtering.ts:search"
         ```
 
     === "vectordb (deprecated)"
@@ -91,7 +91,7 @@ For example, the following filter string is acceptable:
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/filtering.test.ts:vec_search"
+        --8<-- "nodejs/examples/filtering.ts:vec_search"
         ```
 
     === "vectordb (deprecated)"
@@ -169,7 +169,7 @@ You can also filter your data without search.
     === "@lancedb/lancedb"
 
         ```ts
-        --8<-- "nodejs/examples/filtering.test.ts:sql_search"
+        --8<-- "nodejs/examples/filtering.ts:sql_search"
         ```
 
     === "vectordb (deprecated)"

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.13.0-final.0</version>
+        <version>0.13.0-beta.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.13.0-final.0</version>
+    <version>0.13.0-beta.1</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.13.0",
+  "version": "0.13.0-beta.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.13.0",
+      "version": "0.13.0-beta.1",
       "cpu": [
         "x64",
         "arm64"
@@ -52,12 +52,12 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.13.0",
-        "@lancedb/vectordb-darwin-x64": "0.13.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.13.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.13.0",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.13.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.13.0"
+        "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
+        "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
+        "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.13.0-beta.1",
+        "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -328,9 +328,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.13.0.tgz",
-      "integrity": "sha512-8hdcjkRmgrdQYf1jN+DyZae40LIv8UUfnWy70Uid5qy63sSvRW/+MvIdqIPFr9QlLUXmpyyQuX0y3bZhUR99cQ==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.13.0-beta.1.tgz",
+      "integrity": "sha512-beOrf6selCzzhLgDG8Nibma4nO/CSnA1wUKRmlJHEPtGcg7PW18z6MP/nfwQMpMR/FLRfTo8pPTbpzss47MiQQ==",
       "cpu": [
         "arm64"
       ],
@@ -340,9 +340,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.13.0.tgz",
-      "integrity": "sha512-fWzAY4l5SQtNfMYh80v+M66ugZHhdxbkpk5mNEv6Zsug3DL6kRj3Uv31/i0wgzY6F5G3LUlbjZerN+eTnDLwOw==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.13.0-beta.1.tgz",
+      "integrity": "sha512-YdraGRF/RbJRkKh0v3xT03LUhq47T2GtCvJ5gZp8wKlh4pHa8LuhLU0DIdvmG/DT5vuQA+td8HDkBm/e3EOdNg==",
       "cpu": [
         "x64"
       ],
@@ -352,9 +352,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.13.0.tgz",
-      "integrity": "sha512-ltwAT9baOSuR5YiGykQXPC8/HGYF13vpI47qxhP9yfgiz9pA8EUn8p8YrBRzq7J4DIZ4b8JSVDXQnMIqEtB4Kg==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.13.0-beta.1.tgz",
+      "integrity": "sha512-Pp0O/uhEqof1oLaWrNbv+Ym+q8kBkiCqaA5+2eAZ6a3e9U+Ozkvb0FQrHuyi9adJ5wKQ4NabyQE9BMf2bYpOnQ==",
       "cpu": [
         "arm64"
       ],
@@ -364,9 +364,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.13.0.tgz",
-      "integrity": "sha512-MiT/RBlMPGGRh7BX+MXwRuNiiUnKmuDcHH8nm88IH28T7TQxXIbA9w6UpSg5m9f3DgKQI2K8oLi29oKIB8ZwDQ==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.13.0-beta.1.tgz",
+      "integrity": "sha512-y8nxOye4egfWF5FGED9EfkmZ1O5HnRLU4a61B8m5JSpkivO9v2epTcbYN0yt/7ZFCgtqMfJ8VW4Mi7qQcz3KDA==",
       "cpu": [
         "x64"
       ],
@@ -376,9 +376,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.13.0.tgz",
-      "integrity": "sha512-SovP/hwWYLJIy65DKbVuXlBPTb/nwvVpTO6dh9zRch+L5ek6JmVAkwsfeTS2p5bMa8VPujsCXYUAVuCDEJU8wg==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.13.0-beta.1.tgz",
+      "integrity": "sha512-STMDP9dp0TBLkB3ro+16pKcGy6bmbhRuEZZZ1Tp5P75yTPeVh4zIgWkidMdU1qBbEYM7xacnsp9QAwgLnMU/Ow==",
       "cpu": [
         "x64"
       ],
@@ -1501,9 +1501,9 @@
       "dev": true
     },
     "node_modules/cross-spawn": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
-      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
       "dev": true,
       "dependencies": {
         "path-key": "^3.1.0",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.13.0",
+  "version": "0.13.0-beta.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -89,11 +89,11 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.13.0",
-    "@lancedb/vectordb-darwin-x64": "0.13.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.13.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.13.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.13.0",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.13.0"
+    "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
+    "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.13.0-beta.1"
   }
 }

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.13.0"
+version = "0.13.0-beta.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true

nodejs/__test__/table.test.ts

@@ -187,81 +187,6 @@ describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
       },
     );
 
-    // TODO: https://github.com/lancedb/lancedb/issues/1832
-    it.skip("should be able to omit nullable fields", async () => {
-      const db = await connect(tmpDir.name);
-      const schema = new arrow.Schema([
-        new arrow.Field(
-          "vector",
-          new arrow.FixedSizeList(
-            2,
-            new arrow.Field("item", new arrow.Float64()),
-          ),
-          true,
-        ),
-        new arrow.Field("item", new arrow.Utf8(), true),
-        new arrow.Field("price", new arrow.Float64(), false),
-      ]);
-      const table = await db.createEmptyTable("test", schema);
-
-      const data1 = { item: "foo", price: 10.0 };
-      await table.add([data1]);
-      const data2 = { vector: [3.1, 4.1], price: 2.0 };
-      await table.add([data2]);
-      const data3 = { vector: [5.9, 26.5], item: "bar", price: 3.0 };
-      await table.add([data3]);
-
-      let res = await table.query().limit(10).toArray();
-      const resVector = res.map((r) => r.get("vector").toArray());
-      expect(resVector).toEqual([null, data2.vector, data3.vector]);
-      const resItem = res.map((r) => r.get("item").toArray());
-      expect(resItem).toEqual(["foo", null, "bar"]);
-      const resPrice = res.map((r) => r.get("price").toArray());
-      expect(resPrice).toEqual([10.0, 2.0, 3.0]);
-
-      const data4 = { item: "foo" };
-      // We can't omit a column if it's not nullable
-      await expect(table.add([data4])).rejects.toThrow("Invalid user input");
-
-      // But we can alter columns to make them nullable
-      await table.alterColumns([{ path: "price", nullable: true }]);
-      await table.add([data4]);
-
-      res = (await table.query().limit(10).toArray()).map((r) => r.toJSON());
-      expect(res).toEqual([data1, data2, data3, data4]);
-    });
-
-    it("should be able to insert nullable data for non-nullable fields", async () => {
-      const db = await connect(tmpDir.name);
-      const schema = new arrow.Schema([
-        new arrow.Field("x", new arrow.Float64(), false),
-        new arrow.Field("id", new arrow.Utf8(), false),
-      ]);
-      const table = await db.createEmptyTable("test", schema);
-
-      const data1 = { x: 4.1, id: "foo" };
-      await table.add([data1]);
-      const res = (await table.query().toArray())[0];
-      expect(res.x).toEqual(data1.x);
-      expect(res.id).toEqual(data1.id);
-
-      const data2 = { x: null, id: "bar" };
-      await expect(table.add([data2])).rejects.toThrow(
-        "declared as non-nullable but contains null values",
-      );
-
-      // But we can alter columns to make them nullable
-      await table.alterColumns([{ path: "x", nullable: true }]);
-      await table.add([data2]);
-
-      const res2 = await table.query().toArray();
-      expect(res2.length).toBe(2);
-      expect(res2[0].x).toEqual(data1.x);
-      expect(res2[0].id).toEqual(data1.id);
-      expect(res2[1].x).toBeNull();
-      expect(res2[1].id).toEqual(data2.id);
-    });
-
     it("should return the table as an instance of an arrow table", async () => {
       const arrowTbl = await table.toArrow();
       expect(arrowTbl).toBeInstanceOf(ArrowTable);
@@ -477,54 +402,6 @@ describe("When creating an index", () => {
     expect(rst.numRows).toBe(1);
   });
 
-  it("should create and search IVF_HNSW indices", async () => {
-    await tbl.createIndex("vec", {
-      config: Index.hnswSq(),
-    });
-
-    // check index directory
-    const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
-    expect(fs.readdirSync(indexDir)).toHaveLength(1);
-    const indices = await tbl.listIndices();
-    expect(indices.length).toBe(1);
-    expect(indices[0]).toEqual({
-      name: "vec_idx",
-      indexType: "IvfHnswSq",
-      columns: ["vec"],
-    });
-
-    // Search without specifying the column
-    let rst = await tbl
-      .query()
-      .limit(2)
-      .nearestTo(queryVec)
-      .distanceType("dot")
-      .toArrow();
-    expect(rst.numRows).toBe(2);
-
-    // Search using `vectorSearch`
-    rst = await tbl.vectorSearch(queryVec).limit(2).toArrow();
-    expect(rst.numRows).toBe(2);
-
-    // Search with specifying the column
-    const rst2 = await tbl
-      .query()
-      .limit(2)
-      .nearestTo(queryVec)
-      .column("vec")
-      .toArrow();
-    expect(rst2.numRows).toBe(2);
-    expect(rst.toString()).toEqual(rst2.toString());
-
-    // test offset
-    rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
-    expect(rst.numRows).toBe(1);
-
-    // test ef
-    rst = await tbl.query().limit(2).nearestTo(queryVec).ef(100).toArrow();
-    expect(rst.numRows).toBe(2);
-  });
-
   it("should be able to query unindexed data", async () => {
     await tbl.createIndex("vec");
     await tbl.add([
@@ -1121,18 +998,4 @@ describe("column name options", () => {
     const results = await table.query().where("`camelCase` = 1").toArray();
     expect(results[0].camelCase).toBe(1);
   });
-
-  test("can make multiple vector queries in one go", async () => {
-    const results = await table
-      .query()
-      .nearestTo([0.1, 0.2])
-      .addQueryVector([0.1, 0.2])
-      .limit(1)
-      .toArray();
-    console.log(results);
-    expect(results.length).toBe(2);
-    results.sort((a, b) => a.query_index - b.query_index);
-    expect(results[0].query_index).toBe(0);
-    expect(results[1].query_index).toBe(1);
-  });
 });

nodejs/biome.json

@@ -9,8 +9,7 @@
       "**/native.js",
       "**/native.d.ts",
       "**/npm/**/*",
-      "**/.vscode/**",
-      "./examples/*"
+      "**/.vscode/**"
     ]
   },
   "formatter": {

nodejs/examples/ann_indexes.test.ts

@@ -1,57 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-// --8<-- [start:import]
-import * as lancedb from "@lancedb/lancedb";
-import { VectorQuery } from "@lancedb/lancedb";
-// --8<-- [end:import]
-import { withTempDirectory } from "./util.ts";
-
-test("ann index examples", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    // --8<-- [start:ingest]
-    const db = await lancedb.connect(databaseDir);
-
-    const data = Array.from({ length: 5_000 }, (_, i) => ({
-      vector: Array(128).fill(i),
-      id: `${i}`,
-      content: "",
-      longId: `${i}`,
-    }));
-
-    const table = await db.createTable("my_vectors", data, {
-      mode: "overwrite",
-    });
-    await table.createIndex("vector", {
-      config: lancedb.Index.ivfPq({
-        numPartitions: 10,
-        numSubVectors: 16,
-      }),
-    });
-    // --8<-- [end:ingest]
-
-    // --8<-- [start:search1]
-    const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;
-    const results1 = await search.nprobes(20).refineFactor(10).toArray();
-    // --8<-- [end:search1]
-    expect(results1.length).toBe(2);
-
-    // --8<-- [start:search2]
-    const results2 = await table
-      .search(Array(128).fill(1.2))
-      .where("id != '1141'")
-      .limit(2)
-      .toArray();
-    // --8<-- [end:search2]
-    expect(results2.length).toBe(2);
-
-    // --8<-- [start:search3]
-    const results3 = await table
-      .search(Array(128).fill(1.2))
-      .select(["id"])
-      .limit(2)
-      .toArray();
-    // --8<-- [end:search3]
-    expect(results3.length).toBe(2);
-  });
-}, 100_000);

nodejs/examples/ann_indexes.ts

@@ -0,0 +1,49 @@
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+
+// --8<-- [start:ingest]
+const db = await lancedb.connect("/tmp/lancedb/");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: `${i}`,
+  content: "",
+  longId: `${i}`,
+}));
+
+const table = await db.createTable("my_vectors", data, { mode: "overwrite" });
+await table.createIndex("vector", {
+  config: lancedb.Index.ivfPq({
+    numPartitions: 16,
+    numSubVectors: 48,
+  }),
+});
+// --8<-- [end:ingest]
+
+// --8<-- [start:search1]
+const _results1 = await table
+  .search(Array(1536).fill(1.2))
+  .limit(2)
+  .nprobes(20)
+  .refineFactor(10)
+  .toArray();
+// --8<-- [end:search1]
+
+// --8<-- [start:search2]
+const _results2 = await table
+  .search(Array(1536).fill(1.2))
+  .where("id != '1141'")
+  .limit(2)
+  .toArray();
+// --8<-- [end:search2]
+
+// --8<-- [start:search3]
+const _results3 = await table
+  .search(Array(1536).fill(1.2))
+  .select(["id"])
+  .limit(2)
+  .toArray();
+// --8<-- [end:search3]
+
+console.log("Ann indexes: done");

nodejs/examples/basic.test.ts

@@ -1,175 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-// --8<--  [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import * as arrow from "apache-arrow";
-import {
-  Field,
-  FixedSizeList,
-  Float16,
-  Int32,
-  Schema,
-  Utf8,
-} from "apache-arrow";
-// --8<-- [end:imports]
-import { withTempDirectory } from "./util.ts";
-
-test("basic table examples", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    // --8<-- [start:connect]
-    const db = await lancedb.connect(databaseDir);
-    // --8<-- [end:connect]
-    {
-      // --8<-- [start:create_table]
-      const _tbl = await db.createTable(
-        "myTable",
-        [
-          { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-          { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-        ],
-        { mode: "overwrite" },
-      );
-      // --8<-- [end:create_table]
-
-      const data = [
-        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-      ];
-
-      {
-        // --8<-- [start:create_table_exists_ok]
-        const tbl = await db.createTable("myTable", data, {
-          existOk: true,
-        });
-        // --8<-- [end:create_table_exists_ok]
-        expect(await tbl.countRows()).toBe(2);
-      }
-      {
-        // --8<-- [start:create_table_overwrite]
-        const tbl = await db.createTable("myTable", data, {
-          mode: "overwrite",
-        });
-        // --8<-- [end:create_table_overwrite]
-        expect(await tbl.countRows()).toBe(2);
-      }
-    }
-
-    await db.dropTable("myTable");
-
-    {
-      // --8<-- [start:create_table_with_schema]
-      const schema = new arrow.Schema([
-        new arrow.Field(
-          "vector",
-          new arrow.FixedSizeList(
-            2,
-            new arrow.Field("item", new arrow.Float32(), true),
-          ),
-        ),
-        new arrow.Field("item", new arrow.Utf8(), true),
-        new arrow.Field("price", new arrow.Float32(), true),
-      ]);
-      const data = [
-        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-      ];
-      const tbl = await db.createTable("myTable", data, {
-        schema,
-      });
-      // --8<-- [end:create_table_with_schema]
-      expect(await tbl.countRows()).toBe(2);
-    }
-
-    {
-      // --8<-- [start:create_empty_table]
-
-      const schema = new arrow.Schema([
-        new arrow.Field("id", new arrow.Int32()),
-        new arrow.Field("name", new arrow.Utf8()),
-      ]);
-
-      const emptyTbl = await db.createEmptyTable("empty_table", schema);
-      // --8<-- [end:create_empty_table]
-      expect(await emptyTbl.countRows()).toBe(0);
-    }
-    {
-      // --8<-- [start:open_table]
-      const _tbl = await db.openTable("myTable");
-      // --8<-- [end:open_table]
-    }
-
-    {
-      // --8<-- [start:table_names]
-      const tableNames = await db.tableNames();
-      // --8<-- [end:table_names]
-      expect(tableNames).toEqual(["empty_table", "myTable"]);
-    }
-
-    const tbl = await db.openTable("myTable");
-    {
-      // --8<-- [start:add_data]
-      const data = [
-        { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
-        { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
-      ];
-      await tbl.add(data);
-      // --8<-- [end:add_data]
-    }
-    {
-      // --8<-- [start:vector_search]
-      const res = await tbl.search([100, 100]).limit(2).toArray();
-      // --8<-- [end:vector_search]
-      expect(res.length).toBe(2);
-    }
-    {
-      const data = Array.from({ length: 1000 })
-        .fill(null)
-        .map(() => ({
-          vector: [Math.random(), Math.random()],
-          item: "autogen",
-          price: Math.round(Math.random() * 100),
-        }));
-
-      await tbl.add(data);
-    }
-
-    // --8<-- [start:create_index]
-    await tbl.createIndex("vector");
-    // --8<-- [end:create_index]
-
-    // --8<-- [start:delete_rows]
-    await tbl.delete('item = "fizz"');
-    // --8<-- [end:delete_rows]
-
-    // --8<-- [start:drop_table]
-    await db.dropTable("myTable");
-    // --8<-- [end:drop_table]
-    await db.dropTable("empty_table");
-
-    {
-      // --8<-- [start:create_f16_table]
-      const db = await lancedb.connect(databaseDir);
-      const dim = 16;
-      const total = 10;
-      const f16Schema = new Schema([
-        new Field("id", new Int32()),
-        new Field(
-          "vector",
-          new FixedSizeList(dim, new Field("item", new Float16(), true)),
-          false,
-        ),
-      ]);
-      const data = lancedb.makeArrowTable(
-        Array.from(Array(total), (_, i) => ({
-          id: i,
-          vector: Array.from(Array(dim), Math.random),
-        })),
-        { schema: f16Schema },
-      );
-      const _table = await db.createTable("f16_tbl", data);
-      // --8<-- [end:create_f16_table]
-      await db.dropTable("f16_tbl");
-    }
-  });
-});

nodejs/examples/basic.ts

@@ -0,0 +1,162 @@
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import * as arrow from "apache-arrow";
+import {
+  Field,
+  FixedSizeList,
+  Float16,
+  Int32,
+  Schema,
+  Utf8,
+} from "apache-arrow";
+
+// --8<-- [end:imports]
+
+// --8<-- [start:connect]
+const uri = "/tmp/lancedb/";
+const db = await lancedb.connect(uri);
+// --8<-- [end:connect]
+{
+  // --8<-- [start:create_table]
+  const tbl = await db.createTable(
+    "myTable",
+    [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ],
+    { mode: "overwrite" },
+  );
+  // --8<-- [end:create_table]
+
+  const data = [
+    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+  ];
+
+  {
+    // --8<-- [start:create_table_exists_ok]
+    const tbl = await db.createTable("myTable", data, {
+      existsOk: true,
+    });
+    // --8<-- [end:create_table_exists_ok]
+  }
+  {
+    // --8<-- [start:create_table_overwrite]
+    const _tbl = await db.createTable("myTable", data, {
+      mode: "overwrite",
+    });
+    // --8<-- [end:create_table_overwrite]
+  }
+}
+
+{
+  // --8<-- [start:create_table_with_schema]
+  const schema = new arrow.Schema([
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+    ),
+    new arrow.Field("item", new arrow.Utf8(), true),
+    new arrow.Field("price", new arrow.Float32(), true),
+  ]);
+  const data = [
+    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+  ];
+  const _tbl = await db.createTable("myTable", data, {
+    schema,
+  });
+  // --8<-- [end:create_table_with_schema]
+}
+
+{
+  // --8<-- [start:create_empty_table]
+
+  const schema = new arrow.Schema([
+    new arrow.Field("id", new arrow.Int32()),
+    new arrow.Field("name", new arrow.Utf8()),
+  ]);
+
+  const empty_tbl = await db.createEmptyTable("empty_table", schema);
+  // --8<-- [end:create_empty_table]
+}
+{
+  // --8<-- [start:open_table]
+  const _tbl = await db.openTable("myTable");
+  // --8<-- [end:open_table]
+}
+
+{
+  // --8<-- [start:table_names]
+  const tableNames = await db.tableNames();
+  console.log(tableNames);
+  // --8<-- [end:table_names]
+}
+
+const tbl = await db.openTable("myTable");
+{
+  // --8<-- [start:add_data]
+  const data = [
+    { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
+    { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
+  ];
+  await tbl.add(data);
+  // --8<-- [end:add_data]
+}
+{
+  // --8<-- [start:vector_search]
+  const _res = tbl.search([100, 100]).limit(2).toArray();
+  // --8<-- [end:vector_search]
+}
+{
+  const data = Array.from({ length: 1000 })
+    .fill(null)
+    .map(() => ({
+      vector: [Math.random(), Math.random()],
+      item: "autogen",
+      price: Math.round(Math.random() * 100),
+    }));
+
+  await tbl.add(data);
+}
+
+// --8<-- [start:create_index]
+await tbl.createIndex("vector");
+// --8<-- [end:create_index]
+
+// --8<-- [start:delete_rows]
+await tbl.delete('item = "fizz"');
+// --8<-- [end:delete_rows]
+
+// --8<-- [start:drop_table]
+await db.dropTable("myTable");
+// --8<-- [end:drop_table]
+await db.dropTable("empty_table");
+
+{
+  // --8<-- [start:create_f16_table]
+  const db = await lancedb.connect("/tmp/lancedb");
+  const dim = 16;
+  const total = 10;
+  const f16Schema = new Schema([
+    new Field("id", new Int32()),
+    new Field(
+      "vector",
+      new FixedSizeList(dim, new Field("item", new Float16(), true)),
+      false,
+    ),
+  ]);
+  const data = lancedb.makeArrowTable(
+    Array.from(Array(total), (_, i) => ({
+      id: i,
+      vector: Array.from(Array(dim), Math.random),
+    })),
+    { schema: f16Schema },
+  );
+  const _table = await db.createTable("f16_tbl", data);
+  // --8<-- [end:create_f16_table]
+  await db.dropTable("f16_tbl");
+}

nodejs/examples/custom_embedding_function.test.ts

@@ -1,76 +0,0 @@
-import { FeatureExtractionPipeline, pipeline } from "@huggingface/transformers";
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-// --8<-- [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import {
-  LanceSchema,
-  TextEmbeddingFunction,
-  getRegistry,
-  register,
-} from "@lancedb/lancedb/embedding";
-// --8<-- [end:imports]
-import { withTempDirectory } from "./util.ts";
-
-// --8<-- [start:embedding_impl]
-@register("sentence-transformers")
-class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
-  name = "Xenova/all-miniLM-L6-v2";
-  #ndims!: number;
-  extractor!: FeatureExtractionPipeline;
-
-  async init() {
-    this.extractor = await pipeline("feature-extraction", this.name, {
-      dtype: "fp32",
-    });
-    this.#ndims = await this.generateEmbeddings(["hello"]).then(
-      (e) => e[0].length,
-    );
-  }
-
-  ndims() {
-    return this.#ndims;
-  }
-
-  toJSON() {
-    return {
-      name: this.name,
-    };
-  }
-  async generateEmbeddings(texts: string[]) {
-    const output = await this.extractor(texts, {
-      pooling: "mean",
-      normalize: true,
-    });
-    return output.tolist();
-  }
-}
-// -8<-- [end:embedding_impl]
-
-test("Registry examples", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    // --8<-- [start:call_custom_function]
-    const registry = getRegistry();
-
-    const sentenceTransformer = await registry
-      .get<SentenceTransformersEmbeddings>("sentence-transformers")!
-      .create();
-
-    const schema = LanceSchema({
-      vector: sentenceTransformer.vectorField(),
-      text: sentenceTransformer.sourceField(),
-    });
-
-    const db = await lancedb.connect(databaseDir);
-    const table = await db.createEmptyTable("table", schema, {
-      mode: "overwrite",
-    });
-
-    await table.add([{ text: "hello" }, { text: "world" }]);
-
-    const results = await table.search("greeting").limit(1).toArray();
-    // -8<-- [end:call_custom_function]
-    expect(results.length).toBe(1);
-  });
-}, 100_000);

nodejs/examples/custom_embedding_function.ts

@@ -0,0 +1,64 @@
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import {
+  LanceSchema,
+  TextEmbeddingFunction,
+  getRegistry,
+  register,
+} from "@lancedb/lancedb/embedding";
+import { pipeline } from "@xenova/transformers";
+// --8<-- [end:imports]
+
+// --8<-- [start:embedding_impl]
+@register("sentence-transformers")
+class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
+  name = "Xenova/all-miniLM-L6-v2";
+  #ndims!: number;
+  extractor: any;
+
+  async init() {
+    this.extractor = await pipeline("feature-extraction", this.name);
+    this.#ndims = await this.generateEmbeddings(["hello"]).then(
+      (e) => e[0].length,
+    );
+  }
+
+  ndims() {
+    return this.#ndims;
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+    };
+  }
+  async generateEmbeddings(texts: string[]) {
+    const output = await this.extractor(texts, {
+      pooling: "mean",
+      normalize: true,
+    });
+    return output.tolist();
+  }
+}
+// -8<-- [end:embedding_impl]
+
+// --8<-- [start:call_custom_function]
+const registry = getRegistry();
+
+const sentenceTransformer = await registry
+  .get<SentenceTransformersEmbeddings>("sentence-transformers")!
+  .create();
+
+const schema = LanceSchema({
+  vector: sentenceTransformer.vectorField(),
+  text: sentenceTransformer.sourceField(),
+});
+
+const db = await lancedb.connect("/tmp/db");
+const table = await db.createEmptyTable("table", schema, { mode: "overwrite" });
+
+await table.add([{ text: "hello" }, { text: "world" }]);
+
+const results = await table.search("greeting").limit(1).toArray();
+console.log(results[0].text);
+// -8<-- [end:call_custom_function]

nodejs/examples/embedding.test.ts

@@ -1,96 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-// --8<-- [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import "@lancedb/lancedb/embedding/openai";
-import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
-import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
-import { type Float, Float32, Utf8 } from "apache-arrow";
-// --8<-- [end:imports]
-import { withTempDirectory } from "./util.ts";
-
-const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test;
-
-openAiTest("openai embeddings", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    // --8<-- [start:openai_embeddings]
-    const db = await lancedb.connect(databaseDir);
-    const func = getRegistry()
-      .get("openai")
-      ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
-
-    const wordsSchema = LanceSchema({
-      text: func.sourceField(new Utf8()),
-      vector: func.vectorField(),
-    });
-    const tbl = await db.createEmptyTable("words", wordsSchema, {
-      mode: "overwrite",
-    });
-    await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
-
-    const query = "greetings";
-    const actual = (await tbl.search(query).limit(1).toArray())[0];
-    // --8<-- [end:openai_embeddings]
-    expect(actual).toHaveProperty("text");
-  });
-});
-
-test("custom embedding function", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    // --8<-- [start:embedding_function]
-    const db = await lancedb.connect(databaseDir);
-
-    @register("my_embedding")
-    class MyEmbeddingFunction extends EmbeddingFunction<string> {
-      toJSON(): object {
-        return {};
-      }
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): Float {
-        return new Float32();
-      }
-      async computeQueryEmbeddings(_data: string) {
-        // This is a placeholder for a real embedding function
-        return [1, 2, 3];
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        // This is a placeholder for a real embedding function
-        return Array.from({ length: data.length }).fill([
-          1, 2, 3,
-        ]) as number[][];
-      }
-    }
-
-    const func = new MyEmbeddingFunction();
-
-    const data = [{ text: "pepperoni" }, { text: "pineapple" }];
-
-    // Option 1: manually specify the embedding function
-    const table = await db.createTable("vectors", data, {
-      embeddingFunction: {
-        function: func,
-        sourceColumn: "text",
-        vectorColumn: "vector",
-      },
-      mode: "overwrite",
-    });
-
-    // Option 2: provide the embedding function through a schema
-
-    const schema = LanceSchema({
-      text: func.sourceField(new Utf8()),
-      vector: func.vectorField(),
-    });
-
-    const table2 = await db.createTable("vectors2", data, {
-      schema,
-      mode: "overwrite",
-    });
-    // --8<-- [end:embedding_function]
-    expect(await table.countRows()).toBe(2);
-    expect(await table2.countRows()).toBe(2);
-  });
-});

nodejs/examples/embedding.ts

@@ -0,0 +1,83 @@
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { type Float, Float32, Utf8 } from "apache-arrow";
+// --8<-- [end:imports]
+
+{
+  // --8<-- [start:openai_embeddings]
+
+  const db = await lancedb.connect("/tmp/db");
+  const func = getRegistry()
+    .get("openai")
+    ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+  const wordsSchema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+  const tbl = await db.createEmptyTable("words", wordsSchema, {
+    mode: "overwrite",
+  });
+  await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+  const query = "greetings";
+  const actual = (await (await tbl.search(query)).limit(1).toArray())[0];
+
+  // --8<-- [end:openai_embeddings]
+  console.log("result = ", actual.text);
+}
+
+{
+  // --8<-- [start:embedding_function]
+  const db = await lancedb.connect("/tmp/db");
+
+  @register("my_embedding")
+  class MyEmbeddingFunction extends EmbeddingFunction<string> {
+    toJSON(): object {
+      return {};
+    }
+    ndims() {
+      return 3;
+    }
+    embeddingDataType(): Float {
+      return new Float32();
+    }
+    async computeQueryEmbeddings(_data: string) {
+      // This is a placeholder for a real embedding function
+      return [1, 2, 3];
+    }
+    async computeSourceEmbeddings(data: string[]) {
+      // This is a placeholder for a real embedding function
+      return Array.from({ length: data.length }).fill([1, 2, 3]) as number[][];
+    }
+  }
+
+  const func = new MyEmbeddingFunction();
+
+  const data = [{ text: "pepperoni" }, { text: "pineapple" }];
+
+  // Option 1: manually specify the embedding function
+  const table = await db.createTable("vectors", data, {
+    embeddingFunction: {
+      function: func,
+      sourceColumn: "text",
+      vectorColumn: "vector",
+    },
+    mode: "overwrite",
+  });
+
+  // Option 2: provide the embedding function through a schema
+
+  const schema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+
+  const table2 = await db.createTable("vectors2", data, {
+    schema,
+    mode: "overwrite",
+  });
+  // --8<-- [end:embedding_function]
+}

nodejs/examples/filtering.test.ts

@@ -1,42 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-import * as lancedb from "@lancedb/lancedb";
-import { withTempDirectory } from "./util.ts";
-
-test("filtering examples", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    const db = await lancedb.connect(databaseDir);
-
-    const data = Array.from({ length: 10_000 }, (_, i) => ({
-      vector: Array(1536).fill(i),
-      id: i,
-      item: `item ${i}`,
-      strId: `${i}`,
-    }));
-
-    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
-
-    // --8<-- [start:search]
-    const _result = await tbl
-      .search(Array(1536).fill(0.5))
-      .limit(1)
-      .where("id = 10")
-      .toArray();
-    // --8<-- [end:search]
-
-    // --8<-- [start:vec_search]
-    const result = await (
-      tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery
-    )
-      .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-      .postfilter()
-      .toArray();
-    // --8<-- [end:vec_search]
-    expect(result.length).toBe(0);
-
-    // --8<-- [start:sql_search]
-    await tbl.query().where("id = 10").limit(10).toArray();
-    // --8<-- [end:sql_search]
-  });
-});

nodejs/examples/filtering.ts

@@ -0,0 +1,34 @@
+import * as lancedb from "@lancedb/lancedb";
+
+const db = await lancedb.connect("data/sample-lancedb");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: i,
+  item: `item ${i}`,
+  strId: `${i}`,
+}));
+
+const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+// --8<-- [start:search]
+const _result = await tbl
+  .search(Array(1536).fill(0.5))
+  .limit(1)
+  .where("id = 10")
+  .toArray();
+// --8<-- [end:search]
+
+// --8<-- [start:vec_search]
+await tbl
+  .search(Array(1536).fill(0))
+  .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+  .postfilter()
+  .toArray();
+// --8<-- [end:vec_search]
+
+// --8<-- [start:sql_search]
+await tbl.query().where("id = 10").limit(10).toArray();
+// --8<-- [end:sql_search]
+
+console.log("SQL search: done");

nodejs/examples/full_text_search.test.ts

@@ -1,45 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-import * as lancedb from "@lancedb/lancedb";
-import { withTempDirectory } from "./util.ts";
-
-test("full text search", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    const db = await lancedb.connect(databaseDir);
-
-    const words = [
-      "apple",
-      "banana",
-      "cherry",
-      "date",
-      "elderberry",
-      "fig",
-      "grape",
-    ];
-
-    const data = Array.from({ length: 10_000 }, (_, i) => ({
-      vector: Array(1536).fill(i),
-      id: i,
-      item: `item ${i}`,
-      strId: `${i}`,
-      doc: words[i % words.length],
-    }));
-
-    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
-
-    await tbl.createIndex("doc", {
-      config: lancedb.Index.fts(),
-    });
-
-    // --8<-- [start:full_text_search]
-    const result = await tbl
-      .query()
-      .nearestToText("apple")
-      .select(["id", "doc"])
-      .limit(10)
-      .toArray();
-    expect(result.length).toBe(10);
-    // --8<-- [end:full_text_search]
-  });
-});

nodejs/examples/full_text_search.ts

@@ -0,0 +1,52 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as lancedb from "@lancedb/lancedb";
+
+const db = await lancedb.connect("data/sample-lancedb");
+
+const words = [
+  "apple",
+  "banana",
+  "cherry",
+  "date",
+  "elderberry",
+  "fig",
+  "grape",
+];
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: i,
+  item: `item ${i}`,
+  strId: `${i}`,
+  doc: words[i % words.length],
+}));
+
+const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+await tbl.createIndex("doc", {
+  config: lancedb.Index.fts(),
+});
+
+// --8<-- [start:full_text_search]
+let result = await tbl
+  .search("apple")
+  .select(["id", "doc"])
+  .limit(10)
+  .toArray();
+console.log(result);
+// --8<-- [end:full_text_search]
+
+console.log("SQL search: done");

nodejs/examples/jest.config.cjs

@@ -1,6 +0,0 @@
-/** @type {import('ts-jest').JestConfigWithTsJest} */
-module.exports = {
-  preset: "ts-jest",
-  testEnvironment: "node",
-  testPathIgnorePatterns: ["./dist"],
-};

nodejs/examples/jsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

nodejs/examples/package.json

@@ -5,29 +5,24 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
-    "//2": "--testEnvironment is needed to run jest with sentence-transformers",
-    "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
-    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
-    "lint": "biome check *.ts && biome format *.ts",
-    "lint-ci": "biome ci .",
-    "lint-fix": "biome check --write *.ts && npm run format",
-    "format": "biome format --write *.ts"
+    "test": "echo \"Error: no test specified\" && exit 1"
   },
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "dependencies": {
-    "@huggingface/transformers": "^3.0.2",
-    "@lancedb/lancedb": "file:../dist",
-    "openai": "^4.29.2",
-    "sharp": "^0.33.5"
+    "@lancedb/lancedb": "file:../",
+    "@xenova/transformers": "^2.17.2"
   },
   "devDependencies": {
-    "@biomejs/biome": "^1.7.3",
-    "@jest/globals": "^29.7.0",
-    "jest": "^29.7.0",
-    "jest-environment-node-single-context": "^29.4.0",
-    "ts-jest": "^29.2.5",
     "typescript": "^5.5.4"
+  },
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
   }
 }

nodejs/examples/search.test.ts

@@ -1,42 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-// --8<-- [start:import]
-import * as lancedb from "@lancedb/lancedb";
-// --8<-- [end:import]
-import { withTempDirectory } from "./util.ts";
-
-test("full text search", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    {
-      const db = await lancedb.connect(databaseDir);
-
-      const data = Array.from({ length: 10_000 }, (_, i) => ({
-        vector: Array(128).fill(i),
-        id: `${i}`,
-        content: "",
-        longId: `${i}`,
-      }));
-
-      await db.createTable("my_vectors", data);
-    }
-
-    // --8<-- [start:search1]
-    const db = await lancedb.connect(databaseDir);
-    const tbl = await db.openTable("my_vectors");
-
-    const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();
-    // --8<-- [end:search1]
-    expect(results1.length).toBe(10);
-
-    // --8<-- [start:search2]
-    const results2 = await (
-      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
-    )
-      .distanceType("cosine")
-      .limit(10)
-      .toArray();
-    // --8<-- [end:search2]
-    expect(results2.length).toBe(10);
-  });
-});

nodejs/examples/search.ts

@@ -0,0 +1,38 @@
+// --8<-- [end:import]
+import * as fs from "node:fs";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+
+async function setup() {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  const data = Array.from({ length: 10_000 }, (_, i) => ({
+    vector: Array(1536).fill(i),
+    id: `${i}`,
+    content: "",
+    longId: `${i}`,
+  }));
+
+  await db.createTable("my_vectors", data);
+}
+
+await setup();
+
+// --8<-- [start:search1]
+const db = await lancedb.connect("data/sample-lancedb");
+const tbl = await db.openTable("my_vectors");
+
+const _results1 = await tbl.search(Array(1536).fill(1.2)).limit(10).toArray();
+// --8<-- [end:search1]
+
+// --8<-- [start:search2]
+const _results2 = await tbl
+  .search(Array(1536).fill(1.2))
+  .distanceType("cosine")
+  .limit(10)
+  .toArray();
+console.log(_results2);
+// --8<-- [end:search2]
+
+console.log("search: done");

nodejs/examples/sentence-transformers.js

@@ -0,0 +1,50 @@
+import * as lancedb from "@lancedb/lancedb";
+
+import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
+import { Utf8 } from "apache-arrow";
+
+const db = await lancedb.connect("/tmp/db");
+const func = await getRegistry().get("huggingface").create();
+
+const facts = [
+  "Albert Einstein was a theoretical physicist.",
+  "The capital of France is Paris.",
+  "The Great Wall of China is one of the Seven Wonders of the World.",
+  "Python is a popular programming language.",
+  "Mount Everest is the highest mountain in the world.",
+  "Leonardo da Vinci painted the Mona Lisa.",
+  "Shakespeare wrote Hamlet.",
+  "The human body has 206 bones.",
+  "The speed of light is approximately 299,792 kilometers per second.",
+  "Water boils at 100 degrees Celsius.",
+  "The Earth orbits the Sun.",
+  "The Pyramids of Giza are located in Egypt.",
+  "Coffee is one of the most popular beverages in the world.",
+  "Tokyo is the capital city of Japan.",
+  "Photosynthesis is the process by which plants make their food.",
+  "The Pacific Ocean is the largest ocean on Earth.",
+  "Mozart was a prolific composer of classical music.",
+  "The Internet is a global network of computers.",
+  "Basketball is a sport played with a ball and a hoop.",
+  "The first computer virus was created in 1983.",
+  "Artificial neural networks are inspired by the human brain.",
+  "Deep learning is a subset of machine learning.",
+  "IBM's Watson won Jeopardy! in 2011.",
+  "The first computer programmer was Ada Lovelace.",
+  "The first chatbot was ELIZA, created in the 1960s.",
+].map((text) => ({ text }));
+
+const factsSchema = LanceSchema({
+  text: func.sourceField(new Utf8()),
+  vector: func.vectorField(),
+});
+
+const tbl = await db.createTable("facts", facts, {
+  mode: "overwrite",
+  schema: factsSchema,
+});
+
+const query = "How many bones are in the human body?";
+const actual = await tbl.search(query).limit(1).toArray();
+
+console.log("Answer: ", actual[0]["text"]);

nodejs/examples/sentence-transformers.test.ts

@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { expect, test } from "@jest/globals";
-import { withTempDirectory } from "./util.ts";
-
-import * as lancedb from "@lancedb/lancedb";
-import "@lancedb/lancedb/embedding/transformers";
-import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
-import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
-import { Utf8 } from "apache-arrow";
-
-test("full text search", async () => {
-  await withTempDirectory(async (databaseDir) => {
-    const db = await lancedb.connect(databaseDir);
-    console.log(getRegistry());
-    const func = (await getRegistry()
-      .get("huggingface")
-      ?.create()) as EmbeddingFunction;
-
-    const facts = [
-      "Albert Einstein was a theoretical physicist.",
-      "The capital of France is Paris.",
-      "The Great Wall of China is one of the Seven Wonders of the World.",
-      "Python is a popular programming language.",
-      "Mount Everest is the highest mountain in the world.",
-      "Leonardo da Vinci painted the Mona Lisa.",
-      "Shakespeare wrote Hamlet.",
-      "The human body has 206 bones.",
-      "The speed of light is approximately 299,792 kilometers per second.",
-      "Water boils at 100 degrees Celsius.",
-      "The Earth orbits the Sun.",
-      "The Pyramids of Giza are located in Egypt.",
-      "Coffee is one of the most popular beverages in the world.",
-      "Tokyo is the capital city of Japan.",
-      "Photosynthesis is the process by which plants make their food.",
-      "The Pacific Ocean is the largest ocean on Earth.",
-      "Mozart was a prolific composer of classical music.",
-      "The Internet is a global network of computers.",
-      "Basketball is a sport played with a ball and a hoop.",
-      "The first computer virus was created in 1983.",
-      "Artificial neural networks are inspired by the human brain.",
-      "Deep learning is a subset of machine learning.",
-      "IBM's Watson won Jeopardy! in 2011.",
-      "The first computer programmer was Ada Lovelace.",
-      "The first chatbot was ELIZA, created in the 1960s.",
-    ].map((text) => ({ text }));
-
-    const factsSchema = LanceSchema({
-      text: func.sourceField(new Utf8()),
-      vector: func.vectorField(),
-    });
-
-    const tbl = await db.createTable("facts", facts, {
-      mode: "overwrite",
-      schema: factsSchema,
-    });
-
-    const query = "How many bones are in the human body?";
-    const actual = await tbl.search(query).limit(1).toArray();
-
-    expect(actual[0]["text"]).toBe("The human body has 206 bones.");
-  });
-}, 100_000);

nodejs/examples/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "include": ["*.test.ts"],
-  "compilerOptions": {
-    "target": "es2022",
-    "module": "NodeNext",
-    "declaration": true,
-    "outDir": "./dist",
-    "strict": true,
-    "allowJs": true,
-    "resolveJsonModule": true,
-    "emitDecoratorMetadata": true,
-    "experimentalDecorators": true,
-    "moduleResolution": "NodeNext",
-    "allowImportingTsExtensions": true,
-    "emitDeclarationOnly": true
-  }
-}

nodejs/examples/util.ts

@@ -1,16 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import * as fs from "fs";
-import { tmpdir } from "os";
-import * as path from "path";
-
-export async function withTempDirectory(
-  fn: (tempDir: string) => Promise<void>,
-) {
-  const tmpDirPath = fs.mkdtempSync(path.join(tmpdir(), "temp-dir-"));
-  try {
-    await fn(tmpDirPath);
-  } finally {
-    fs.rmSync(tmpDirPath, { recursive: true });
-  }
-}

nodejs/jest.config.js

@@ -4,5 +4,4 @@ module.exports = {
   testEnvironment: "node",
   moduleDirectories: ["node_modules", "./dist"],
   moduleFileExtensions: ["js", "ts"],
-  modulePathIgnorePatterns: ["<rootDir>/examples/"],
 };

nodejs/lancedb/embedding/index.ts

@@ -19,6 +19,9 @@ import { EmbeddingFunctionConfig, getRegistry } from "./registry";
 
 export { EmbeddingFunction, TextEmbeddingFunction } from "./embedding_function";
 
+// We need to explicitly export '*' so that the `register` decorator actually registers the class.
+export * from "./openai";
+export * from "./transformers";
 export * from "./registry";
 
 /**

nodejs/lancedb/embedding/registry.ts

@@ -17,6 +17,8 @@ import {
   type EmbeddingFunctionConstructor,
 } from "./embedding_function";
 import "reflect-metadata";
+import { OpenAIEmbeddingFunction } from "./openai";
+import { TransformersEmbeddingFunction } from "./transformers";
 
 type CreateReturnType<T> = T extends { init: () => Promise<void> }
   ? Promise<T>
@@ -71,6 +73,10 @@ export class EmbeddingFunctionRegistry {
     };
   }
 
+  get(name: "openai"): EmbeddingFunctionCreate<OpenAIEmbeddingFunction>;
+  get(
+    name: "huggingface",
+  ): EmbeddingFunctionCreate<TransformersEmbeddingFunction>;
   get<T extends EmbeddingFunction<unknown>>(
     name: string,
   ): EmbeddingFunctionCreate<T> | undefined;

nodejs/lancedb/embedding/transformers.ts

@@ -47,8 +47,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
   string,
   Partial<XenovaTransformerOptions>
 > {
-  #model?: import("@huggingface/transformers").PreTrainedModel;
-  #tokenizer?: import("@huggingface/transformers").PreTrainedTokenizer;
+  #model?: import("@xenova/transformers").PreTrainedModel;
+  #tokenizer?: import("@xenova/transformers").PreTrainedTokenizer;
   #modelName: XenovaTransformerOptions["model"];
   #initialized = false;
   #tokenizerOptions: XenovaTransformerOptions["tokenizerOptions"];
@@ -92,19 +92,18 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
     try {
       // SAFETY:
       // since typescript transpiles `import` to `require`, we need to do this in an unsafe way
-      // We can't use `require` because `@huggingface/transformers` is an ESM module
+      // We can't use `require` because `@xenova/transformers` is an ESM module
       // and we can't use `import` directly because typescript will transpile it to `require`.
       // and we want to remain compatible with both ESM and CJS modules
       // so we use `eval` to bypass typescript for this specific import.
-      transformers = await eval('import("@huggingface/transformers")');
+      transformers = await eval('import("@xenova/transformers")');
     } catch (e) {
-      throw new Error(`error loading @huggingface/transformers\nReason: ${e}`);
+      throw new Error(`error loading @xenova/transformers\nReason: ${e}`);
     }
 
     try {
       this.#model = await transformers.AutoModel.from_pretrained(
         this.#modelName,
-        { dtype: "fp32" },
       );
     } catch (e) {
       throw new Error(
@@ -129,8 +128,7 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
     } else {
       const config = this.#model!.config;
 
-      // biome-ignore lint/style/useNamingConvention: we don't control this name.
-      const ndims = (config as unknown as { hidden_size: number }).hidden_size;
+      const ndims = config["hidden_size"];
       if (!ndims) {
         throw new Error(
           "hidden_size not found in model config, you may need to manually specify the embedding dimensions. ",
@@ -185,7 +183,7 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
 }
 
 const tensorDiv = (
-  src: import("@huggingface/transformers").Tensor,
+  src: import("@xenova/transformers").Tensor,
   divBy: number,
 ) => {
   for (let i = 0; i < src.data.length; ++i) {

nodejs/lancedb/query.ts

@@ -385,20 +385,6 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
     return this;
   }
 
-  /**
-   * Set the number of candidates to consider during the search
-   *
-   * This argument is only used when the vector column has an HNSW index.
-   * If there is no index then this value is ignored.
-   *
-   * Increasing this value will increase the recall of your query but will
-   * also increase the latency of your query. The default value is 1.5*limit.
-   */
-  ef(ef: number): VectorQuery {
-    super.doCall((inner) => inner.ef(ef));
-    return this;
-  }
-
   /**
    * Set the vector column to query
    *
@@ -506,42 +492,6 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
     super.doCall((inner) => inner.bypassVectorIndex());
     return this;
   }
-
-  /*
-   * Add a query vector to the search
-   *
-   * This method can be called multiple times to add multiple query vectors
-   * to the search. If multiple query vectors are added, then they will be searched
-   * in parallel, and the results will be concatenated. A column called `query_index`
-   * will be added to indicate the index of the query vector that produced the result.
-   *
-   * Performance wise, this is equivalent to running multiple queries concurrently.
-   */
-  addQueryVector(vector: IntoVector): VectorQuery {
-    if (vector instanceof Promise) {
-      const res = (async () => {
-        try {
-          const v = await vector;
-          const arr = Float32Array.from(v);
-          //
-          // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-          const value: any = this.addQueryVector(arr);
-          const inner = value.inner as
-            | NativeVectorQuery
-            | Promise<NativeVectorQuery>;
-          return inner;
-        } catch (e) {
-          return Promise.reject(e);
-        }
-      })();
-      return new VectorQuery(res);
-    } else {
-      super.doCall((inner) => {
-        inner.addQueryVector(Float32Array.from(vector));
-      });
-      return this;
-    }
-  }
 }
 
 /** A builder for LanceDB queries. */
@@ -621,9 +571,4 @@ export class Query extends QueryBase<NativeQuery> {
       return new VectorQuery(vectorQuery);
     }
   }
-
-  nearestToText(query: string, columns?: string[]): Query {
-    this.doCall((inner) => inner.fullTextSearch(query, columns));
-    return this;
-  }
 }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.13.0",
+	"version": "0.13.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.13.0",
+	"version": "0.13.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.13.0",
+	"version": "0.13.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.13.0",
+	"version": "0.13.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.13.0",
+  "version": "0.13.0-beta.1",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.13.0",
+	"version": "0.13.0-beta.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -10,13 +10,11 @@
     "vector database",
     "ann"
   ],
-  "version": "0.13.0",
+  "version": "0.13.0-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
-    "./embedding": "./dist/embedding/index.js",
-    "./embedding/openai": "./dist/embedding/openai.js",
-    "./embedding/transformers": "./dist/embedding/transformers.js"
+    "./embedding": "./dist/embedding/index.js"
   },
   "types": "dist/index.d.ts",
   "napi": {
@@ -87,7 +85,7 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@huggingface/transformers": "^3.0.2",
+    "@xenova/transformers": ">=2.17 < 3",
     "openai": "^4.29.2"
   },
   "peerDependencies": {

nodejs/src/query.rs

@@ -135,16 +135,6 @@ impl VectorQuery {
         self.inner = self.inner.clone().column(&column);
     }
 
-    #[napi]
-    pub fn add_query_vector(&mut self, vector: Float32Array) -> Result<()> {
-        self.inner = self
-            .inner
-            .clone()
-            .add_query_vector(vector.as_ref())
-            .default_error()?;
-        Ok(())
-    }
-
     #[napi]
     pub fn distance_type(&mut self, distance_type: String) -> napi::Result<()> {
         let distance_type = parse_distance_type(distance_type)?;
@@ -167,11 +157,6 @@ impl VectorQuery {
         self.inner = self.inner.clone().nprobes(nprobe as usize);
     }
 
-    #[napi]
-    pub fn ef(&mut self, ef: u32) {
-        self.inner = self.inner.clone().ef(ef as usize);
-    }
-
     #[napi]
     pub fn bypass_vector_index(&mut self) {
         self.inner = self.inner.clone().bypass_vector_index()

nodejs/tsconfig.json

@@ -12,7 +12,7 @@
     "experimentalDecorators": true,
     "moduleResolution": "Node"
   },
-  "exclude": ["./dist/*", "./examples/*"],
+  "exclude": ["./dist/*"],
   "typedocOptions": {
     "entryPoints": ["lancedb/index.ts"],
     "out": "../docs/src/javascript/",

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.0"
+current_version = "0.16.0-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.16.0"
+version = "0.16.0-beta.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -15,7 +15,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 arrow = { version = "52.1", features = ["pyarrow"] }
-lancedb = { path = "../rust/lancedb", default-features = false }
+lancedb = { path = "../rust/lancedb" }
 env_logger.workspace = true
 pyo3 = { version = "0.21", features = ["extension-module", "abi3-py38", "gil-refs"] }
 # Using this fork for now: https://github.com/awestlake87/pyo3-asyncio/issues/119
@@ -33,11 +33,6 @@ pyo3-build-config = { version = "0.20.3", features = [
 ] }
 
 [features]
-default = ["default-tls", "remote"]
+default = ["remote"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]
-
-# TLS
-default-tls = ["lancedb/default-tls"]
-native-tls = ["lancedb/native-tls"]
-rustls-tls = ["lancedb/rustls-tls"]

python/pyproject.toml

@@ -4,7 +4,7 @@ name = "lancedb"
 dependencies = [
     "deprecation",
     "nest-asyncio~=1.0",
-    "pylance==0.19.3b1",
+    "pylance==0.19.2-beta.3",
     "tqdm>=4.27.0",
     "pydantic>=1.10",
     "packaging",

python/python/lancedb/embeddings/registry.py

@@ -1,6 +1,15 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 import json
 from typing import Dict, Optional
 
@@ -161,7 +170,7 @@ def register(name):
     return __REGISTRY__.get_instance().register(name)
 
 
-def get_registry() -> EmbeddingFunctionRegistry:
+def get_registry():
     """
     Utility function to get the global instance of the registry
 

python/python/lancedb/index.py

@@ -110,7 +110,16 @@ class FTS:
         remove_stop_words: bool = False,
         ascii_folding: bool = False,
     ):
-        self._inner = LanceDbIndex.fts(with_position=with_position)
+        self._inner = LanceDbIndex.fts(
+            with_position=with_position,
+            base_tokenizer=base_tokenizer,
+            language=language,
+            max_token_length=max_token_length,
+            lower_case=lower_case,
+            stem=stem,
+            remove_stop_words=remove_stop_words,
+            ascii_folding=ascii_folding,
+        )
 
 
 class HnswPq:

python/python/lancedb/query.py

@@ -131,8 +131,6 @@ class Query(pydantic.BaseModel):
 
     fast_search: bool = False
 
-    ef: Optional[int] = None
-
 
 class LanceQueryBuilder(ABC):
     """An abstract query builder. Subclasses are defined for vector search,
@@ -259,7 +257,6 @@ class LanceQueryBuilder(ABC):
         self._with_row_id = False
         self._vector = None
         self._text = None
-        self._ef = None
 
     @deprecation.deprecated(
         deprecated_in="0.3.1",
@@ -641,28 +638,6 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         self._nprobes = nprobes
         return self
 
-    def ef(self, ef: int) -> LanceVectorQueryBuilder:
-        """Set the number of candidates to consider during search.
-
-        Higher values will yield better recall (more likely to find vectors if
-        they exist) at the expense of latency.
-
-        This only applies to the HNSW-related index.
-        The default value is 1.5 * limit.
-
-        Parameters
-        ----------
-        ef: int
-            The number of candidates to consider during search.
-
-        Returns
-        -------
-        LanceVectorQueryBuilder
-            The LanceQueryBuilder object.
-        """
-        self._ef = ef
-        return self
-
     def refine_factor(self, refine_factor: int) -> LanceVectorQueryBuilder:
         """Set the refine factor to use, increasing the number of vectors sampled.
 
@@ -725,7 +700,6 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
             with_row_id=self._with_row_id,
             offset=self._offset,
             fast_search=self._fast_search,
-            ef=self._ef,
         )
         result_set = self._table._execute_query(query, batch_size)
         if self._reranker is not None:
@@ -969,16 +943,12 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
 
 class LanceEmptyQueryBuilder(LanceQueryBuilder):
     def to_arrow(self) -> pa.Table:
-        query = Query(
+        ds = self._table.to_lance()
+        return ds.to_table(
             columns=self._columns,
             filter=self._where,
-            k=self._limit or 10,
-            with_row_id=self._with_row_id,
-            vector=[],
-            # not actually respected in remote query
-            offset=self._offset or 0,
+            limit=self._limit,
         )
-        return self._table._execute_query(query).read_all()
 
     def rerank(self, reranker: Reranker) -> LanceEmptyQueryBuilder:
         """Rerank the results using the specified reranker.
@@ -1097,8 +1067,6 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             self._vector_query.nprobes(self._nprobes)
         if self._refine_factor:
             self._vector_query.refine_factor(self._refine_factor)
-        if self._ef:
-            self._vector_query.ef(self._ef)
 
         with ThreadPoolExecutor() as executor:
             fts_future = executor.submit(self._fts_query.with_row_id(True).to_arrow)
@@ -1225,29 +1193,6 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         self._nprobes = nprobes
         return self
 
-    def ef(self, ef: int) -> LanceHybridQueryBuilder:
-        """
-        Set the number of candidates to consider during search.
-
-        Higher values will yield better recall (more likely to find vectors if
-        they exist) at the expense of latency.
-
-        This only applies to the HNSW-related index.
-        The default value is 1.5 * limit.
-
-        Parameters
-        ----------
-        ef: int
-            The number of candidates to consider during search.
-
-        Returns
-        -------
-        LanceHybridQueryBuilder
-            The LanceHybridQueryBuilder object.
-        """
-        self._ef = ef
-        return self
-
     def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceHybridQueryBuilder:
         """Set the distance metric to use.
 
@@ -1546,8 +1491,7 @@ class AsyncQuery(AsyncQueryBase):
         return pa.array(vec)
 
     def nearest_to(
-        self,
-        query_vector: Union[VEC, Tuple, List[VEC]],
+        self, query_vector: Optional[Union[VEC, Tuple]] = None
     ) -> AsyncVectorQuery:
         """
         Find the nearest vectors to the given query vector.
@@ -1585,33 +1529,10 @@ class AsyncQuery(AsyncQueryBase):
 
         Vector searches always have a [limit][].  If `limit` has not been called then
         a default `limit` of 10 will be used.
-
-        Typically, a single vector is passed in as the query. However, you can also
-        pass in multiple vectors.  This can be useful if you want to find the nearest
-        vectors to multiple query vectors. This is not expected to be faster than
-        making multiple queries concurrently; it is just a convenience method.
-        If multiple vectors are passed in then an additional column `query_index`
-        will be added to the results.  This column will contain the index of the
-        query vector that the result is nearest to.
         """
-        if query_vector is None:
-            raise ValueError("query_vector can not be None")
-
-        if (
-            isinstance(query_vector, list)
-            and len(query_vector) > 0
-            and not isinstance(query_vector[0], (float, int))
-        ):
-            # multiple have been passed
-            query_vectors = [AsyncQuery._query_vec_to_array(v) for v in query_vector]
-            new_self = self._inner.nearest_to(query_vectors[0])
-            for v in query_vectors[1:]:
-                new_self.add_query_vector(v)
-            return AsyncVectorQuery(new_self)
-        else:
-            return AsyncVectorQuery(
-                self._inner.nearest_to(AsyncQuery._query_vec_to_array(query_vector))
-            )
+        return AsyncVectorQuery(
+            self._inner.nearest_to(AsyncQuery._query_vec_to_array(query_vector))
+        )
 
     def nearest_to_text(
         self, query: str, columns: Union[str, List[str]] = []
@@ -1673,7 +1594,7 @@ class AsyncVectorQuery(AsyncQueryBase):
         """
         Set the number of partitions to search (probe)
 
-        This argument is only used when the vector column has an IVF-based index.
+        This argument is only used when the vector column has an IVF PQ index.
         If there is no index then this value is ignored.
 
         The IVF stage of IVF PQ divides the input into partitions (clusters) of
@@ -1695,21 +1616,6 @@ class AsyncVectorQuery(AsyncQueryBase):
         self._inner.nprobes(nprobes)
         return self
 
-    def ef(self, ef: int) -> AsyncVectorQuery:
-        """
-        Set the number of candidates to consider during search
-
-        This argument is only used when the vector column has an HNSW index.
-        If there is no index then this value is ignored.
-
-        Increasing this value will increase the recall of your query but will also
-        increase the latency of your query.  The default value is 1.5 * limit.  This
-        default is good for many cases but the best value to use will depend on your
-        data and the recall that you need to achieve.
-        """
-        self._inner.ef(ef)
-        return self
-
     def refine_factor(self, refine_factor: int) -> AsyncVectorQuery:
         """
         A multiplier to control how many additional rows are taken during the refine

python/python/lancedb/remote/table.py

@@ -86,12 +86,6 @@ class RemoteTable(Table):
         """to_pandas() is not yet supported on LanceDB cloud."""
         return NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
 
-    def checkout(self, version):
-        return self._loop.run_until_complete(self._table.checkout(version))
-
-    def checkout_latest(self):
-        return self._loop.run_until_complete(self._table.checkout_latest())
-
     def list_indices(self):
         """List all the indices on the table"""
         return self._loop.run_until_complete(self._table.list_indices())
@@ -138,8 +132,25 @@ class RemoteTable(Table):
         *,
         replace: bool = False,
         with_position: bool = True,
+        # tokenizer configs:
+        base_tokenizer: str = "simple",
+        language: str = "English",
+        max_token_length: Optional[int] = 40,
+        lower_case: bool = True,
+        stem: bool = False,
+        remove_stop_words: bool = False,
+        ascii_folding: bool = False,
     ):
-        config = FTS(with_position=with_position)
+        config = FTS(
+            with_position=with_position,
+            base_tokenizer=base_tokenizer,
+            language=language,
+            max_token_length=max_token_length,
+            lower_case=lower_case,
+            stem=stem,
+            remove_stop_words=remove_stop_words,
+            ascii_folding=ascii_folding,
+        )
         self._loop.run_until_complete(
             self._table.create_index(column, config=config, replace=replace)
         )
@@ -333,6 +344,10 @@ class RemoteTable(Table):
             - and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
+        # empty query builder is not supported in saas, raise error
+        if query is None and query_type != "hybrid":
+            raise ValueError("Empty query is not supported")
+
         return LanceQueryBuilder.create(
             self,
             query,

python/python/lancedb/rerankers/voyageai.py

@@ -13,7 +13,7 @@
 
 import os
 from functools import cached_property
-from typing import Optional
+from typing import Union, Optional
 
 import pyarrow as pa
 

python/python/lancedb/table.py

@@ -73,21 +73,6 @@ pl = safe_import_polars()
 QueryType = Literal["vector", "fts", "hybrid", "auto"]
 
 
-def _pd_schema_without_embedding_funcs(
-    schema: Optional[pa.Schema], columns: List[str]
-) -> Optional[pa.Schema]:
-    """Return a schema without any embedding function columns"""
-    if schema is None:
-        return None
-    embedding_functions = EmbeddingFunctionRegistry.get_instance().parse_functions(
-        schema.metadata
-    )
-    if not embedding_functions:
-        return schema
-    columns = set(columns)
-    return pa.schema([field for field in schema if field.name in columns])
-
-
 def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
     if _check_for_hugging_face(data):
         # Huggingface datasets
@@ -118,10 +103,10 @@ def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
         elif isinstance(data[0], pa.RecordBatch):
             return pa.Table.from_batches(data, schema=schema)
         else:
-            return pa.Table.from_pylist(data, schema=schema)
+            return pa.Table.from_pylist(data)
     elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
-        raw_schema = _pd_schema_without_embedding_funcs(schema, data.columns.to_list())
-        table = pa.Table.from_pandas(data, preserve_index=False, schema=raw_schema)
+        # Do not add schema here, since schema may contains the vector column
+        table = pa.Table.from_pandas(data, preserve_index=False)
         # Do not serialize Pandas metadata
         meta = table.schema.metadata if table.schema.metadata is not None else {}
         meta = {k: v for k, v in meta.items() if k != b"pandas"}
@@ -187,8 +172,6 @@ def sanitize_create_table(
         schema = schema.to_arrow_schema()
 
     if data is not None:
-        if metadata is None and schema is not None:
-            metadata = schema.metadata
         data, schema = _sanitize_data(
             data,
             schema,
@@ -1012,18 +995,6 @@ class Table(ABC):
             The names of the columns to drop.
         """
 
-    @abstractmethod
-    def checkout(self):
-        """
-        TODO comments
-        """
-
-    @abstractmethod
-    def checkout_latest(self):
-        """
-        TODO comments
-        """
-
     @cached_property
     def _dataset_uri(self) -> str:
         return _table_uri(self._conn.uri, self.name)
@@ -1579,7 +1550,7 @@ class LanceTable(Table):
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null".
+            One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
@@ -1863,7 +1834,7 @@ class LanceTable(Table):
             data but will validate against any schema that's specified.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null".
+            One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         embedding_functions: list of EmbeddingFunctionModel, default None
@@ -1971,7 +1942,6 @@ class LanceTable(Table):
                 "metric": query.metric,
                 "nprobes": query.nprobes,
                 "refine_factor": query.refine_factor,
-                "ef": query.ef,
             }
         return ds.scanner(
             columns=query.columns,
@@ -2164,11 +2134,13 @@ def _sanitize_schema(
         vector column to fixed_size_list(float32) if necessary.
     on_bad_vectors: str, default "error"
         What to do if any of the vectors are not the same size or contains NaNs.
-        One of "error", "drop", "fill", "null".
+        One of "error", "drop", "fill".
     fill_value: float, default 0.
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     if schema is not None:
+        if data.schema == schema:
+            return data
         # cast the columns to the expected types
         data = data.combine_chunks()
         for field in schema:
@@ -2188,7 +2160,6 @@ def _sanitize_schema(
                     vector_column_name=field.name,
                     on_bad_vectors=on_bad_vectors,
                     fill_value=fill_value,
-                    table_schema=schema,
                 )
         return pa.Table.from_arrays(
             [data[name] for name in schema.names], schema=schema
@@ -2209,7 +2180,6 @@ def _sanitize_schema(
 def _sanitize_vector_column(
     data: pa.Table,
     vector_column_name: str,
-    table_schema: Optional[pa.Schema] = None,
     on_bad_vectors: str = "error",
     fill_value: float = 0.0,
 ) -> pa.Table:
@@ -2224,16 +2194,12 @@ def _sanitize_vector_column(
         The name of the vector column.
     on_bad_vectors: str, default "error"
         What to do if any of the vectors are not the same size or contains NaNs.
-        One of "error", "drop", "fill", "null".
+        One of "error", "drop", "fill".
     fill_value: float, default 0.0
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     # ChunkedArray is annoying to work with, so we combine chunks here
     vec_arr = data[vector_column_name].combine_chunks()
-    if table_schema is not None:
-        field = table_schema.field(vector_column_name)
-    else:
-        field = None
     typ = data[vector_column_name].type
     if pa.types.is_list(typ) or pa.types.is_large_list(typ):
         # if it's a variable size list array,
@@ -2260,11 +2226,7 @@ def _sanitize_vector_column(
                 data, fill_value, on_bad_vectors, vec_arr, vector_column_name
             )
     else:
-        if (
-            field is not None
-            and not field.nullable
-            and pc.any(pc.is_null(vec_arr.values)).as_py()
-        ) or (pc.any(pc.is_nan(vec_arr.values)).as_py()):
+        if pc.any(pc.is_null(vec_arr.values, nan_is_null=True)).as_py():
             data = _sanitize_nans(
                 data, fill_value, on_bad_vectors, vec_arr, vector_column_name
             )
@@ -2308,12 +2270,6 @@ def _sanitize_jagged(data, fill_value, on_bad_vectors, vec_arr, vector_column_na
         )
     elif on_bad_vectors == "drop":
         data = data.filter(correct_ndims)
-    elif on_bad_vectors == "null":
-        data = data.set_column(
-            data.column_names.index(vector_column_name),
-            vector_column_name,
-            pc.if_else(correct_ndims, vec_arr, pa.scalar(None)),
-        )
     return data
 
 
@@ -2330,8 +2286,7 @@ def _sanitize_nans(
         raise ValueError(
             f"Vector column {vector_column_name} has NaNs. "
             "Set on_bad_vectors='drop' to remove them, or "
-            "set on_bad_vectors='fill' and fill_value=<value> to replace them. "
-            "Or set on_bad_vectors='null' to replace them with null."
+            "set on_bad_vectors='fill' and fill_value=<value> to replace them."
         )
     elif on_bad_vectors == "fill":
         if fill_value is None:
@@ -2351,17 +2306,6 @@ def _sanitize_nans(
         np_arr = np_arr.reshape(-1, vec_arr.type.list_size)
         not_nulls = np.any(np_arr, axis=1)
         data = data.filter(~not_nulls)
-    elif on_bad_vectors == "null":
-        # null = pa.nulls(len(vec_arr)).cast(vec_arr.type)
-        # values = pc.if_else(pc.is_nan(vec_arr.values), fill_value, vec_arr.values)
-        np_arr = np.isnan(vec_arr.values.to_numpy(zero_copy_only=False))
-        np_arr = np_arr.reshape(-1, vec_arr.type.list_size)
-        no_nans = np.any(np_arr, axis=1)
-        data = data.set_column(
-            data.column_names.index(vector_column_name),
-            vector_column_name,
-            pc.if_else(no_nans, vec_arr, pa.scalar(None)),
-        )
     return data
 
 
@@ -2627,7 +2571,7 @@ class AsyncTable:
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null".
+            One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
@@ -2710,7 +2654,7 @@ class AsyncTable:
 
     def vector_search(
         self,
-        query_vector: Union[VEC, Tuple],
+        query_vector: Optional[Union[VEC, Tuple]] = None,
     ) -> AsyncVectorQuery:
         """
         Search the table with a given query vector.
@@ -2749,8 +2693,6 @@ class AsyncTable:
                 async_query = async_query.refine_factor(query.refine_factor)
             if query.vector_column:
                 async_query = async_query.column(query.vector_column)
-            if query.ef:
-                async_query = async_query.ef(query.ef)
 
         if not query.prefilter:
             async_query = async_query.postfilter()

python/python/tests/test_embeddings.py

@@ -1,6 +1,15 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 from typing import List, Union
 from unittest.mock import MagicMock, patch
 
@@ -9,7 +18,6 @@ import lancedb
 import numpy as np
 import pyarrow as pa
 import pytest
-import pandas as pd
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings import (
     EmbeddingFunctionConfig,
@@ -81,15 +89,14 @@ def test_embedding_function(tmp_path):
 
 
 def test_embedding_with_bad_results(tmp_path):
-    @register("null-embedding")
-    class NullEmbeddingFunction(TextEmbeddingFunction):
+    @register("mock-embedding")
+    class MockEmbeddingFunction(TextEmbeddingFunction):
         def ndims(self):
             return 128
 
         def generate_embeddings(
             self, texts: Union[List[str], np.ndarray]
         ) -> list[Union[np.array, None]]:
-            # Return None, which is bad if field is non-nullable
             return [
                 None if i % 2 == 0 else np.random.randn(self.ndims())
                 for i in range(len(texts))
@@ -97,17 +104,13 @@ def test_embedding_with_bad_results(tmp_path):
 
     db = lancedb.connect(tmp_path)
     registry = EmbeddingFunctionRegistry.get_instance()
-    model = registry.get("null-embedding").create()
+    model = registry.get("mock-embedding").create()
 
     class Schema(LanceModel):
         text: str = model.SourceField()
         vector: Vector(model.ndims()) = model.VectorField()
 
     table = db.create_table("test", schema=Schema, mode="overwrite")
-    with pytest.raises(ValueError):
-        # Default on_bad_vectors is "error"
-        table.add([{"text": "hello world"}])
-
     table.add(
         [{"text": "hello world"}, {"text": "bar"}],
         on_bad_vectors="drop",
@@ -117,169 +120,13 @@ def test_embedding_with_bad_results(tmp_path):
     assert len(table) == 1
     assert df.iloc[0]["text"] == "bar"
 
-    @register("nan-embedding")
-    class NanEmbeddingFunction(TextEmbeddingFunction):
-        def ndims(self):
-            return 128
-
-        def generate_embeddings(
-            self, texts: Union[List[str], np.ndarray]
-        ) -> list[Union[np.array, None]]:
-            # Return NaN to produce bad vectors
-            return [
-                [np.NAN] * 128 if i % 2 == 0 else np.random.randn(self.ndims())
-                for i in range(len(texts))
-            ]
-
-    db = lancedb.connect(tmp_path)
-    registry = EmbeddingFunctionRegistry.get_instance()
-    model = registry.get("nan-embedding").create()
-
-    table = db.create_table("test2", schema=Schema, mode="overwrite")
-    table.alter_columns(dict(path="vector", nullable=True))
-    table.add(
-        [{"text": "hello world"}, {"text": "bar"}],
-        on_bad_vectors="null",
-    )
-    assert len(table) == 2
-    tbl = table.to_arrow()
-    assert tbl["vector"].null_count == 1
-
-
-def test_with_existing_vectors(tmp_path):
-    @register("mock-embedding")
-    class MockEmbeddingFunction(TextEmbeddingFunction):
-        def ndims(self):
-            return 128
-
-        def generate_embeddings(
-            self, texts: Union[List[str], np.ndarray]
-        ) -> List[np.array]:
-            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
-
-    registry = get_registry()
-    model = registry.get("mock-embedding").create()
-
-    class Schema(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    db = lancedb.connect(tmp_path)
-    tbl = db.create_table("test", schema=Schema, mode="overwrite")
-    tbl.add([{"text": "hello world", "vector": np.zeros(128).tolist()}])
-
-    embeddings = tbl.to_arrow()["vector"].to_pylist()
-    assert not np.any(embeddings), "all zeros"
-
-
-def test_embedding_function_with_pandas(tmp_path):
-    @register("mock-embedding")
-    class _MockEmbeddingFunction(TextEmbeddingFunction):
-        def ndims(self):
-            return 128
-
-        def generate_embeddings(
-            self, texts: Union[List[str], np.ndarray]
-        ) -> List[np.array]:
-            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
-
-    registery = get_registry()
-    func = registery.get("mock-embedding").create()
-
-    class TestSchema(LanceModel):
-        text: str = func.SourceField()
-        val: int
-        vector: Vector(func.ndims()) = func.VectorField()
-
-    df = pd.DataFrame(
-        {
-            "text": ["hello world", "goodbye world"],
-            "val": [1, 2],
-            "not-used": ["s1", "s3"],
-        }
-    )
-    db = lancedb.connect(tmp_path)
-    tbl = db.create_table("test", schema=TestSchema, mode="overwrite", data=df)
-    schema = tbl.schema
-    assert schema.field("text").type == pa.string()
-    assert schema.field("val").type == pa.int64()
-    assert schema.field("vector").type == pa.list_(pa.float32(), 128)
-
-    df = pd.DataFrame(
-        {
-            "text": ["extra", "more"],
-            "val": [4, 5],
-            "misc-col": ["s1", "s3"],
-        }
-    )
-    tbl.add(df)
-
-    assert tbl.count_rows() == 4
-    embeddings = tbl.to_arrow()["vector"]
-    assert embeddings.null_count == 0
-
-    df = pd.DataFrame(
-        {
-            "text": ["with", "embeddings"],
-            "val": [6, 7],
-            "vector": [np.zeros(128).tolist(), np.zeros(128).tolist()],
-        }
-    )
-    tbl.add(df)
-
-    embeddings = tbl.search().where("val > 5").to_arrow()["vector"].to_pylist()
-    assert not np.any(embeddings), "all zeros"
-
-
-def test_multiple_embeddings_for_pandas(tmp_path):
-    @register("mock-embedding")
-    class MockFunc1(TextEmbeddingFunction):
-        def ndims(self):
-            return 128
-
-        def generate_embeddings(
-            self, texts: Union[List[str], np.ndarray]
-        ) -> List[np.array]:
-            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
-
-    @register("mock-embedding2")
-    class MockFunc2(TextEmbeddingFunction):
-        def ndims(self):
-            return 512
-
-        def generate_embeddings(
-            self, texts: Union[List[str], np.ndarray]
-        ) -> List[np.array]:
-            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
-
-    registery = get_registry()
-    func1 = registery.get("mock-embedding").create()
-    func2 = registery.get("mock-embedding2").create()
-
-    class TestSchema(LanceModel):
-        text: str = func1.SourceField()
-        val: int
-        vec1: Vector(func1.ndims()) = func1.VectorField()
-        prompt: str = func2.SourceField()
-        vec2: Vector(func2.ndims()) = func2.VectorField()
-
-    df = pd.DataFrame(
-        {
-            "text": ["hello world", "goodbye world"],
-            "val": [1, 2],
-            "prompt": ["hello", "goodbye"],
-        }
-    )
-    db = lancedb.connect(tmp_path)
-    tbl = db.create_table("test", schema=TestSchema, mode="overwrite", data=df)
-
-    schema = tbl.schema
-    assert schema.field("text").type == pa.string()
-    assert schema.field("val").type == pa.int64()
-    assert schema.field("vec1").type == pa.list_(pa.float32(), 128)
-    assert schema.field("prompt").type == pa.string()
-    assert schema.field("vec2").type == pa.list_(pa.float32(), 512)
-    assert tbl.count_rows() == 2
+    # table = db.create_table("test2", schema=Schema, mode="overwrite")
+    # table.add(
+    #     [{"text": "hello world"}, {"text": "bar"}],
+    # )
+    # assert len(table) == 2
+    # tbl = table.to_arrow()
+    # assert tbl["vector"].null_count == 1
 
 
 @pytest.mark.slow

python/python/tests/test_query.py

@@ -1,9 +1,21 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 
 import unittest.mock as mock
 from datetime import timedelta
+from typing import Optional
 
+import lance
 import lancedb
 from lancedb.index import IvfPq
 import numpy as np
@@ -11,15 +23,41 @@ import pandas.testing as tm
 import pyarrow as pa
 import pytest
 import pytest_asyncio
+from lancedb.db import LanceDBConnection
 from lancedb.pydantic import LanceModel, Vector
 from lancedb.query import AsyncQueryBase, LanceVectorQueryBuilder, Query
 from lancedb.table import AsyncTable, LanceTable
 
 
-@pytest.fixture(scope="module")
-def table(tmpdir_factory) -> lancedb.table.Table:
-    tmp_path = str(tmpdir_factory.mktemp("data"))
-    db = lancedb.connect(tmp_path)
+class MockTable:
+    def __init__(self, tmp_path):
+        self.uri = tmp_path
+        self._conn = LanceDBConnection(self.uri)
+
+    def to_lance(self):
+        return lance.dataset(self.uri)
+
+    def _execute_query(self, query, batch_size: Optional[int] = None):
+        ds = self.to_lance()
+        return ds.scanner(
+            columns=query.columns,
+            filter=query.filter,
+            prefilter=query.prefilter,
+            nearest={
+                "column": query.vector_column,
+                "q": query.vector,
+                "k": query.k,
+                "metric": query.metric,
+                "nprobes": query.nprobes,
+                "refine_factor": query.refine_factor,
+            },
+            batch_size=batch_size,
+            offset=query.offset,
+        ).to_reader()
+
+
+@pytest.fixture
+def table(tmp_path) -> MockTable:
     df = pa.table(
         {
             "vector": pa.array(
@@ -30,7 +68,8 @@ def table(tmpdir_factory) -> lancedb.table.Table:
             "float_field": pa.array([1.0, 2.0]),
         }
     )
-    return db.create_table("test", df)
+    lance.write_dataset(df, tmp_path)
+    return MockTable(tmp_path)
 
 
 @pytest_asyncio.fixture
@@ -87,12 +126,6 @@ def test_query_builder(table):
     assert all(np.array(rs[0]["vector"]) == [1, 2])
 
 
-def test_with_row_id(table: lancedb.table.Table):
-    rs = table.search().with_row_id(True).to_arrow()
-    assert "_rowid" in rs.column_names
-    assert rs["_rowid"].to_pylist() == [0, 1]
-
-
 def test_vector_query_with_no_limit(table):
     with pytest.raises(ValueError):
         LanceVectorQueryBuilder(table, [0, 0], "vector").limit(0).select(
@@ -332,12 +365,6 @@ async def test_query_to_pandas_async(table_async: AsyncTable):
     assert df.shape == (0, 4)
 
 
-@pytest.mark.asyncio
-async def test_none_query(table_async: AsyncTable):
-    with pytest.raises(ValueError):
-        await table_async.query().nearest_to(None).to_arrow()
-
-
 @pytest.mark.asyncio
 async def test_fast_search_async(tmp_path):
     db = await lancedb.connect_async(tmp_path)

python/python/tests/test_remote_db.py

@@ -185,7 +185,6 @@ def test_query_sync_minimal():
             "k": 10,
             "prefilter": False,
             "refine_factor": None,
-            "ef": None,
             "vector": [1.0, 2.0, 3.0],
             "nprobes": 20,
         }
@@ -198,23 +197,6 @@ def test_query_sync_minimal():
         assert data == expected
 
 
-def test_query_sync_empty_query():
-    def handler(body):
-        assert body == {
-            "k": 10,
-            "filter": "true",
-            "vector": [],
-            "columns": ["id"],
-        }
-
-        return pa.table({"id": [1, 2, 3]})
-
-    with query_test_table(handler) as table:
-        data = table.search(None).where("true").select(["id"]).limit(10).to_list()
-        expected = [{"id": 1}, {"id": 2}, {"id": 3}]
-        assert data == expected
-
-
 def test_query_sync_maximal():
     def handler(body):
         assert body == {
@@ -224,7 +206,6 @@ def test_query_sync_maximal():
             "refine_factor": 10,
             "vector": [1.0, 2.0, 3.0],
             "nprobes": 5,
-            "ef": None,
             "filter": "id > 0",
             "columns": ["id", "name"],
             "vector_column": "vector2",
@@ -248,17 +229,6 @@ def test_query_sync_maximal():
         )
 
 
-def test_query_sync_multiple_vectors():
-    def handler(_body):
-        return pa.table({"id": [1]})
-
-    with query_test_table(handler) as table:
-        results = table.search([[1, 2, 3], [4, 5, 6]]).limit(1).to_list()
-        assert len(results) == 2
-        results.sort(key=lambda x: x["query_index"])
-        assert results == [{"id": 1, "query_index": 0}, {"id": 1, "query_index": 1}]
-
-
 def test_query_sync_fts():
     def handler(body):
         assert body == {
@@ -320,7 +290,6 @@ def test_query_sync_hybrid():
                 "refine_factor": None,
                 "vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
                 "nprobes": 20,
-                "ef": None,
                 "with_row_id": True,
             }
             return pa.table({"_rowid": [1, 2, 3], "_distance": [0.1, 0.2, 0.3]})

python/python/tests/test_table.py

@@ -240,121 +240,6 @@ def test_add(db):
     _add(table, schema)
 
 
-def test_add_subschema(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
-            pa.field("item", pa.string(), nullable=True),
-            pa.field("price", pa.float64(), nullable=False),
-        ]
-    )
-    table = db.create_table("test", schema=schema)
-
-    data = {"price": 10.0, "item": "foo"}
-    table.add([data])
-    data = {"price": 2.0, "vector": [3.1, 4.1]}
-    table.add([data])
-    data = {"price": 3.0, "vector": [5.9, 26.5], "item": "bar"}
-    table.add([data])
-
-    expected = pa.table(
-        {
-            "vector": [None, [3.1, 4.1], [5.9, 26.5]],
-            "item": ["foo", None, "bar"],
-            "price": [10.0, 2.0, 3.0],
-        },
-        schema=schema,
-    )
-    assert table.to_arrow() == expected
-
-    data = {"item": "foo"}
-    # We can't omit a column if it's not nullable
-    with pytest.raises(OSError, match="Invalid user input"):
-        table.add([data])
-
-    # We can add it if we make the column nullable
-    table.alter_columns(dict(path="price", nullable=True))
-    table.add([data])
-
-    expected_schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
-            pa.field("item", pa.string(), nullable=True),
-            pa.field("price", pa.float64(), nullable=True),
-        ]
-    )
-    expected = pa.table(
-        {
-            "vector": [None, [3.1, 4.1], [5.9, 26.5], None],
-            "item": ["foo", None, "bar", "foo"],
-            "price": [10.0, 2.0, 3.0, None],
-        },
-        schema=expected_schema,
-    )
-    assert table.to_arrow() == expected
-
-
-def test_add_nullability(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float32(), 2), nullable=False),
-            pa.field("id", pa.string(), nullable=False),
-        ]
-    )
-    table = db.create_table("test", schema=schema)
-
-    nullable_schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
-            pa.field("id", pa.string(), nullable=True),
-        ]
-    )
-    data = pa.table(
-        {
-            "vector": [[3.1, 4.1], [5.9, 26.5]],
-            "id": ["foo", "bar"],
-        },
-        schema=nullable_schema,
-    )
-    # We can add nullable schema if it doesn't actually contain nulls
-    table.add(data)
-
-    expected = data.cast(schema)
-    assert table.to_arrow() == expected
-
-    data = pa.table(
-        {
-            "vector": [None],
-            "id": ["baz"],
-        },
-        schema=nullable_schema,
-    )
-    # We can't add nullable schema if it contains nulls
-    with pytest.raises(Exception, match="Vector column vector has NaNs"):
-        table.add(data)
-
-    # But we can make it nullable
-    table.alter_columns(dict(path="vector", nullable=True))
-    table.add(data)
-
-    expected_schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
-            pa.field("id", pa.string(), nullable=False),
-        ]
-    )
-    expected = pa.table(
-        {
-            "vector": [[3.1, 4.1], [5.9, 26.5], None],
-            "id": ["foo", "bar", "baz"],
-        },
-        schema=expected_schema,
-    )
-    assert table.to_arrow() == expected
-
-
 def test_add_pydantic_model(db):
     # https://github.com/lancedb/lancedb/issues/562
 
@@ -1007,15 +892,10 @@ def test_empty_query(db):
     table = LanceTable.create(db, "my_table2", data=[{"id": i} for i in range(100)])
     df = table.search().select(["id"]).to_pandas()
     assert len(df) == 10
-    # None is the same as default
     df = table.search().select(["id"]).limit(None).to_pandas()
-    assert len(df) == 10
-    # invalid limist is the same as None, wihch is the same as default
+    assert len(df) == 100
     df = table.search().select(["id"]).limit(-1).to_pandas()
-    assert len(df) == 10
-    # valid limit should work
-    df = table.search().select(["id"]).limit(42).to_pandas()
-    assert len(df) == 42
+    assert len(df) == 100
 
 
 def test_search_with_schema_inf_single_vector(db):

python/src/query.rs

@@ -142,13 +142,6 @@ impl VectorQuery {
         self.inner = self.inner.clone().only_if(predicate);
     }
 
-    pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> {
-        let data: ArrayData = ArrayData::from_pyarrow_bound(&vector)?;
-        let array = make_array(data);
-        self.inner = self.inner.clone().add_query_vector(array).infer_error()?;
-        Ok(())
-    }
-
     pub fn select(&mut self, columns: Vec<(String, String)>) {
         self.inner = self.inner.clone().select(Select::dynamic(&columns));
     }
@@ -195,10 +188,6 @@ impl VectorQuery {
         self.inner = self.inner.clone().nprobes(nprobe as usize);
     }
 
-    pub fn ef(&mut self, ef: u32) {
-        self.inner = self.inner.clone().ef(ef as usize);
-    }
-
     pub fn bypass_vector_index(&mut self) {
         self.inner = self.inner.clone().bypass_vector_index()
     }

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.13.0"
+version = "0.13.0-beta.1"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.13.0"
+version = "0.13.0-beta.1"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -46,18 +46,10 @@ serde = { version = "^1" }
 serde_json = { version = "1" }
 async-openai = { version = "0.20.0", optional = true }
 serde_with = { version = "3.8.1" }
-aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
 # For remote feature
-reqwest = { version = "0.12.0", default-features = false, features = [
-    "charset",
-    "gzip",
-    "http2",
-    "json",
-    "macos-system-configuration",
-    "stream",
-], optional = true }
-rand = { version = "0.8.3", features = ["small_rng"], optional = true }
-http = { version = "1", optional = true } # Matching what is in reqwest
+reqwest = { version = "0.12.0", features = ["gzip", "json", "stream"], optional = true }
+rand = { version = "0.8.3", features = ["small_rng"], optional = true}
+http = { version = "1",  optional = true } # Matching what is in reqwest
 uuid = { version = "1.7.0", features = ["v4"], optional = true }
 polars-arrow = { version = ">=0.37,<0.40.0", optional = true }
 polars = { version = ">=0.37,<0.40.0", optional = true }
@@ -80,13 +72,11 @@ aws-config = { version = "1.0" }
 aws-smithy-runtime = { version = "1.3" }
 http-body = "1" # Matching reqwest
 
-
 [features]
-default = ["default-tls"]
+default = []
 remote = ["dep:reqwest", "dep:http", "dep:rand", "dep:uuid"]
 fp16kernels = ["lance-linalg/fp16kernels"]
 s3-test = []
-bedrock = ["dep:aws-sdk-bedrockruntime"]
 openai = ["dep:async-openai", "dep:reqwest"]
 polars = ["dep:polars-arrow", "dep:polars"]
 sentence-transformers = [
@@ -97,11 +87,6 @@ sentence-transformers = [
     "dep:tokenizers"
 ]
 
-# TLS
-default-tls = ["reqwest?/default-tls"]
-native-tls = ["reqwest?/native-tls"]
-rustls-tls = ["reqwest?/rustls-tls"]
-
 [[example]]
 name = "openai"
 required-features = ["openai"]
@@ -109,7 +94,3 @@ required-features = ["openai"]
 [[example]]
 name = "sentence_transformers"
 required-features = ["sentence-transformers"]
-
-[[example]]
-name = "bedrock"
-required-features = ["bedrock"]

rust/lancedb/examples/bedrock.rs

@@ -1,89 +0,0 @@
-use std::{iter::once, sync::Arc};
-
-use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
-use arrow_schema::{DataType, Field, Schema};
-use aws_config::Region;
-use aws_sdk_bedrockruntime::Client;
-use futures::StreamExt;
-use lancedb::{
-    arrow::IntoArrow,
-    connect,
-    embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
-    query::{ExecutableQuery, QueryBase},
-    Result,
-};
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    let tempdir = tempfile::tempdir().unwrap();
-    let tempdir = tempdir.path().to_str().unwrap();
-
-    // create Bedrock embedding function
-    let region: String = "us-east-1".to_string();
-    let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
-        .region(Region::new(region))
-        .load()
-        .await;
-
-    let embedding = Arc::new(BedrockEmbeddingFunction::new(
-        Client::new(&config), // AWS Region
-    ));
-
-    let db = connect(tempdir).execute().await?;
-    db.embedding_registry()
-        .register("bedrock", embedding.clone())?;
-
-    let table = db
-        .create_table("vectors", make_data())
-        .add_embedding(EmbeddingDefinition::new(
-            "text",
-            "bedrock",
-            Some("embeddings"),
-        ))?
-        .execute()
-        .await?;
-
-    // execute vector search
-    let query = Arc::new(StringArray::from_iter_values(once("something warm")));
-    let query_vector = embedding.compute_query_embeddings(query)?;
-    let mut results = table
-        .vector_search(query_vector)?
-        .limit(1)
-        .execute()
-        .await?;
-
-    let rb = results.next().await.unwrap()?;
-    let out = rb
-        .column_by_name("text")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<StringArray>()
-        .unwrap();
-    let text = out.iter().next().unwrap().unwrap();
-    println!("Closest match: {}", text);
-    Ok(())
-}
-
-fn make_data() -> impl IntoArrow {
-    let schema = Schema::new(vec![
-        Field::new("id", DataType::Int32, true),
-        Field::new("text", DataType::Utf8, false),
-        Field::new("price", DataType::Float64, false),
-    ]);
-
-    let id = Int32Array::from(vec![1, 2, 3, 4]);
-    let text = StringArray::from_iter_values(vec![
-        "Black T-Shirt",
-        "Leather Jacket",
-        "Winter Parka",
-        "Hooded Sweatshirt",
-    ]);
-    let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
-    let schema = Arc::new(schema);
-    let rb = RecordBatch::try_new(
-        schema.clone(),
-        vec![Arc::new(id), Arc::new(text), Arc::new(price)],
-    )
-    .unwrap();
-    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
-}

rust/lancedb/src/embeddings.rs

@@ -17,9 +17,6 @@ pub mod openai;
 #[cfg(feature = "sentence-transformers")]
 pub mod sentence_transformers;
 
-#[cfg(feature = "bedrock")]
-pub mod bedrock;
-
 use lance::arrow::RecordBatchExt;
 use std::{
     borrow::Cow,

rust/lancedb/src/embeddings/bedrock.rs

@@ -1,210 +0,0 @@
-use aws_sdk_bedrockruntime::Client as BedrockClient;
-use std::{borrow::Cow, fmt::Formatter, str::FromStr, sync::Arc};
-
-use arrow::array::{AsArray, Float32Builder};
-use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array};
-use arrow_data::ArrayData;
-use arrow_schema::DataType;
-use serde_json::{json, Value};
-
-use super::EmbeddingFunction;
-use crate::{Error, Result};
-
-use tokio::runtime::Handle;
-use tokio::task::block_in_place;
-
-#[derive(Debug)]
-pub enum BedrockEmbeddingModel {
-    TitanEmbedding,
-    CohereLarge,
-}
-
-impl BedrockEmbeddingModel {
-    fn ndims(&self) -> usize {
-        match self {
-            Self::TitanEmbedding => 1536,
-            Self::CohereLarge => 1024,
-        }
-    }
-
-    fn model_id(&self) -> &str {
-        match self {
-            Self::TitanEmbedding => "amazon.titan-embed-text-v1",
-            Self::CohereLarge => "cohere.embed-english-v3",
-        }
-    }
-}
-
-impl FromStr for BedrockEmbeddingModel {
-    type Err = Error;
-
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        match s {
-            "titan-embed-text-v1" => Ok(Self::TitanEmbedding),
-            "cohere-embed-english-v3" => Ok(Self::CohereLarge),
-            _ => Err(Error::InvalidInput {
-                message: "Invalid model. Available models are: 'titan-embed-text-v1', 'cohere-embed-english-v3'".to_string()
-            }),
-        }
-    }
-}
-
-pub struct BedrockEmbeddingFunction {
-    model: BedrockEmbeddingModel,
-    client: BedrockClient,
-}
-
-impl BedrockEmbeddingFunction {
-    pub fn new(client: BedrockClient) -> Self {
-        Self {
-            model: BedrockEmbeddingModel::TitanEmbedding,
-            client,
-        }
-    }
-
-    pub fn with_model(client: BedrockClient, model: BedrockEmbeddingModel) -> Self {
-        Self { model, client }
-    }
-}
-
-impl EmbeddingFunction for BedrockEmbeddingFunction {
-    fn name(&self) -> &str {
-        "bedrock"
-    }
-
-    fn source_type(&self) -> Result<Cow<DataType>> {
-        Ok(Cow::Owned(DataType::Utf8))
-    }
-
-    fn dest_type(&self) -> Result<Cow<DataType>> {
-        let n_dims = self.model.ndims();
-        Ok(Cow::Owned(DataType::new_fixed_size_list(
-            DataType::Float32,
-            n_dims as i32,
-            false,
-        )))
-    }
-
-    fn compute_source_embeddings(&self, source: ArrayRef) -> Result<ArrayRef> {
-        let len = source.len();
-        let n_dims = self.model.ndims();
-        let inner = self.compute_inner(source)?;
-
-        let fsl = DataType::new_fixed_size_list(DataType::Float32, n_dims as i32, false);
-
-        let array_data = ArrayData::builder(fsl)
-            .len(len)
-            .add_child_data(inner.into_data())
-            .build()?;
-
-        Ok(Arc::new(FixedSizeListArray::from(array_data)))
-    }
-
-    fn compute_query_embeddings(&self, input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
-        let arr = self.compute_inner(input)?;
-        Ok(Arc::new(arr))
-    }
-}
-
-impl std::fmt::Debug for BedrockEmbeddingFunction {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("BedrockEmbeddingFunction")
-            .field("model", &self.model)
-            // Skip client field as it doesn't implement Debug
-            .finish()
-    }
-}
-
-impl BedrockEmbeddingFunction {
-    fn compute_inner(&self, source: Arc<dyn Array>) -> Result<Float32Array> {
-        if source.is_nullable() {
-            return Err(Error::InvalidInput {
-                message: "Expected non-nullable data type".to_string(),
-            });
-        }
-
-        if !matches!(source.data_type(), DataType::Utf8 | DataType::LargeUtf8) {
-            return Err(Error::InvalidInput {
-                message: "Expected Utf8 data type".to_string(),
-            });
-        }
-
-        let mut builder = Float32Builder::new();
-
-        let texts = match source.data_type() {
-            DataType::Utf8 => source
-                .as_string::<i32>()
-                .into_iter()
-                .map(|s| s.expect("array is non-nullable").to_string())
-                .collect::<Vec<String>>(),
-            DataType::LargeUtf8 => source
-                .as_string::<i64>()
-                .into_iter()
-                .map(|s| s.expect("array is non-nullable").to_string())
-                .collect::<Vec<String>>(),
-            _ => unreachable!(),
-        };
-
-        for text in texts {
-            let request_body = match self.model {
-                BedrockEmbeddingModel::TitanEmbedding => {
-                    json!({
-                        "inputText": text
-                    })
-                }
-                BedrockEmbeddingModel::CohereLarge => {
-                    json!({
-                        "texts": [text],
-                        "input_type": "search_document"
-                    })
-                }
-            };
-
-            let client = self.client.clone();
-            let model_id = self.model.model_id().to_string();
-            let request_body = request_body.clone();
-
-            let response = block_in_place(move || {
-                Handle::current().block_on(async move {
-                    client
-                        .invoke_model()
-                        .model_id(model_id)
-                        .body(aws_sdk_bedrockruntime::primitives::Blob::new(
-                            serde_json::to_vec(&request_body).unwrap(),
-                        ))
-                        .send()
-                        .await
-                })
-            })
-            .unwrap();
-
-            let response_json: Value =
-                serde_json::from_slice(response.body.as_ref()).map_err(|e| Error::Runtime {
-                    message: format!("Failed to parse response: {}", e),
-                })?;
-
-            let embedding = match self.model {
-                BedrockEmbeddingModel::TitanEmbedding => response_json["embedding"]
-                    .as_array()
-                    .ok_or_else(|| Error::Runtime {
-                        message: "Missing embedding in response".to_string(),
-                    })?
-                    .iter()
-                    .map(|v| v.as_f64().unwrap() as f32)
-                    .collect::<Vec<f32>>(),
-                BedrockEmbeddingModel::CohereLarge => response_json["embeddings"][0]
-                    .as_array()
-                    .ok_or_else(|| Error::Runtime {
-                        message: "Missing embeddings in response".to_string(),
-                    })?
-                    .iter()
-                    .map(|v| v.as_f64().unwrap() as f32)
-                    .collect::<Vec<f32>>(),
-            };
-
-            builder.append_slice(&embedding);
-        }
-
-        Ok(builder.finish())
-    }
-}

rust/lancedb/src/query.rs

@@ -475,7 +475,6 @@ impl<T: HasQuery> QueryBase for T {
 
 /// Options for controlling the execution of a query
 #[non_exhaustive]
-#[derive(Debug, Clone)]
 pub struct QueryExecutionOptions {
     /// The maximum number of rows that will be contained in a single
     /// `RecordBatch` delivered by the query.
@@ -651,7 +650,7 @@ impl Query {
     pub fn nearest_to(self, vector: impl IntoQueryVector) -> Result<VectorQuery> {
         let mut vector_query = self.into_vector();
         let query_vector = vector.to_query_vector(&DataType::Float32, "default")?;
-        vector_query.query_vector.push(query_vector);
+        vector_query.query_vector = Some(query_vector);
         Ok(vector_query)
     }
 }
@@ -702,11 +701,8 @@ pub struct VectorQuery {
     // the column based on the dataset's schema.
     pub(crate) column: Option<String>,
     // IVF PQ - ANN search.
-    pub(crate) query_vector: Vec<Arc<dyn Array>>,
+    pub(crate) query_vector: Option<Arc<dyn Array>>,
     pub(crate) nprobes: usize,
-    // The number of candidates to return during the refine step for HNSW,
-    // defaults to 1.5 * limit.
-    pub(crate) ef: Option<usize>,
     pub(crate) refine_factor: Option<u32>,
     pub(crate) distance_type: Option<DistanceType>,
     /// Default is true. Set to false to enforce a brute force search.
@@ -718,9 +714,8 @@ impl VectorQuery {
         Self {
             base,
             column: None,
-            query_vector: Vec::new(),
+            query_vector: None,
             nprobes: 20,
-            ef: None,
             refine_factor: None,
             distance_type: None,
             use_index: true,
@@ -739,22 +734,6 @@ impl VectorQuery {
         self
     }
 
-    /// Add another query vector to the search.
-    ///
-    /// Multiple searches will be dispatched as part of the query.
-    /// This is a convenience method for adding multiple query vectors
-    /// to the search. It is not expected to be faster than issuing
-    /// multiple queries concurrently.
-    ///
-    /// The output data will contain an additional columns `query_index` which
-    /// will contain the index of the query vector that was used to generate the
-    /// result.
-    pub fn add_query_vector(mut self, vector: impl IntoQueryVector) -> Result<Self> {
-        let query_vector = vector.to_query_vector(&DataType::Float32, "default")?;
-        self.query_vector.push(query_vector);
-        Ok(self)
-    }
-
     /// Set the number of partitions to search (probe)
     ///
     /// This argument is only used when the vector column has an IVF PQ index.
@@ -780,18 +759,6 @@ impl VectorQuery {
         self
     }
 
-    /// Set the number of candidates to return during the refine step for HNSW
-    ///
-    /// This argument is only used when the vector column has an HNSW index.
-    /// If there is no index then this value is ignored.
-    ///
-    /// Increasing this value will increase the recall of your query but will
-    /// also increase the latency of your query.  The default value is 1.5*limit.
-    pub fn ef(mut self, ef: usize) -> Self {
-        self.ef = Some(ef);
-        self
-    }
-
     /// A multiplier to control how many additional rows are taken during the refine step
     ///
     /// This argument is only used when the vector column has an IVF PQ index.
@@ -887,7 +854,6 @@ mod tests {
     use std::sync::Arc;
 
     use super::*;
-    use arrow::{compute::concat_batches, datatypes::Int32Type};
     use arrow_array::{
         cast::AsArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
         RecordBatchReader,
@@ -917,10 +883,7 @@ mod tests {
 
         let vector = Float32Array::from_iter_values([0.1, 0.2]);
         let query = table.query().nearest_to(&[0.1, 0.2]).unwrap();
-        assert_eq!(
-            *query.query_vector.first().unwrap().as_ref().as_primitive(),
-            vector
-        );
+        assert_eq!(*query.query_vector.unwrap().as_ref().as_primitive(), vector);
 
         let new_vector = Float32Array::from_iter_values([9.8, 8.7]);
 
@@ -936,7 +899,7 @@ mod tests {
             .refine_factor(999);
 
         assert_eq!(
-            *query.query_vector.first().unwrap().as_ref().as_primitive(),
+            *query.query_vector.unwrap().as_ref().as_primitive(),
             new_vector
         );
         assert_eq!(query.base.limit.unwrap(), 100);
@@ -1234,34 +1197,4 @@ mod tests {
             assert!(batch.column_by_name("_rowid").is_some());
         }
     }
-
-    #[tokio::test]
-    async fn test_multiple_query_vectors() {
-        let tmp_dir = tempdir().unwrap();
-        let table = make_test_table(&tmp_dir).await;
-        let query = table
-            .query()
-            .nearest_to(&[0.1, 0.2, 0.3, 0.4])
-            .unwrap()
-            .add_query_vector(&[0.5, 0.6, 0.7, 0.8])
-            .unwrap()
-            .limit(1);
-
-        let plan = query.explain_plan(true).await.unwrap();
-        assert!(plan.contains("UnionExec"));
-
-        let results = query
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        let results = concat_batches(&results[0].schema(), &results).unwrap();
-        assert_eq!(results.num_rows(), 2); // One result for each query vector.
-        let query_index = results["query_index"].as_primitive::<Int32Type>();
-        // We don't guarantee order.
-        assert!(query_index.values().contains(&0));
-        assert!(query_index.values().contains(&1));
-    }
 }

rust/lancedb/src/remote/table.rs

@@ -6,7 +6,7 @@ use crate::index::IndexStatistics;
 use crate::query::Select;
 use crate::table::AddDataMode;
 use crate::utils::{supported_btree_data_type, supported_vector_data_type};
-use crate::{Error, Table};
+use crate::Error;
 use arrow_array::RecordBatchReader;
 use arrow_ipc::reader::FileReader;
 use arrow_schema::{DataType, SchemaRef};
@@ -22,7 +22,6 @@ use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::dataset::{ColumnAlteration, NewColumnTransform};
 use lance_datafusion::exec::OneShotExec;
 use serde::{Deserialize, Serialize};
-use tokio::sync::RwLock;
 
 use crate::{
     connection::NoData,
@@ -44,32 +43,17 @@ pub struct RemoteTable<S: HttpSend = Sender> {
     #[allow(dead_code)]
     client: RestfulLanceDbClient<S>,
     name: String,
-
-    version: RwLock<Option<u64>>,
 }
 
 impl<S: HttpSend> RemoteTable<S> {
     pub fn new(client: RestfulLanceDbClient<S>, name: String) -> Self {
-        Self {
-            client,
-            name,
-            version: RwLock::new(None),
-        }
+        Self { client, name }
     }
 
     async fn describe(&self) -> Result<TableDescription> {
-        let version = self.current_version().await;
-        self.describe_version(version).await
-    }
-
-    async fn describe_version(&self, version: Option<u64>) -> Result<TableDescription> {
-        let mut request = self
+        let request = self
             .client
             .post(&format!("/v1/table/{}/describe/", self.name));
-
-        let body = serde_json::json!({ "version": version });
-        request = request.json(&body);
-
         let (request_id, response) = self.client.send(request, true).await?;
 
         let response = self.check_table_response(&request_id, response).await?;
@@ -201,90 +185,6 @@ impl<S: HttpSend> RemoteTable<S> {
 
         Ok(())
     }
-
-    fn apply_vector_query_params(
-        mut body: serde_json::Value,
-        query: &VectorQuery,
-    ) -> Result<Vec<serde_json::Value>> {
-        Self::apply_query_params(&mut body, &query.base)?;
-
-        // Apply general parameters, before we dispatch based on number of query vectors.
-        body["prefilter"] = query.base.prefilter.into();
-        body["distance_type"] = serde_json::json!(query.distance_type.unwrap_or_default());
-        body["nprobes"] = query.nprobes.into();
-        body["ef"] = query.ef.into();
-        body["refine_factor"] = query.refine_factor.into();
-        if let Some(vector_column) = query.column.as_ref() {
-            body["vector_column"] = serde_json::Value::String(vector_column.clone());
-        }
-        if !query.use_index {
-            body["bypass_vector_index"] = serde_json::Value::Bool(true);
-        }
-
-        fn vector_to_json(vector: &arrow_array::ArrayRef) -> Result<serde_json::Value> {
-            match vector.data_type() {
-                DataType::Float32 => {
-                    let array = vector
-                        .as_any()
-                        .downcast_ref::<arrow_array::Float32Array>()
-                        .unwrap();
-                    Ok(serde_json::Value::Array(
-                        array
-                            .values()
-                            .iter()
-                            .map(|v| {
-                                serde_json::Value::Number(
-                                    serde_json::Number::from_f64(*v as f64).unwrap(),
-                                )
-                            })
-                            .collect(),
-                    ))
-                }
-                _ => Err(Error::InvalidInput {
-                    message: "VectorQuery vector must be of type Float32".into(),
-                }),
-            }
-        }
-
-        match query.query_vector.len() {
-            0 => {
-                // Server takes empty vector, not null or undefined.
-                body["vector"] = serde_json::Value::Array(Vec::new());
-                Ok(vec![body])
-            }
-            1 => {
-                body["vector"] = vector_to_json(&query.query_vector[0])?;
-                Ok(vec![body])
-            }
-            _ => {
-                let mut bodies = Vec::with_capacity(query.query_vector.len());
-                for vector in &query.query_vector {
-                    let mut body = body.clone();
-                    body["vector"] = vector_to_json(vector)?;
-                    bodies.push(body);
-                }
-                Ok(bodies)
-            }
-        }
-    }
-
-    async fn check_mutable(&self) -> Result<()> {
-        let read_guard = self.version.read().await;
-        match *read_guard {
-            None => Ok(()),
-            Some(version) => Err(Error::NotSupported {
-                message: format!(
-                    "Cannot mutate table reference fixed at version {}. Call checkout_latest() to get a mutable table reference.",
-                    version
-                )
-            })
-        }
-    }
-
-    async fn current_version(&self) -> Option<u64> {
-        let read_guard = self.version.read().await;
-        *read_guard
-    }
 }
 
 #[derive(Deserialize)]
@@ -312,11 +212,7 @@ mod test_utils {
             T: Into<reqwest::Body>,
         {
             let client = client_with_handler(handler);
-            Self {
-                client,
-                name,
-                version: RwLock::new(None),
-            }
+            Self { client, name }
         }
     }
 }
@@ -335,30 +231,17 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
     async fn version(&self) -> Result<u64> {
         self.describe().await.map(|desc| desc.version)
     }
-    async fn checkout(&self, version: u64) -> Result<()> {
-        // check that the version exists
-        self.describe_version(Some(version))
-            .await
-            .map_err(|e| match e {
-                // try to map the error to a more user-friendly error telling them
-                // specifically that the version does not exist
-                Error::TableNotFound { name } => Error::TableNotFound {
-                    name: format!("{} (version: {})", name, version),
-                },
-                e => e,
-            })?;
-
-        let mut write_guard = self.version.write().await;
-        *write_guard = Some(version);
-        Ok(())
+    async fn checkout(&self, _version: u64) -> Result<()> {
+        Err(Error::NotSupported {
+            message: "checkout is not supported on LanceDB cloud.".into(),
+        })
     }
     async fn checkout_latest(&self) -> Result<()> {
-        let mut write_guard = self.version.write().await;
-        *write_guard = None;
-        Ok(())
+        Err(Error::NotSupported {
+            message: "checkout is not supported on LanceDB cloud.".into(),
+        })
     }
     async fn restore(&self) -> Result<()> {
-        self.check_mutable().await?;
         Err(Error::NotSupported {
             message: "restore is not supported on LanceDB cloud.".into(),
         })
@@ -372,13 +255,10 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
             .client
             .post(&format!("/v1/table/{}/count_rows/", self.name));
 
-        let version = self.current_version().await;
-
         if let Some(filter) = filter {
-            request = request.json(&serde_json::json!({ "predicate": filter, "version": version }));
+            request = request.json(&serde_json::json!({ "predicate": filter }));
         } else {
-            let body = serde_json::json!({ "version": version });
-            request = request.json(&body);
+            request = request.json(&serde_json::json!({}));
         }
 
         let (request_id, response) = self.client.send(request, true).await?;
@@ -398,7 +278,6 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
         add: AddDataBuilder<NoData>,
         data: Box<dyn RecordBatchReader + Send>,
     ) -> Result<()> {
-        self.check_mutable().await?;
         let body = Self::reader_as_body(data)?;
         let mut request = self
             .client
@@ -427,30 +306,51 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let request = self.client.post(&format!("/v1/table/{}/query/", self.name));
 
-        let version = self.current_version().await;
-        let body = serde_json::json!({ "version": version });
-        let bodies = Self::apply_vector_query_params(body, query)?;
+        let mut body = serde_json::Value::Object(Default::default());
+        Self::apply_query_params(&mut body, &query.base)?;
 
-        let mut futures = Vec::with_capacity(bodies.len());
-        for body in bodies {
-            let request = request.try_clone().unwrap().json(&body);
-            let future = async move {
-                let (request_id, response) = self.client.send(request, true).await?;
-                self.read_arrow_stream(&request_id, response).await
-            };
-            futures.push(future);
-        }
-        let streams = futures::future::try_join_all(futures).await?;
-        if streams.len() == 1 {
-            let stream = streams.into_iter().next().unwrap();
-            Ok(Arc::new(OneShotExec::new(stream)))
+        body["prefilter"] = query.base.prefilter.into();
+        body["distance_type"] = serde_json::json!(query.distance_type.unwrap_or_default());
+        body["nprobes"] = query.nprobes.into();
+        body["refine_factor"] = query.refine_factor.into();
+
+        let vector: Vec<f32> = if let Some(vector) = query.query_vector.as_ref() {
+            match vector.data_type() {
+                DataType::Float32 => vector
+                    .as_any()
+                    .downcast_ref::<arrow_array::Float32Array>()
+                    .unwrap()
+                    .values()
+                    .iter()
+                    .cloned()
+                    .collect(),
+                _ => {
+                    return Err(Error::InvalidInput {
+                        message: "VectorQuery vector must be of type Float32".into(),
+                    })
+                }
+            }
         } else {
-            let stream_execs = streams
-                .into_iter()
-                .map(|stream| Arc::new(OneShotExec::new(stream)) as Arc<dyn ExecutionPlan>)
-                .collect();
-            Table::multi_vector_plan(stream_execs)
+            // Server takes empty vector, not null or undefined.
+            Vec::new()
+        };
+        body["vector"] = serde_json::json!(vector);
+
+        if let Some(vector_column) = query.column.as_ref() {
+            body["vector_column"] = serde_json::Value::String(vector_column.clone());
         }
+
+        if !query.use_index {
+            body["bypass_vector_index"] = serde_json::Value::Bool(true);
+        }
+
+        let request = request.json(&body);
+
+        let (request_id, response) = self.client.send(request, true).await?;
+
+        let stream = self.read_arrow_stream(&request_id, response).await?;
+
+        Ok(Arc::new(OneShotExec::new(stream)))
     }
 
     async fn plain_query(
@@ -463,8 +363,7 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
             .post(&format!("/v1/table/{}/query/", self.name))
             .header(CONTENT_TYPE, JSON_CONTENT_TYPE);
 
-        let version = self.current_version().await;
-        let mut body = serde_json::json!({ "version": version });
+        let mut body = serde_json::Value::Object(Default::default());
         Self::apply_query_params(&mut body, query)?;
         // Empty vector can be passed if no vector search is performed.
         body["vector"] = serde_json::Value::Array(Vec::new());
@@ -478,7 +377,6 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
         Ok(DatasetRecordBatchStream::new(stream))
     }
     async fn update(&self, update: UpdateBuilder) -> Result<u64> {
-        self.check_mutable().await?;
         let request = self
             .client
             .post(&format!("/v1/table/{}/update/", self.name));
@@ -500,7 +398,6 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
         Ok(0) // TODO: support returning number of modified rows once supported in SaaS.
     }
     async fn delete(&self, predicate: &str) -> Result<()> {
-        self.check_mutable().await?;
         let body = serde_json::json!({ "predicate": predicate });
         let request = self
             .client
@@ -512,7 +409,6 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
     }
 
     async fn create_index(&self, mut index: IndexBuilder) -> Result<()> {
-        self.check_mutable().await?;
         let request = self
             .client
             .post(&format!("/v1/table/{}/create_index/", self.name));
@@ -591,7 +487,6 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
         params: MergeInsertBuilder,
         new_data: Box<dyn RecordBatchReader + Send>,
     ) -> Result<()> {
-        self.check_mutable().await?;
         let query = MergeInsertRequest::try_from(params)?;
         let body = Self::reader_as_body(new_data)?;
         let request = self
@@ -608,7 +503,6 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
         Ok(())
     }
     async fn optimize(&self, _action: OptimizeAction) -> Result<OptimizeStats> {
-        self.check_mutable().await?;
         Err(Error::NotSupported {
             message: "optimize is not supported on LanceDB cloud.".into(),
         })
@@ -618,19 +512,16 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
         _transforms: NewColumnTransform,
         _read_columns: Option<Vec<String>>,
     ) -> Result<()> {
-        self.check_mutable().await?;
         Err(Error::NotSupported {
             message: "add_columns is not yet supported.".into(),
         })
     }
     async fn alter_columns(&self, _alterations: &[ColumnAlteration]) -> Result<()> {
-        self.check_mutable().await?;
         Err(Error::NotSupported {
             message: "alter_columns is not yet supported.".into(),
         })
     }
     async fn drop_columns(&self, _columns: &[&str]) -> Result<()> {
-        self.check_mutable().await?;
         Err(Error::NotSupported {
             message: "drop_columns is not yet supported.".into(),
         })
@@ -638,13 +529,9 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
 
     async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
         // Make request to list the indices
-        let mut request = self
+        let request = self
             .client
             .post(&format!("/v1/table/{}/index/list/", self.name));
-        let version = self.current_version().await;
-        let body = serde_json::json!({ "version": version });
-        request = request.json(&body);
-
         let (request_id, response) = self.client.send(request, true).await?;
         let response = self.check_table_response(&request_id, response).await?;
 
@@ -694,14 +581,10 @@ impl<S: HttpSend> TableInternal for RemoteTable<S> {
     }
 
     async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {
-        let mut request = self.client.post(&format!(
+        let request = self.client.post(&format!(
             "/v1/table/{}/index/{}/stats/",
             self.name, index_name
         ));
-        let version = self.current_version().await;
-        let body = serde_json::json!({ "version": version });
-        request = request.json(&body);
-
         let (request_id, response) = self.client.send(request, true).await?;
 
         if response.status() == StatusCode::NOT_FOUND {
@@ -772,7 +655,6 @@ mod tests {
 
     use super::*;
 
-    use arrow::{array::AsArray, compute::concat_batches, datatypes::Int32Type};
     use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator};
     use arrow_schema::{DataType, Field, Schema};
     use futures::{future::BoxFuture, StreamExt, TryFutureExt};
@@ -879,10 +761,7 @@ mod tests {
                 request.headers().get("Content-Type").unwrap(),
                 JSON_CONTENT_TYPE
             );
-            assert_eq!(
-                request.body().unwrap().as_bytes().unwrap(),
-                br#"{"version":null}"#
-            );
+            assert_eq!(request.body().unwrap().as_bytes().unwrap(), br#"{}"#);
 
             http::Response::builder().status(200).body("42").unwrap()
         });
@@ -899,7 +778,7 @@ mod tests {
             );
             assert_eq!(
                 request.body().unwrap().as_bytes().unwrap(),
-                br#"{"predicate":"a > 10","version":null}"#
+                br#"{"predicate":"a > 10"}"#
             );
 
             http::Response::builder().status(200).body("42").unwrap()
@@ -1198,9 +1077,7 @@ mod tests {
                 "prefilter": true,
                 "distance_type": "l2",
                 "nprobes": 20,
-                "ef": Option::<usize>::None,
                 "refine_factor": null,
-                "version": null,
             });
             // Pass vector separately to make sure it matches f32 precision.
             expected_body["vector"] = vec![0.1f32, 0.2, 0.3].into();
@@ -1245,9 +1122,7 @@ mod tests {
                 "bypass_vector_index": true,
                 "columns": ["a", "b"],
                 "nprobes": 12,
-                "ef": Option::<usize>::None,
                 "refine_factor": 2,
-                "version": null,
             });
             // Pass vector separately to make sure it matches f32 precision.
             expected_body["vector"] = vec![0.1f32, 0.2, 0.3].into();
@@ -1303,7 +1178,6 @@ mod tests {
                 "k": 10,
                 "vector": [],
                 "with_row_id": true,
-                "version": null
             });
             assert_eq!(body, expected_body);
 
@@ -1333,52 +1207,6 @@ mod tests {
             .unwrap();
     }
 
-    #[tokio::test]
-    async fn test_query_multiple_vectors() {
-        let table = Table::new_with_handler("my_table", |request| {
-            assert_eq!(request.method(), "POST");
-            assert_eq!(request.url().path(), "/v1/table/my_table/query/");
-            assert_eq!(
-                request.headers().get("Content-Type").unwrap(),
-                JSON_CONTENT_TYPE
-            );
-            let data = RecordBatch::try_new(
-                Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
-                vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-            )
-            .unwrap();
-            let response_body = write_ipc_file(&data);
-            http::Response::builder()
-                .status(200)
-                .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
-                .body(response_body)
-                .unwrap()
-        });
-
-        let query = table
-            .query()
-            .nearest_to(vec![0.1, 0.2, 0.3])
-            .unwrap()
-            .add_query_vector(vec![0.4, 0.5, 0.6])
-            .unwrap();
-        let plan = query.explain_plan(true).await.unwrap();
-        assert!(plan.contains("UnionExec"), "Plan: {}", plan);
-
-        let results = query
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        let results = concat_batches(&results[0].schema(), &results).unwrap();
-
-        let query_index = results["query_index"].as_primitive::<Int32Type>();
-        // We don't guarantee order.
-        assert!(query_index.values().contains(&0));
-        assert!(query_index.values().contains(&1));
-    }
-
     #[tokio::test]
     async fn test_create_index() {
         let cases = [
@@ -1533,195 +1361,4 @@ mod tests {
         let indices = table.index_stats("my_index").await.unwrap();
         assert!(indices.is_none());
     }
-
-    #[tokio::test]
-    async fn test_passes_version() {
-        let table = Table::new_with_handler("my_table", |request| {
-            let body = request.body().unwrap().as_bytes().unwrap();
-            let body: serde_json::Value = serde_json::from_slice(body).unwrap();
-            let version = body
-                .as_object()
-                .unwrap()
-                .get("version")
-                .unwrap()
-                .as_u64()
-                .unwrap();
-            assert_eq!(version, 42);
-
-            let response_body = match request.url().path() {
-                "/v1/table/my_table/describe/" => {
-                    serde_json::json!({
-                        "version": 42,
-                        "schema": { "fields": [] }
-                    })
-                }
-                "/v1/table/my_table/index/list/" => {
-                    serde_json::json!({
-                        "indexes": []
-                    })
-                }
-                "/v1/table/my_table/index/my_idx/stats/" => {
-                    serde_json::json!({
-                        "num_indexed_rows": 100000,
-                        "num_unindexed_rows": 0,
-                        "index_type": "IVF_PQ",
-                        "distance_type": "l2"
-                    })
-                }
-                "/v1/table/my_table/count_rows/" => {
-                    serde_json::json!(1000)
-                }
-                "/v1/table/my_table/query/" => {
-                    let expected_data = RecordBatch::try_new(
-                        Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
-                        vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-                    )
-                    .unwrap();
-                    let expected_data_ref = expected_data.clone();
-                    let response_body = write_ipc_file(&expected_data_ref);
-                    return http::Response::builder()
-                        .status(200)
-                        .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
-                        .body(response_body)
-                        .unwrap();
-                }
-
-                path => panic!("Unexpected path: {}", path),
-            };
-
-            http::Response::builder()
-                .status(200)
-                .body(
-                    serde_json::to_string(&response_body)
-                        .unwrap()
-                        .as_bytes()
-                        .to_vec(),
-                )
-                .unwrap()
-        });
-
-        table.checkout(42).await.unwrap();
-
-        // ensure that version is passed to the /describe endpoint
-        let version = table.version().await.unwrap();
-        assert_eq!(version, 42);
-
-        // ensure it's passed to other read API calls
-        table.list_indices().await.unwrap();
-        table.index_stats("my_idx").await.unwrap();
-        table.count_rows(None).await.unwrap();
-        table
-            .query()
-            .nearest_to(vec![0.1, 0.2, 0.3])
-            .unwrap()
-            .execute()
-            .await
-            .unwrap();
-    }
-
-    #[tokio::test]
-    async fn test_fails_if_checkout_version_doesnt_exist() {
-        let table = Table::new_with_handler("my_table", |request| {
-            let body = request.body().unwrap().as_bytes().unwrap();
-            let body: serde_json::Value = serde_json::from_slice(body).unwrap();
-            let version = body
-                .as_object()
-                .unwrap()
-                .get("version")
-                .unwrap()
-                .as_u64()
-                .unwrap();
-            if version != 42 {
-                return http::Response::builder()
-                    .status(404)
-                    .body(format!("Table my_table (version: {}) not found", version))
-                    .unwrap();
-            }
-
-            let response_body = match request.url().path() {
-                "/v1/table/my_table/describe/" => {
-                    serde_json::json!({
-                        "version": 42,
-                        "schema": { "fields": [] }
-                    })
-                }
-                _ => panic!("Unexpected path"),
-            };
-
-            http::Response::builder()
-                .status(200)
-                .body(serde_json::to_string(&response_body).unwrap())
-                .unwrap()
-        });
-
-        let res = table.checkout(43).await;
-        println!("{:?}", res);
-        assert!(
-            matches!(res, Err(Error::TableNotFound { name }) if name == "my_table (version: 43)")
-        );
-    }
-
-    #[tokio::test]
-    async fn test_timetravel_immutable() {
-        let table = Table::new_with_handler::<String>("my_table", |request| {
-            let response_body = match request.url().path() {
-                "/v1/table/my_table/describe/" => {
-                    serde_json::json!({
-                        "version": 42,
-                        "schema": { "fields": [] }
-                    })
-                }
-                _ => panic!("Should not have made a request: {:?}", request),
-            };
-
-            http::Response::builder()
-                .status(200)
-                .body(serde_json::to_string(&response_body).unwrap())
-                .unwrap()
-        });
-
-        table.checkout(42).await.unwrap();
-
-        // Ensure that all mutable operations fail.
-        let res = table
-            .update()
-            .column("a", "a + 1")
-            .column("b", "b - 1")
-            .only_if("b > 10")
-            .execute()
-            .await;
-        assert!(matches!(res, Err(Error::NotSupported { .. })));
-
-        let batch = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-        let data = Box::new(RecordBatchIterator::new(
-            [Ok(batch.clone())],
-            batch.schema(),
-        ));
-        let res = table.merge_insert(&["some_col"]).execute(data).await;
-        assert!(matches!(res, Err(Error::NotSupported { .. })));
-
-        let res = table.delete("id in (1, 2, 3)").await;
-        assert!(matches!(res, Err(Error::NotSupported { .. })));
-
-        let data = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-        let res = table
-            .add(RecordBatchIterator::new([Ok(data.clone())], data.schema()))
-            .execute()
-            .await;
-        assert!(matches!(res, Err(Error::NotSupported { .. })));
-
-        let res = table
-            .create_index(&["a"], Index::IvfPq(Default::default()))
-            .execute()
-            .await;
-        assert!(matches!(res, Err(Error::NotSupported { .. })));
-    }
 }

rust/lancedb/src/table.rs

@@ -24,9 +24,6 @@ use arrow_array::{RecordBatchIterator, RecordBatchReader};
 use arrow_schema::{Field, Schema, SchemaRef};
 use async_trait::async_trait;
 use datafusion_physical_plan::display::DisplayableExecutionPlan;
-use datafusion_physical_plan::projection::ProjectionExec;
-use datafusion_physical_plan::repartition::RepartitionExec;
-use datafusion_physical_plan::union::UnionExec;
 use datafusion_physical_plan::ExecutionPlan;
 use futures::{StreamExt, TryStreamExt};
 use lance::dataset::builder::DatasetBuilder;
@@ -975,57 +972,6 @@ impl Table {
     ) -> Result<Option<IndexStatistics>> {
         self.inner.index_stats(index_name.as_ref()).await
     }
-
-    // Take many execution plans and map them into a single plan that adds
-    // a query_index column and unions them.
-    pub(crate) fn multi_vector_plan(
-        plans: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        if plans.is_empty() {
-            return Err(Error::InvalidInput {
-                message: "No plans provided".to_string(),
-            });
-        }
-        // Projection to keeping all existing columns
-        let first_plan = plans[0].clone();
-        let project_all_columns = first_plan
-            .schema()
-            .fields()
-            .iter()
-            .enumerate()
-            .map(|(i, field)| {
-                let expr =
-                    datafusion_physical_plan::expressions::Column::new(field.name().as_str(), i);
-                let expr = Arc::new(expr) as Arc<dyn datafusion_physical_plan::PhysicalExpr>;
-                (expr, field.name().clone())
-            })
-            .collect::<Vec<_>>();
-
-        let projected_plans = plans
-            .into_iter()
-            .enumerate()
-            .map(|(plan_i, plan)| {
-                let query_index = datafusion_common::ScalarValue::Int32(Some(plan_i as i32));
-                let query_index_expr =
-                    datafusion_physical_plan::expressions::Literal::new(query_index);
-                let query_index_expr =
-                    Arc::new(query_index_expr) as Arc<dyn datafusion_physical_plan::PhysicalExpr>;
-                let mut projections = vec![(query_index_expr, "query_index".to_string())];
-                projections.extend_from_slice(&project_all_columns);
-                let projection = ProjectionExec::try_new(projections, plan).unwrap();
-                Arc::new(projection) as Arc<dyn datafusion_physical_plan::ExecutionPlan>
-            })
-            .collect::<Vec<_>>();
-
-        let unioned = Arc::new(UnionExec::new(projected_plans));
-        // We require 1 partition in the final output
-        let repartitioned = RepartitionExec::try_new(
-            unioned,
-            datafusion_physical_plan::Partitioning::RoundRobinBatch(1),
-        )
-        .unwrap();
-        Ok(Arc::new(repartitioned))
-    }
 }
 
 impl From<NativeTable> for Table {
@@ -1838,25 +1784,9 @@ impl TableInternal for NativeTable {
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let ds_ref = self.dataset.get().await?;
 
-        if query.query_vector.len() > 1 {
-            // If there are multiple query vectors, create a plan for each of them and union them.
-            let query_vecs = query.query_vector.clone();
-            let plan_futures = query_vecs
-                .into_iter()
-                .map(|query_vector| {
-                    let mut sub_query = query.clone();
-                    sub_query.query_vector = vec![query_vector];
-                    let options_ref = options.clone();
-                    async move { self.create_plan(&sub_query, options_ref).await }
-                })
-                .collect::<Vec<_>>();
-            let plans = futures::future::try_join_all(plan_futures).await?;
-            return Table::multi_vector_plan(plans);
-        }
-
         let mut scanner: Scanner = ds_ref.scan();
 
-        if let Some(query_vector) = query.query_vector.first() {
+        if let Some(query_vector) = query.query_vector.as_ref() {
             // If there is a vector query, default to limit=10 if unspecified
             let column = if let Some(col) = query.column.as_ref() {
                 col.clone()
@@ -1898,15 +1828,19 @@ impl TableInternal for NativeTable {
                 query_vector,
                 query.base.limit.unwrap_or(DEFAULT_TOP_K),
             )?;
+            scanner.limit(
+                query.base.limit.map(|limit| limit as i64),
+                query.base.offset.map(|offset| offset as i64),
+            )?;
+        } else {
+            // If there is no vector query, it's ok to not have a limit
+            scanner.limit(
+                query.base.limit.map(|limit| limit as i64),
+                query.base.offset.map(|offset| offset as i64),
+            )?;
         }
-        scanner.limit(
-            query.base.limit.map(|limit| limit as i64),
-            query.base.offset.map(|offset| offset as i64),
-        )?;
+
         scanner.nprobs(query.nprobes);
-        if let Some(ef) = query.ef {
-            scanner.ef(ef);
-        }
         scanner.use_index(query.use_index);
         scanner.prefilter(query.base.prefilter);
         match query.base.select {