more edit

code for cloud doc
2025-12-25 06:19:57 +00:00 · 2024-11-14 13:33:25 -08:00 · 2024-11-13 22:05:09 -08:00 · 2024-11-13 22:03:53 -08:00 · 2024-11-13 21:57:05 -08:00 · 2024-11-13 11:10:56 -08:00
91 changed files with 9350 additions and 2177 deletions
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.12.0"
+current_version = "0.13.0-beta.1"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -92,6 +92,11 @@ glob = "node/package.json"
 replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""
 search = "\"@lancedb/vectordb-win32-x64-msvc\": \"{current_version}\""

+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-win32-arm64-msvc\": \"{new_version}\""
+search = "\"@lancedb/vectordb-win32-arm64-msvc\": \"{current_version}\""
+
 # Cargo files
 # ------------
 [[tool.bumpversion.files]]
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -38,3 +38,7 @@ rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm
 # not found errors on systems that are missing it.
 [target.x86_64-pc-windows-msvc]
 rustflags = ["-Ctarget-feature=+crt-static"]
+
+# Experimental target for Arm64 Windows
+[target.aarch64-pc-windows-msvc]
+rustflags = ["-Ctarget-feature=+crt-static"]
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -31,7 +31,7 @@ jobs:
      - name: Install dependecies needed for ubuntu
        run: |
          sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default        
+          rustup update && rustup default
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
@@ -41,8 +41,8 @@ jobs:
      - name: Build Python
        working-directory: python
        run: |
-          python -m pip install -e .
-          python -m pip install -r ../docs/requirements.txt
+          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .
+          python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
      - name: Set up node
        uses: actions/setup-node@v3
        with:
--- a/.github/workflows/docs_test.yml
+++ b/.github/workflows/docs_test.yml
@@ -49,7 +49,7 @@ jobs:
    - name: Build Python
      working-directory: docs/test
      run:
-        python -m pip install -r requirements.txt
+        python -m pip install --extra-index-url https://pypi.fury.io/lancedb/ -r requirements.txt
    - name: Create test files
      run: |
        cd docs/test
--- a/.github/workflows/nodejs.yml
+++ b/.github/workflows/nodejs.yml
@@ -53,6 +53,9 @@ jobs:
        cargo clippy --all --all-features -- -D warnings
        npm ci
        npm run lint-ci
+    - name: Lint examples
+      working-directory: nodejs/examples
+      run: npm ci && npm run lint-ci
  linux:
    name: Linux (NodeJS ${{ matrix.node-version }})
    timeout-minutes: 30
@@ -91,6 +94,19 @@ jobs:
      env:
        S3_TEST: "1"
      run: npm run test
+    - name: Setup examples
+      working-directory: nodejs/examples
+      run: npm ci
+    - name: Test examples
+      working-directory: ./
+      env:
+        OPENAI_API_KEY: test
+        OPENAI_BASE_URL: http://0.0.0.0:8000
+      run: |
+        python ci/mock_openai.py &
+        ss -ltnp | grep :8000
+        cd nodejs/examples
+        npm test
  macos:
    timeout-minutes: 30
    runs-on: "macos-14"
--- a/.github/workflows/npm-publish.yml
+++ b/.github/workflows/npm-publish.yml
@@ -226,6 +226,109 @@ jobs:
          path: |
            node/dist/lancedb-vectordb-win32*.tgz

+  node-windows-arm64:
+    name: vectordb win32-arm64-msvc
+    runs-on: windows-4x-arm
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Git
+        run: |
+          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+        shell: powershell
+      - name: Add Git to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+        shell: powershell
+      - name: Configure Git symlinks
+        run: git config --global core.symlinks true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install Visual Studio Build Tools
+        run: |
+          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+            "--installPath", "C:\BuildTools", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+        shell: powershell
+      - name: Add Visual Studio Build Tools to PATH
+        run: |
+          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+
+          # Add MSVC runtime libraries to LIB
+          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" + 
+                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
+                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
+
+          # Add INCLUDE paths
+          $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
+          Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
+        shell: powershell
+      - name: Install Rust
+        run: |
+          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+        shell: powershell
+      - name: Add Rust to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+        shell: powershell
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install 7-Zip ARM
+        run: |
+          New-Item -Path 'C:\7zip' -ItemType Directory
+          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+        shell: powershell
+      - name: Add 7-Zip to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+        shell: powershell
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          if (Test-Path 'C:\protoc') {
+              Write-Host "Protoc directory exists, skipping installation"
+              return
+          }
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+        shell: powershell
+      - name: Add Protoc to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 aarch64-pc-windows-msvc
+      - name: Upload Windows ARM64 Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-native-windows-arm64
+          path: |
+            node/dist/*.node
+
  nodejs-windows:
    name: lancedb ${{ matrix.target }}
    runs-on: windows-2022
@@ -260,9 +363,102 @@ jobs:
          path: |
            nodejs/dist/*.node

+  nodejs-windows-arm64:
+    name: lancedb win32-arm64-msvc
+    runs-on: windows-4x-arm
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Git
+        run: |
+          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+        shell: powershell
+      - name: Add Git to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+        shell: powershell
+      - name: Configure Git symlinks
+        run: git config --global core.symlinks true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install Visual Studio Build Tools
+        run: |
+          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+            "--installPath", "C:\BuildTools", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+        shell: powershell
+      - name: Add Visual Studio Build Tools to PATH
+        run: |
+          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+
+          $env:LIB = ""
+          Add-Content $env:GITHUB_ENV "LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+        shell: powershell
+      - name: Install Rust
+        run: |
+          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+        shell: powershell
+      - name: Add Rust to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+        shell: powershell
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install 7-Zip ARM
+        run: |
+          New-Item -Path 'C:\7zip' -ItemType Directory
+          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+        shell: powershell
+      - name: Add 7-Zip to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+        shell: powershell
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          if (Test-Path 'C:\protoc') {
+              Write-Host "Protoc directory exists, skipping installation"
+              return
+          }
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+        shell: powershell
+      - name: Add Protoc to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts_nodejs.ps1 aarch64-pc-windows-msvc
+      - name: Upload Windows ARM64 Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows-arm64
+          path: |
+            nodejs/dist/*.node
+
  release:
    name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux, node-windows]
+    needs: [node, node-macos, node-linux, node-windows, node-windows-arm64]
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
@@ -302,7 +498,7 @@ jobs:

  release-nodejs:
    name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows, nodejs-windows-arm64]
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -138,7 +138,7 @@ jobs:
        run: rm -rf target/wheels
  windows:
    name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 30
+    timeout-minutes: 60
    strategy:
      matrix:
        config:
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -35,21 +35,22 @@ jobs:
      CC: clang-18
      CXX: clang++-18
    steps:
-    - uses: actions/checkout@v4
-      with:
+      - uses: actions/checkout@v4
+        with:
          fetch-depth: 0
          lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: rust
-    - name: Install dependencies
-      run: |
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install dependencies
+        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Run format
-      run: cargo fmt --all -- --check
-    - name: Run clippy
-      run: cargo clippy --workspace --tests --all-features -- -D warnings
+      - name: Run format
+        run: cargo fmt --all -- --check
+      - name: Run clippy
+        run: cargo clippy --workspace --tests --all-features -- -D warnings
+
  linux:
    timeout-minutes: 30
    # To build all features, we need more disk space than is available
@@ -65,37 +66,38 @@ jobs:
      CC: clang-18
      CXX: clang++-18
    steps:
-    - uses: actions/checkout@v4
-      with:
+      - uses: actions/checkout@v4
+        with:
          fetch-depth: 0
          lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
+      - uses: Swatinem/rust-cache@v2
+        with:
          workspaces: rust
-    - name: Install dependencies
-      run: |
+      - name: Install dependencies
+        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Make Swap
-      run: |
-        sudo fallocate -l 16G /swapfile
-        sudo chmod 600 /swapfile
-        sudo mkswap /swapfile
-        sudo swapon /swapfile
-    - name: Start S3 integration test environment
-      working-directory: .
-      run: docker compose up --detach --wait
-    - name: Build
-      run: cargo build --all-features
-    - name: Run tests
-      run: cargo test --all-features
-    - name: Run examples
-      run: cargo run --example simple
+      - name: Make Swap
+        run: |
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+      - name: Start S3 integration test environment
+        working-directory: .
+        run: docker compose up --detach --wait
+      - name: Build
+        run: cargo build --all-features
+      - name: Run tests
+        run: cargo test --all-features
+      - name: Run examples
+        run: cargo run --example simple
+
  macos:
    timeout-minutes: 30
    strategy:
      matrix:
-        mac-runner: [ "macos-13", "macos-14" ]
+        mac-runner: ["macos-13", "macos-14"]
    runs-on: "${{ matrix.mac-runner }}"
    defaults:
      run:
@@ -104,8 +106,8 @@ jobs:
    steps:
      - uses: actions/checkout@v4
        with:
-            fetch-depth: 0
-            lfs: true
+          fetch-depth: 0
+          lfs: true
      - name: CPU features
        run: sysctl -a | grep cpu
      - uses: Swatinem/rust-cache@v2
@@ -118,6 +120,7 @@ jobs:
      - name: Run tests
        # Run with everything except the integration tests.
        run: cargo test --features remote,fp16kernels
+
  windows:
    runs-on: windows-2022
    steps:
@@ -139,3 +142,99 @@ jobs:
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo build
          cargo test
+
+  windows-arm64:
+    runs-on: windows-4x-arm
+    steps:
+      - name: Install Git
+        run: |
+          Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+          Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+        shell: powershell
+      - name: Add Git to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+          $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+        shell: powershell
+      - name: Configure Git symlinks
+        run: git config --global core.symlinks true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install Visual Studio Build Tools
+        run: |
+          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+          Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+            "--installPath", "C:\BuildTools", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+            "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+            "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+            "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+        shell: powershell
+      - name: Add Visual Studio Build Tools to PATH
+        run: |
+          $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+          $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+          Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+
+          # Add MSVC runtime libraries to LIB
+          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" + 
+                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
+                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
+
+          # Add INCLUDE paths
+          $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
+                        "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
+          Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
+        shell: powershell
+      - name: Install Rust
+        run: |
+          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+        shell: powershell
+      - name: Add Rust to PATH
+        run: |
+          Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+        shell: powershell
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install 7-Zip ARM
+        run: |
+          New-Item -Path 'C:\7zip' -ItemType Directory
+          Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+          Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+        shell: powershell
+      - name: Add 7-Zip to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+        shell: powershell
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          if (Test-Path 'C:\protoc') {
+            Write-Host "Protoc directory exists, skipping installation"
+            return
+          }
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+        shell: powershell
+      - name: Add Protoc to PATH
+        run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Run tests
+        run: |
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
+          cargo build --target aarch64-pc-windows-msvc
+          cargo test --target aarch64-pc-windows-msvc
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@
 [![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/)
 [![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd)
 [![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20LanceDB%20Guru-006BFF?style=for-the-badge)](https://gurubase.io/g/lancedb)

 </p>

--- a/ci/build_windows_artifacts.ps1
+++ b/ci/build_windows_artifacts.ps1
@@ -3,6 +3,7 @@
 # Targets supported:
 # - x86_64-pc-windows-msvc
 # - i686-pc-windows-msvc
+# - aarch64-pc-windows-msvc

 function Prebuild-Rust {
    param (
@@ -31,7 +32,7 @@ function Build-NodeBinaries {

 $targets = $args[0]
 if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
+    $targets = "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"
 }

 Write-Host "Building artifacts for targets: $targets"
--- a/ci/build_windows_artifacts_nodejs.ps1
+++ b/ci/build_windows_artifacts_nodejs.ps1
@@ -3,6 +3,7 @@
 # Targets supported:
 # - x86_64-pc-windows-msvc
 # - i686-pc-windows-msvc
+# - aarch64-pc-windows-msvc

 function Prebuild-Rust {
    param (
@@ -31,7 +32,7 @@ function Build-NodeBinaries {

 $targets = $args[0]
 if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
+    $targets = "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc"
 }

 Write-Host "Building artifacts for targets: $targets"
--- a/ci/mock_openai.py
+++ b/ci/mock_openai.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
+import argparse
+import json
+import http.server
+
+
+class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers["Content-Length"])
+        post_data = self.rfile.read(content_length)
+        post_data = json.loads(post_data.decode("utf-8"))
+        # See: https://platform.openai.com/docs/api-reference/embeddings/create
+
+        if isinstance(post_data["input"], str):
+            num_inputs = 1
+        else:
+            num_inputs = len(post_data["input"])
+
+        model = post_data.get("model", "text-embedding-ada-002")
+
+        data = []
+        for i in range(num_inputs):
+            data.append({
+                "object": "embedding",
+                "embedding": [0.1] * 1536,
+                "index": i,
+            })
+
+        response = {
+            "object": "list",
+            "data": data,
+            "model": model,
+            "usage": {
+                "prompt_tokens": 0,
+                "total_tokens": 0,
+            }
+        }
+
+        self.send_response(200)
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode("utf-8"))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Mock OpenAI embeddings API endpoint")
+    parser.add_argument("--port", type=int, default=8000, help="Port to listen on")
+    args = parser.parse_args()
+    port = args.port
+
+    print(f"server started on port {port}. Press Ctrl-C to stop.")
+    print(f"To use, set OPENAI_BASE_URL=http://localhost:{port} in your environment.")
+
+    with http.server.HTTPServer(("0.0.0.0", port), MockOpenAIRequestHandler) as server:
+        server.serve_forever()
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -222,10 +222,12 @@ nav:
          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
      - ☁️ LanceDB Cloud:
          - Overview: cloud/index.md
-          - API reference:
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
+          - Quickstart: cloud/quickstart.md
+          - Best Practices: cloud/best_practices.md
+          # - API reference:
+          #     - 🐍 Python: python/saas-python.md
+          #     - 👾 JavaScript: javascript/modules.md
+          #     - REST API: cloud/rest.md

  - Quick start: basic.md
  - Concepts:
@@ -348,10 +350,17 @@ nav:
      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
  - LanceDB Cloud:
      - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
+      - Quickstart: cloud/quickstart.md
+      - Work with data: 
+        - Ingest data: cloud/ingest_data.md
+        - Update data: cloud/update_data.md
+        - Build an index: cloud/build_index.md
+        - Vector search: cloud/vector_search.md
+        - Full-text search: cloud/full_text_search.md
+        - Hybrid search: cloud/hybrid_search.md
+        - Metadata Filtering: cloud/metadata_filtering.md
+      - Best Practices: cloud/best_practices.md
+      # - REST API: cloud/rest.md

 extra_css:
  - styles/global.css
--- a/docs/src/ann_indexes.md
+++ b/docs/src/ann_indexes.md
@@ -45,9 +45,9 @@ Lance supports `IVF_PQ` index type by default.
        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.

        ```typescript
-        --8<--- "nodejs/examples/ann_indexes.ts:import"
+        --8<--- "nodejs/examples/ann_indexes.test.ts:import"

-        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:ingest"
        ```

    === "vectordb (deprecated)"
@@ -140,13 +140,15 @@ There are a couple of parameters that can be used to fine-tune the search:

 - **limit** (default: 10): The amount of results that will be returned
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
-  Most of the time, setting nprobes to cover 5-10% of the dataset should achieve high recall with low latency.<br/>
-  e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
-  Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
+    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
+  
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
-  e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
-  Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
+    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
+!!! note 
+    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.
+

 === "Python"

@@ -169,7 +171,7 @@ There are a couple of parameters that can be used to fine-tune the search:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search1"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search1"
        ```

    === "vectordb (deprecated)"
@@ -203,7 +205,7 @@ You can further filter the elements returned by a search using a where clause.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search2"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search2"
        ```

    === "vectordb (deprecated)"
@@ -235,7 +237,7 @@ You can select the columns returned by the query using a select clause.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/ann_indexes.ts:search3"
+        --8<-- "nodejs/examples/ann_indexes.test.ts:search3"
        ```

    === "vectordb (deprecated)"
--- a/docs/src/basic.md
+++ b/docs/src/basic.md
@@ -157,7 +157,7 @@ recommend switching to stable releases.
        import * as lancedb from "@lancedb/lancedb";
        import * as arrow from "apache-arrow";

-        --8<-- "nodejs/examples/basic.ts:connect"
+        --8<-- "nodejs/examples/basic.test.ts:connect"
        ```

    === "vectordb (deprecated)"
@@ -212,7 +212,7 @@ table.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_table"
        ```

    === "vectordb (deprecated)"
@@ -268,7 +268,7 @@ similar to a `CREATE TABLE` statement in SQL.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
        ```

    === "vectordb (deprecated)"
@@ -298,7 +298,7 @@ Once created, you can open a table as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:open_table"
+        --8<-- "nodejs/examples/basic.test.ts:open_table"
        ```

    === "vectordb (deprecated)"
@@ -327,7 +327,7 @@ If you forget the name of your table, you can always get a listing of all table
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:table_names"
+        --8<-- "nodejs/examples/basic.test.ts:table_names"
        ```

    === "vectordb (deprecated)"
@@ -357,7 +357,7 @@ After a table has been created, you can always add more data to it as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:add_data"
+        --8<-- "nodejs/examples/basic.test.ts:add_data"
        ```

    === "vectordb (deprecated)"
@@ -389,7 +389,7 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:vector_search"
+        --8<-- "nodejs/examples/basic.test.ts:vector_search"
        ```

    === "vectordb (deprecated)"
@@ -429,7 +429,7 @@ LanceDB allows you to create an ANN index on a table as follows:
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_index"
+        --8<-- "nodejs/examples/basic.test.ts:create_index"
        ```

    === "vectordb (deprecated)"
@@ -469,7 +469,7 @@ This can delete any number of rows that match the filter.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:delete_rows"
+        --8<-- "nodejs/examples/basic.test.ts:delete_rows"
        ```

    === "vectordb (deprecated)"
@@ -527,7 +527,7 @@ Use the `drop_table()` method on the database to remove a table.
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:drop_table"
+        --8<-- "nodejs/examples/basic.test.ts:drop_table"
        ```

    === "vectordb (deprecated)"
@@ -561,8 +561,8 @@ You can use the embedding API when working with embedding models. It automatical
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/embedding.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
+        --8<-- "nodejs/examples/embedding.test.ts:imports"
+        --8<-- "nodejs/examples/embedding.test.ts:openai_embeddings"
        ```

 === "Rust"
--- a/docs/src/cloud/best_practices.md
+++ b/docs/src/cloud/best_practices.md
@@ -0,0 +1,20 @@
+This section provides a set of recommended best practices to help you get the most out of LanceDB Cloud. By following these guidelines, you can optimize your usage of LanceDB Cloud, improve performance, and ensure a smooth experience.
+
+### Should the db connection be created once and keep it open?
+Yes! It is recommended to establish a single db connection and maintain it throughout your interaction with the tables within. 
+
+LanceDB uses `requests.Session()` for connection pooling, which automatically manages connection reuse and cleanup. This approach avoids the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
+
+### Should a single `open_table` call be made and maintained for subsequent table operations?
+`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the latest version of the data. 
+
+### Row id
+
+### What are the vector indexing types supported by LanceDB Cloud?
+We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff betweeln query latency and query quality.
+
+### Do I need to do anything when there is new data added to a table with an existing index?
+No! LanceDB Cloud triggers an asynchronous background job to index the new vectors. This process will either merge the new vectors into the existing index or initiate a complete re-indexing if needed. 
+
+There is a flag `fast_search` in `table.search()` that allows you to control whether the unindexed rows should be searched or not.  
+
--- a/docs/src/cloud/build_index.md
+++ b/docs/src/cloud/build_index.md
@@ -0,0 +1,64 @@
+LanceDB Cloud supports **vector index**, **scalar index** and **full-text search index**. Compared to open-source version, LanceDB Cloud focuses on **automation**:
+
+- If there is a single vector column in the table, the vector column can be inferred from the schema and the index will be automatically created. 
+
+- Indexing parameters will be automatically tuned for customer's data.
+
+## Vector index
+LanceDB has implemented the state-of-art indexing algorithms (more about [IVF-PQ](https://lancedb.github.io/lancedb/concepts/index_ivfpq/) and [HNSW](https://lancedb.github.io/lancedb/concepts/index_hnsw/)). We currently 
+support the _L2_, _Cosine_ and _Dot_ as distance calculation metrics. You can create multiple vector indices within a table.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:create_index"
+    ```
+
+## Scalar index
+LanceDB Cloud and LanceDB Enterprise supports several types of Scalar indices to accelerate search over scalar columns.
+
+- *BTREE*: The most common type is BTREE. This index is inspired by the btree data structure although only the first few layers of the btree are cached in memory. It will perform well on columns with a large number of unique values and few rows per value.
+- *BITMAP*: this index stores a bitmap for each unique value in the column. This index is useful for columns with a finite number of unique values and many rows per value. 
+    - For example, columns that represent "categories", "labels", or "tags"
+- *LABEL_LIST*: a special index that is used to index list columns whose values have a finite set of possibilities. 
+    - For example, a column that contains lists of tags (e.g. ["tag1", "tag2", "tag3"]) can be indexed with a LABEL_LIST index.
+
+You can create multiple scalar indices within a table.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_scalar_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:create_scalar_index"
+    ```
+
+## Full-text search index
+We provide performant full-text search on LanceDB Cloud, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+!!! note ""
+
+    `use_tantivy` is not available with `create_fts_index` on LanceDB Cloud as we used our native implementation, which has better performance comparing to tantivy.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_fts_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:create_fts_index"
+    ```
--- a/docs/src/cloud/full_text_search.md
+++ b/docs/src/cloud/full_text_search.md
@@ -0,0 +1,14 @@
+The full-text search allows you to 
+incorporate keyword-based search (based on BM25) in your retrieval solutions.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:full_text_search"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:full_text_search"
+    ```
--- a/docs/src/cloud/hybrid_search.md
+++ b/docs/src/cloud/hybrid_search.md
@@ -0,0 +1,10 @@
+We support hybrid search that combines semantic and full-text search via a 
+reranking algorithm of your choice, to get the best of both worlds. LanceDB 
+comes with [built-in rerankers](https://lancedb.github.io/lancedb/reranking/) 
+and you can implement you own _customized reranker_ as well. 
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:hybrid_search"
+    ```
--- a/docs/src/cloud/ingest_data.md
+++ b/docs/src/cloud/ingest_data.md
@@ -0,0 +1,31 @@
+## Insert data
+The LanceDB Cloud SDK for data ingestion remains consistent with our open-source version, 
+ensuring a seamless transition for existing OSS users. 
+!!! note "unsupported parameters in create_table"
+
+    The following two parameters: `mode="overwrite"` and `exist_ok`, are expected to be added by Nov, 2024.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:import-ingest-data"
+    
+    --8<-- "python/python/tests/docs/test_cloud.py:ingest_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    --8<-- "nodejs/examples/cloud.test.ts:ingest_data"
+    ```
+
+## Insert large datasets
+It is recommended to use itertators to add large datasets in batches when creating 
+your table in one go. Data will be automatically compacted for the best query performance.
+!!! info "batch size"
+
+    The batch size .
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:ingest_data_in_batch"
+    ```
--- a/docs/src/cloud/metadata_filtering.md
+++ b/docs/src/cloud/metadata_filtering.md
@@ -0,0 +1,33 @@
+LanceDB Cloud supports rich filtering features of query results based on metadata fields. 
+
+By default, _post-filtering_ is performed on the top-k results returned by the vector search. 
+However, _pre-filtering_ is also an option that performs the filter prior to vector search. 
+This can be useful to narrow down on the search space on a very large dataset to reduce query 
+latency.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:filtering"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:filtering"
+    ```
+We also support standard SQL expressions as predicates for filtering operations. 
+It can be used during vector search, update, and deletion operations.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:sql_filtering"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:sql_filtering"
+    ```
--- a/docs/src/cloud/update_data.md
+++ b/docs/src/cloud/update_data.md
@@ -0,0 +1,49 @@
+LanceDB Cloud efficiently manages updates across many tables. 
+Currently, we offer _update_, _merge_insert_, and _delete_.
+
+## update
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:update_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:update_data"
+    ```
+
+## merge insert
+This merge insert can add rows, update rows, and remove rows all in a single transaction. 
+It combines new data from a source table with existing data in a target table by using a join.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:merge_insert"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:merge_insert"
+    ```
+
+## delete
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:delete_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:delete_data"
+    ```
--- a/docs/src/cloud/vector_search.md
+++ b/docs/src/cloud/vector_search.md
@@ -0,0 +1,21 @@
+Users can also tune the following parameters for better search quality.
+
+ - [nprobes](https://lancedb.github.io/lancedb/js/classes/VectorQuery/#nprobes): 
+ the number of partitions to search (probe).
+ - [refine factor](https://lancedb.github.io/lancedb/js/classes/VectorQuery/#refinefactor): 
+ a multiplier to control how many additional rows are taken during the refine step.
+
+ [Metadata filtering](filtering) combined with the vector search is also supported.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:vector_search"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:vector_search"
+    ```
--- a/docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
+++ b/docs/src/embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
@@ -0,0 +1,51 @@
+# VoyageAI Embeddings
+
+Voyage AI provides cutting-edge embedding and rerankers.
+
+
+Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API.
+
+Supported models are:
+
+- voyage-3
+- voyage-3-lite
+- voyage-finance-2
+- voyage-multilingual-2
+- voyage-law-2
+- voyage-code-2
+
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|--------|---------|
+| `name` | `str` | `"voyage-3"` | The model ID of the model to use. Supported base models for Text Embeddings: voyage-3, voyage-3-lite, voyage-finance-2, voyage-multilingual-2, voyage-law-2, voyage-code-2 |
+| `input_type` | `str` | `None` | Type of the input text. Default to None. Other options: query, document. |
+| `truncation` | `bool` | `True` | Whether to truncate the input texts to fit within the context length. |
+
+
+Usage Example:
+    
+```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    voyageai = EmbeddingFunctionRegistry
+        .get_instance()
+        .get("voyageai")
+        .create(name="voyage-3")
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
+
+    data = [ { "text": "hello world" },
+            { "text": "goodbye world" }]
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+```
--- a/docs/src/embeddings/custom_embedding_function.md
+++ b/docs/src/embeddings/custom_embedding_function.md
@@ -47,9 +47,9 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 === "TypeScript"

    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:imports"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:imports"

-    --8<--- "nodejs/examples/custom_embedding_function.ts:embedding_impl"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:embedding_impl"
    ```


@@ -78,7 +78,7 @@ Now you can use this embedding function to create your table schema and that's i
 === "TypeScript"

    ```ts
-    --8<--- "nodejs/examples/custom_embedding_function.ts:call_custom_function"
+    --8<--- "nodejs/examples/custom_embedding_function.test.ts:call_custom_function"
    ```

 !!! note
--- a/docs/src/embeddings/embedding_functions.md
+++ b/docs/src/embeddings/embedding_functions.md
@@ -94,8 +94,8 @@ the embeddings at all:
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/embedding.ts:imports"
-        --8<-- "nodejs/examples/embedding.ts:embedding_function"
+        --8<-- "nodejs/examples/embedding.test.ts:imports"
+        --8<-- "nodejs/examples/embedding.test.ts:embedding_function"
        ```

    === "vectordb (deprecated)"
@@ -150,7 +150,7 @@ need to worry about it when you query the table:
            .toArray()
        ```

-    === "vectordb (deprecated)
+    === "vectordb (deprecated)"

        ```ts
        const results = await table
--- a/docs/src/embeddings/index.md
+++ b/docs/src/embeddings/index.md
@@ -51,8 +51,8 @@ LanceDB registers the OpenAI embeddings function in the registry as `openai`. Yo
 === "TypeScript"

    ```typescript
-    --8<--- "nodejs/examples/embedding.ts:imports"
-    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
+    --8<--- "nodejs/examples/embedding.test.ts:imports"
+    --8<--- "nodejs/examples/embedding.test.ts:openai_embeddings"
    ```

 === "Rust"
@@ -121,12 +121,10 @@ class Words(LanceModel):
    vector: Vector(func.ndims()) = func.VectorField()

 table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
+table.add([
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+])

 query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
--- a/docs/src/guides/tables.md
+++ b/docs/src/guides/tables.md
@@ -85,13 +85,13 @@ Initialize a LanceDB connection and create a table


        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_table"
        ```

        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use `apache-arrow` to declare a schema

        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table_with_schema"
+        --8<-- "nodejs/examples/basic.test.ts:create_table_with_schema"
        ```

        !!! info "Note"
@@ -100,14 +100,14 @@ Initialize a LanceDB connection and create a table
            passed in will NOT be appended to the table in that case.

        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table_exists_ok"
+        --8<-- "nodejs/examples/basic.test.ts:create_table_exists_ok"
        ```

        Sometimes you want to make sure that you start fresh. If you want to
        overwrite the table, you can pass in mode: "overwrite" to the createTable function.

        ```ts
-        --8<-- "nodejs/examples/basic.ts:create_table_overwrite"
+        --8<-- "nodejs/examples/basic.test.ts:create_table_overwrite"
        ```

    === "vectordb (deprecated)"
@@ -227,7 +227,7 @@ LanceDB supports float16 data type!
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_f16_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_f16_table"
        ```

    === "vectordb (deprecated)"
@@ -455,7 +455,7 @@ You can create an empty table for scenarios where you want to add data to the ta
    === "@lancedb/lancedb"

        ```typescript
-        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        --8<-- "nodejs/examples/basic.test.ts:create_empty_table"
        ```

    === "vectordb (deprecated)"
--- a/docs/src/notebooks/hybrid_search.ipynb
+++ b/docs/src/notebooks/hybrid_search.ipynb
--- a/docs/src/reranking/voyageai.md
+++ b/docs/src/reranking/voyageai.md
@@ -0,0 +1,77 @@
+# Voyage AI Reranker
+
+Voyage AI provides cutting-edge embedding and rerankers.
+
+This re-ranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this re-ranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+
+
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+
+```python
+import numpy
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import VoyageAIReranker
+
+embedder = get_registry().get("sentence-transformers").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = VoyageAIReranker(model_name="rerank-2")
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `None` | The name of the reranker model to use. Available models are: rerank-2, rerank-2-lite |
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
+| `api_key` | `str` | `None` | The API key for the Voyage AI API. If not provided, the `VOYAGE_API_KEY` environment variable is used. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `truncation` | `bool` | `None` | Whether to truncate the input to satisfy the "context length limit" on the query and the documents. |
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
--- a/docs/src/search.md
+++ b/docs/src/search.md
@@ -58,9 +58,9 @@ db.create_table("my_vectors", data=data)
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/search.ts:import"
+        --8<-- "nodejs/examples/search.test.ts:import"

-        --8<-- "nodejs/examples/search.ts:search1"
+        --8<-- "nodejs/examples/search.test.ts:search1"
        ```


@@ -89,7 +89,7 @@ By default, `l2` will be used as metric type. You can specify the metric type as
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/search.ts:search2"
+        --8<-- "nodejs/examples/search.test.ts:search2"
        ```

    === "vectordb (deprecated)"
--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -49,7 +49,7 @@ const tbl = await db.createTable('myVectors', data)
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/filtering.ts:search"
+        --8<-- "nodejs/examples/filtering.test.ts:search"
        ```

    === "vectordb (deprecated)"
@@ -91,7 +91,7 @@ For example, the following filter string is acceptable:
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/filtering.ts:vec_search"
+        --8<-- "nodejs/examples/filtering.test.ts:vec_search"
        ```

    === "vectordb (deprecated)"
@@ -169,7 +169,7 @@ You can also filter your data without search.
    === "@lancedb/lancedb"

        ```ts
-        --8<-- "nodejs/examples/filtering.ts:sql_search"
+        --8<-- "nodejs/examples/filtering.test.ts:sql_search"
        ```

    === "vectordb (deprecated)"
--- a/docs/test/md_testing.py
+++ b/docs/test/md_testing.py
@@ -22,7 +22,8 @@ excluded_globs = [
    "../src/embeddings/available_embedding_models/text_embedding_functions/*.md",
    "../src/embeddings/available_embedding_models/multimodal_embedding_functions/*.md",
    "../src/rag/*.md",
-    "../src/rag/advanced_techniques/*.md"
+    "../src/rag/advanced_techniques/*.md",
+    "../src/cloud/*.md"


 ]
--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -8,7 +8,7 @@
    <parent>
        <groupId>com.lancedb</groupId>
        <artifactId>lancedb-parent</artifactId>
-        <version>0.12.0-final.0</version>
+        <version>0.13.0-beta.1</version>
        <relativePath>../pom.xml</relativePath>
    </parent>

--- a/java/pom.xml
+++ b/java/pom.xml
@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.12.0-final.0</version>
+    <version>0.13.0-beta.1</version>
    <packaging>pom</packaging>

    <name>LanceDB Parent</name>
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "vectordb",
-  "version": "0.12.0",
+  "version": "0.13.0-beta.1",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "vectordb",
-      "version": "0.12.0",
+      "version": "0.13.0-beta.1",
      "cpu": [
        "x64",
        "arm64"
@@ -52,11 +52,12 @@
        "uuid": "^9.0.0"
      },
      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.12.0",
-        "@lancedb/vectordb-darwin-x64": "0.12.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.12.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.12.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.12.0"
+        "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
+        "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
+        "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.13.0-beta.1",
+        "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1"
      },
      "peerDependencies": {
        "@apache-arrow/ts": "^14.0.2",
@@ -327,9 +328,9 @@
      }
    },
    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.12.0.tgz",
-      "integrity": "sha512-9X6UyP/ozHkv39YZ8DWh82m3aeQmUtrVDNuRe3o8has6dJyD/qPYukI8Zked4q8J+86/lgQbr4f+WW2V4Dfc1g==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.13.0-beta.1.tgz",
+      "integrity": "sha512-beOrf6selCzzhLgDG8Nibma4nO/CSnA1wUKRmlJHEPtGcg7PW18z6MP/nfwQMpMR/FLRfTo8pPTbpzss47MiQQ==",
      "cpu": [
        "arm64"
      ],
@@ -339,9 +340,9 @@
      ]
    },
    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.12.0.tgz",
-      "integrity": "sha512-zG+//P3BBpmOiLR+dop68T9AFNxazWlSLF8yVdAtvsqjRzcrrMLR//rIrRcbPHxu8gvvLrMDoDZT+AHd2rElyQ==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.13.0-beta.1.tgz",
+      "integrity": "sha512-YdraGRF/RbJRkKh0v3xT03LUhq47T2GtCvJ5gZp8wKlh4pHa8LuhLU0DIdvmG/DT5vuQA+td8HDkBm/e3EOdNg==",
      "cpu": [
        "x64"
      ],
@@ -351,9 +352,9 @@
      ]
    },
    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.12.0.tgz",
-      "integrity": "sha512-5RiJkcZEdMkK5WUfkV+HVFnJaAergfSiLNgUwJaovEEX8yVChkhrdZFSUj1o/k2k6Ix9mQq+xfIUF+aGN/XnDQ==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.13.0-beta.1.tgz",
+      "integrity": "sha512-Pp0O/uhEqof1oLaWrNbv+Ym+q8kBkiCqaA5+2eAZ6a3e9U+Ozkvb0FQrHuyi9adJ5wKQ4NabyQE9BMf2bYpOnQ==",
      "cpu": [
        "arm64"
      ],
@@ -363,9 +364,9 @@
      ]
    },
    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.12.0.tgz",
-      "integrity": "sha512-JFulRNBHLF0TyE0tThaAB9T7CM3zLquPsBF6oA9b1stVdXbEqVqLMltjem0tqfj30zEoEbAKDPpEKII4CPQMTA==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.13.0-beta.1.tgz",
+      "integrity": "sha512-y8nxOye4egfWF5FGED9EfkmZ1O5HnRLU4a61B8m5JSpkivO9v2epTcbYN0yt/7ZFCgtqMfJ8VW4Mi7qQcz3KDA==",
      "cpu": [
        "x64"
      ],
@@ -375,9 +376,9 @@
      ]
    },
    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.12.0.tgz",
-      "integrity": "sha512-T3s/RzB5dvXBqU3qmS6zyHhF0RHS2sSs81zKzYQy2R2nEVPbnwutFSsdA1wEqEXZlr8uTD9nLbkKJKqRNTXVEg==",
+      "version": "0.13.0-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.13.0-beta.1.tgz",
+      "integrity": "sha512-STMDP9dp0TBLkB3ro+16pKcGy6bmbhRuEZZZ1Tp5P75yTPeVh4zIgWkidMdU1qBbEYM7xacnsp9QAwgLnMU/Ow==",
      "cpu": [
        "x64"
      ],
--- a/node/package.json
+++ b/node/package.json
@@ -1,6 +1,6 @@
 {
  "name": "vectordb",
-  "version": "0.12.0",
+  "version": "0.13.0-beta.1",
  "description": " Serverless, low-latency vector database for AI applications",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
@@ -84,14 +84,16 @@
      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
-      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
+      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc",
+      "aarch64-pc-windows-msvc": "@lancedb/vectordb-win32-arm64-msvc"
    }
  },
  "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.12.0",
-    "@lancedb/vectordb-darwin-x64": "0.12.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.12.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.12.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.12.0"
+    "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
+    "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.13.0-beta.1"
  }
 }
--- a/nodejs/Cargo.toml
+++ b/nodejs/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.12.0"
+version = "0.13.0-beta.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -18,7 +18,7 @@ futures.workspace = true
 lancedb = { path = "../rust/lancedb", features = ["remote"] }
 napi = { version = "2.16.8", default-features = false, features = [
    "napi9",
-    "async",
+    "async"
 ] }
 napi-derive = "2.16.4"
 # Prevent dynamic linking of lzma, which comes from datafusion
--- a/nodejs/biome.json
+++ b/nodejs/biome.json
@@ -9,7 +9,8 @@
      "**/native.js",
      "**/native.d.ts",
      "**/npm/**/*",
-      "**/.vscode/**"
+      "**/.vscode/**",
+      "./examples/*"
    ]
  },
  "formatter": {
--- a/nodejs/examples/ann_indexes.test.ts
+++ b/nodejs/examples/ann_indexes.test.ts
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+import { VectorQuery } from "@lancedb/lancedb";
+// --8<-- [end:import]
+import { withTempDirectory } from "./util.ts";
+
+test("ann index examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:ingest]
+    const db = await lancedb.connect(databaseDir);
+
+    const data = Array.from({ length: 5_000 }, (_, i) => ({
+      vector: Array(128).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    }));
+
+    const table = await db.createTable("my_vectors", data, {
+      mode: "overwrite",
+    });
+    await table.createIndex("vector", {
+      config: lancedb.Index.ivfPq({
+        numPartitions: 10,
+        numSubVectors: 16,
+      }),
+    });
+    // --8<-- [end:ingest]
+
+    // --8<-- [start:search1]
+    const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;
+    const results1 = await search.nprobes(20).refineFactor(10).toArray();
+    // --8<-- [end:search1]
+    expect(results1.length).toBe(2);
+
+    // --8<-- [start:search2]
+    const results2 = await table
+      .search(Array(128).fill(1.2))
+      .where("id != '1141'")
+      .limit(2)
+      .toArray();
+    // --8<-- [end:search2]
+    expect(results2.length).toBe(2);
+
+    // --8<-- [start:search3]
+    const results3 = await table
+      .search(Array(128).fill(1.2))
+      .select(["id"])
+      .limit(2)
+      .toArray();
+    // --8<-- [end:search3]
+    expect(results3.length).toBe(2);
+  });
+}, 100_000);
--- a/nodejs/examples/ann_indexes.ts
+++ b/nodejs/examples/ann_indexes.ts
@@ -1,49 +0,0 @@
-// --8<-- [start:import]
-import * as lancedb from "@lancedb/lancedb";
-// --8<-- [end:import]
-
-// --8<-- [start:ingest]
-const db = await lancedb.connect("/tmp/lancedb/");
-
-const data = Array.from({ length: 10_000 }, (_, i) => ({
-  vector: Array(1536).fill(i),
-  id: `${i}`,
-  content: "",
-  longId: `${i}`,
-}));
-
-const table = await db.createTable("my_vectors", data, { mode: "overwrite" });
-await table.createIndex("vector", {
-  config: lancedb.Index.ivfPq({
-    numPartitions: 16,
-    numSubVectors: 48,
-  }),
-});
-// --8<-- [end:ingest]
-
-// --8<-- [start:search1]
-const _results1 = await table
-  .search(Array(1536).fill(1.2))
-  .limit(2)
-  .nprobes(20)
-  .refineFactor(10)
-  .toArray();
-// --8<-- [end:search1]
-
-// --8<-- [start:search2]
-const _results2 = await table
-  .search(Array(1536).fill(1.2))
-  .where("id != '1141'")
-  .limit(2)
-  .toArray();
-// --8<-- [end:search2]
-
-// --8<-- [start:search3]
-const _results3 = await table
-  .search(Array(1536).fill(1.2))
-  .select(["id"])
-  .limit(2)
-  .toArray();
-// --8<-- [end:search3]
-
-console.log("Ann indexes: done");
--- a/nodejs/examples/basic.test.ts
+++ b/nodejs/examples/basic.test.ts
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import * as arrow from "apache-arrow";
+import {
+  Field,
+  FixedSizeList,
+  Float16,
+  Int32,
+  Schema,
+  Utf8,
+} from "apache-arrow";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+test("basic table examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:connect]
+    const db = await lancedb.connect(databaseDir);
+    // --8<-- [end:connect]
+    {
+      // --8<-- [start:create_table]
+      const _tbl = await db.createTable(
+        "myTable",
+        [
+          { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+          { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+        ],
+        { mode: "overwrite" },
+      );
+      // --8<-- [end:create_table]
+
+      const data = [
+        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+      ];
+
+      {
+        // --8<-- [start:create_table_exists_ok]
+        const tbl = await db.createTable("myTable", data, {
+          existOk: true,
+        });
+        // --8<-- [end:create_table_exists_ok]
+        expect(await tbl.countRows()).toBe(2);
+      }
+      {
+        // --8<-- [start:create_table_overwrite]
+        const tbl = await db.createTable("myTable", data, {
+          mode: "overwrite",
+        });
+        // --8<-- [end:create_table_overwrite]
+        expect(await tbl.countRows()).toBe(2);
+      }
+    }
+
+    await db.dropTable("myTable");
+
+    {
+      // --8<-- [start:create_table_with_schema]
+      const schema = new arrow.Schema([
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(
+            2,
+            new arrow.Field("item", new arrow.Float32(), true),
+          ),
+        ),
+        new arrow.Field("item", new arrow.Utf8(), true),
+        new arrow.Field("price", new arrow.Float32(), true),
+      ]);
+      const data = [
+        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+      ];
+      const tbl = await db.createTable("myTable", data, {
+        schema,
+      });
+      // --8<-- [end:create_table_with_schema]
+      expect(await tbl.countRows()).toBe(2);
+    }
+
+    {
+      // --8<-- [start:create_empty_table]
+
+      const schema = new arrow.Schema([
+        new arrow.Field("id", new arrow.Int32()),
+        new arrow.Field("name", new arrow.Utf8()),
+      ]);
+
+      const emptyTbl = await db.createEmptyTable("empty_table", schema);
+      // --8<-- [end:create_empty_table]
+      expect(await emptyTbl.countRows()).toBe(0);
+    }
+    {
+      // --8<-- [start:open_table]
+      const _tbl = await db.openTable("myTable");
+      // --8<-- [end:open_table]
+    }
+
+    {
+      // --8<-- [start:table_names]
+      const tableNames = await db.tableNames();
+      // --8<-- [end:table_names]
+      expect(tableNames).toEqual(["empty_table", "myTable"]);
+    }
+
+    const tbl = await db.openTable("myTable");
+    {
+      // --8<-- [start:add_data]
+      const data = [
+        { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
+        { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
+      ];
+      await tbl.add(data);
+      // --8<-- [end:add_data]
+    }
+    {
+      // --8<-- [start:vector_search]
+      const res = await tbl.search([100, 100]).limit(2).toArray();
+      // --8<-- [end:vector_search]
+      expect(res.length).toBe(2);
+    }
+    {
+      const data = Array.from({ length: 1000 })
+        .fill(null)
+        .map(() => ({
+          vector: [Math.random(), Math.random()],
+          item: "autogen",
+          price: Math.round(Math.random() * 100),
+        }));
+
+      await tbl.add(data);
+    }
+
+    // --8<-- [start:create_index]
+    await tbl.createIndex("vector");
+    // --8<-- [end:create_index]
+
+    // --8<-- [start:delete_rows]
+    await tbl.delete('item = "fizz"');
+    // --8<-- [end:delete_rows]
+
+    // --8<-- [start:drop_table]
+    await db.dropTable("myTable");
+    // --8<-- [end:drop_table]
+    await db.dropTable("empty_table");
+
+    {
+      // --8<-- [start:create_f16_table]
+      const db = await lancedb.connect(databaseDir);
+      const dim = 16;
+      const total = 10;
+      const f16Schema = new Schema([
+        new Field("id", new Int32()),
+        new Field(
+          "vector",
+          new FixedSizeList(dim, new Field("item", new Float16(), true)),
+          false,
+        ),
+      ]);
+      const data = lancedb.makeArrowTable(
+        Array.from(Array(total), (_, i) => ({
+          id: i,
+          vector: Array.from(Array(dim), Math.random),
+        })),
+        { schema: f16Schema },
+      );
+      const _table = await db.createTable("f16_tbl", data);
+      // --8<-- [end:create_f16_table]
+      await db.dropTable("f16_tbl");
+    }
+  });
+});
--- a/nodejs/examples/basic.ts
+++ b/nodejs/examples/basic.ts
@@ -1,162 +0,0 @@
-// --8<--  [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import * as arrow from "apache-arrow";
-import {
-  Field,
-  FixedSizeList,
-  Float16,
-  Int32,
-  Schema,
-  Utf8,
-} from "apache-arrow";
-
-// --8<-- [end:imports]
-
-// --8<-- [start:connect]
-const uri = "/tmp/lancedb/";
-const db = await lancedb.connect(uri);
-// --8<-- [end:connect]
-{
-  // --8<-- [start:create_table]
-  const tbl = await db.createTable(
-    "myTable",
-    [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ],
-    { mode: "overwrite" },
-  );
-  // --8<-- [end:create_table]
-
-  const data = [
-    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-  ];
-
-  {
-    // --8<-- [start:create_table_exists_ok]
-    const tbl = await db.createTable("myTable", data, {
-      existsOk: true,
-    });
-    // --8<-- [end:create_table_exists_ok]
-  }
-  {
-    // --8<-- [start:create_table_overwrite]
-    const _tbl = await db.createTable("myTable", data, {
-      mode: "overwrite",
-    });
-    // --8<-- [end:create_table_overwrite]
-  }
-}
-
-{
-  // --8<-- [start:create_table_with_schema]
-  const schema = new arrow.Schema([
-    new arrow.Field(
-      "vector",
-      new arrow.FixedSizeList(
-        2,
-        new arrow.Field("item", new arrow.Float32(), true),
-      ),
-    ),
-    new arrow.Field("item", new arrow.Utf8(), true),
-    new arrow.Field("price", new arrow.Float32(), true),
-  ]);
-  const data = [
-    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-  ];
-  const _tbl = await db.createTable("myTable", data, {
-    schema,
-  });
-  // --8<-- [end:create_table_with_schema]
-}
-
-{
-  // --8<-- [start:create_empty_table]
-
-  const schema = new arrow.Schema([
-    new arrow.Field("id", new arrow.Int32()),
-    new arrow.Field("name", new arrow.Utf8()),
-  ]);
-
-  const empty_tbl = await db.createEmptyTable("empty_table", schema);
-  // --8<-- [end:create_empty_table]
-}
-{
-  // --8<-- [start:open_table]
-  const _tbl = await db.openTable("myTable");
-  // --8<-- [end:open_table]
-}
-
-{
-  // --8<-- [start:table_names]
-  const tableNames = await db.tableNames();
-  console.log(tableNames);
-  // --8<-- [end:table_names]
-}
-
-const tbl = await db.openTable("myTable");
-{
-  // --8<-- [start:add_data]
-  const data = [
-    { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
-    { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
-  ];
-  await tbl.add(data);
-  // --8<-- [end:add_data]
-}
-{
-  // --8<-- [start:vector_search]
-  const _res = tbl.search([100, 100]).limit(2).toArray();
-  // --8<-- [end:vector_search]
-}
-{
-  const data = Array.from({ length: 1000 })
-    .fill(null)
-    .map(() => ({
-      vector: [Math.random(), Math.random()],
-      item: "autogen",
-      price: Math.round(Math.random() * 100),
-    }));
-
-  await tbl.add(data);
-}
-
-// --8<-- [start:create_index]
-await tbl.createIndex("vector");
-// --8<-- [end:create_index]
-
-// --8<-- [start:delete_rows]
-await tbl.delete('item = "fizz"');
-// --8<-- [end:delete_rows]
-
-// --8<-- [start:drop_table]
-await db.dropTable("myTable");
-// --8<-- [end:drop_table]
-await db.dropTable("empty_table");
-
-{
-  // --8<-- [start:create_f16_table]
-  const db = await lancedb.connect("/tmp/lancedb");
-  const dim = 16;
-  const total = 10;
-  const f16Schema = new Schema([
-    new Field("id", new Int32()),
-    new Field(
-      "vector",
-      new FixedSizeList(dim, new Field("item", new Float16(), true)),
-      false,
-    ),
-  ]);
-  const data = lancedb.makeArrowTable(
-    Array.from(Array(total), (_, i) => ({
-      id: i,
-      vector: Array.from(Array(dim), Math.random),
-    })),
-    { schema: f16Schema },
-  );
-  const _table = await db.createTable("f16_tbl", data);
-  // --8<-- [end:create_f16_table]
-  await db.dropTable("f16_tbl");
-}
--- a/nodejs/examples/cloud.test.ts
+++ b/nodejs/examples/cloud.test.ts
@@ -0,0 +1,230 @@
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:imports]
+
+// --8<-- [start:generate_data]
+function genData(numRows: number, numVectorDim: number): any[] {
+    const data = [];
+    for (let i = 0; i < numRows; i++) {
+      const vector = [];
+      for (let j = 0; j < numVectorDim; j++) {
+        vector.push(i + j * 0.1);
+      }
+      data.push({
+        id: i,
+        name: `name_${i}`,
+        vector,
+      });
+    }
+    return data;
+}
+// --8<-- [end:generate_data]
+
+test("cloud quickstart", async () => {
+    {
+        // --8<-- [start:connect]
+        const db = await lancedb.connect({
+            uri: "db://your-project-slug",
+            apiKey: "your-api-key",
+            region: "your-cloud-region",
+        });
+        // --8<-- [end:connect]
+        // --8<-- [start:create_table]
+        const tableName = "myTable"
+        const data = genData(5000, 1536)
+        const table = await db.createTable(tableName, data);
+        // --8<-- [end:create_table]
+        // --8<-- [start:create_index_search]
+        // create a vector index
+        await table.createIndex({
+            column: "vector",
+            metric_type: lancedb.MetricType.Cosine,
+            type: "ivf_pq",
+        });
+        const result = await table.search([0.01, 0.02])
+            .select(["vector", "item"])
+            .limit(1)
+            .execute();
+        // --8<-- [end:create_index_search]
+        // --8<-- [start:drop_table]
+        await db.dropTable(tableName);
+        // --8<-- [end:drop_table]
+    }
+});
+
+test("ingest data", async () => {
+    // --8<-- [start:ingest_data]
+    import { Schema, Field, Float32, FixedSizeList, Utf8 } from "apache-arrow";
+
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const data = [
+        { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+        { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+        { vector: [10.2, 100.8], item: "baz", price: 30.0},
+        { vector: [1.4, 9.5], item: "fred", price: 40.0},
+    ]
+    // create an empty table with schema
+    const schema = new Schema([
+        new Field(
+        "vector",
+        new FixedSizeList(2, new Field("float32", new Float32())),
+        ),
+        new Field("item", new Utf8()),
+        new Field("price", new Float32()),
+    ]);
+    const tableName = "myTable";
+    const table = await db.createTable({
+        name: tableName,
+        schema,
+    });
+    await table.add(data);
+    // --8<-- [end:ingest_data]
+});
+
+test("update data", async () => {
+    // --8<-- [start:connect_db_and_open_table]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+    const tableName = "myTable"
+    const table = await db.openTable(tableName);
+    // --8<-- [end:connect_db_and_open_table]
+    // --8<-- [start:update_data]
+    await table.update({
+        where: "price < 20.0",
+        values: { vector: [2, 2], item: "foo-updated" },
+    });
+    // --8<-- [end:update_data]
+    // --8<-- [start:merge_insert]
+      let newData = [
+        {vector: [1, 1], item: 'foo-updated', price: 50.0}
+      ];
+      // upsert
+      await table.mergeInsert("item", newData, {
+        whenMatchedUpdateAll: true,
+        whenNotMatchedInsertAll: true,
+      });
+      // --8<-- [end:merge_insert]
+      // --8<-- [start:delete_data]
+      // delete data
+      const predicate = "price = 30.0";
+      await table.delete(predicate);
+      // --8<-- [end:delete_data]
+});
+
+test("create index", async () => {
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const tableName = "myTable";
+    const table = await db.openTable(tableName);
+    // --8<-- [start:create_index]
+    // the vector column only needs to be specified when there are 
+    // multiple vector columns or the column is not named as "vector"
+    // L2 is used as the default distance metric
+    await table.createIndex({
+            column: "vector",
+            metric_type: lancedb.MetricType.Cosine,
+        });
+
+    // --8<-- [end:create_index]
+    // --8<-- [start:create_scalar_index]
+    await table.createScalarIndex("item");
+    // --8<-- [end:create_scalar_index]
+    // --8<-- [start:create_fts_index]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const tableName = "myTable"
+    const data = [
+        { vector: [3.1, 4.1], text: "Frodo was a happy puppy" },
+        { vector: [5.9, 26.5], text: "There are several kittens playing" },
+    ];
+    const table = createTable(tableName, data);
+    await table.createIndex("text", {
+        config: lancedb.Index.fts(),
+    });
+    // --8<-- [end:create_fts_index]
+});
+
+test("vector search", async () => {
+    // --8<-- [start:vector_search]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const tableName = "myTable"
+    const table = await db.openTable(tableName);
+    const result = await table.search([0.4, 1.4])
+        .where("price > 10.0")
+        .prefilter(true)
+        .select(["item", "vector"])
+        .limit(2)
+        .execute();
+    // --8<-- [end:vector_search]
+});
+
+test("full-text search", async () => {
+    // --8<-- [start:full_text_search]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+
+    const data = [
+    { vector: [3.1, 4.1], text: "Frodo was a happy puppy" },
+    { vector: [5.9, 26.5], text: "There are several kittens playing" },
+    ];
+    const tableName = "myTable"
+    const table = await db.createTable(tableName, data);
+    await table.createIndex("text", {
+        config: lancedb.Index.fts(),
+    });
+
+    await tableName
+        .search("puppy", queryType="fts")
+        .select(["text"])
+        .limit(10)
+        .toArray();
+    // --8<-- [end:full_text_search]
+});
+
+test("metadata filtering", async () => {
+    // --8<-- [start:filtering]
+    const db = await lancedb.connect({
+        uri: "db://your-project-slug",
+        apiKey: "your-api-key",
+        region: "us-east-1"
+    });
+    const tableName = "myTable"
+    const table = await db.openTable(tableName);
+    await table
+        .search(Array(2).fill(0.1))
+        .where("(item IN ('foo', 'bar')) AND (price > 10.0)")
+        .postfilter()
+        .toArray();
+    // --8<-- [end:filtering]
+    // --8<-- [start:sql_filtering]
+    await table
+        .search(Array(2).fill(0.1))
+        .where("(item IN ('foo', 'bar')) AND (price > 10.0)")
+        .postfilter()
+        .toArray();
+    // --8<-- [end:sql_filtering]
+});
--- a/nodejs/examples/custom_embedding_function.test.ts
+++ b/nodejs/examples/custom_embedding_function.test.ts
@@ -0,0 +1,76 @@
+import { FeatureExtractionPipeline, pipeline } from "@huggingface/transformers";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import {
+  LanceSchema,
+  TextEmbeddingFunction,
+  getRegistry,
+  register,
+} from "@lancedb/lancedb/embedding";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+// --8<-- [start:embedding_impl]
+@register("sentence-transformers")
+class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
+  name = "Xenova/all-miniLM-L6-v2";
+  #ndims!: number;
+  extractor!: FeatureExtractionPipeline;
+
+  async init() {
+    this.extractor = await pipeline("feature-extraction", this.name, {
+      dtype: "fp32",
+    });
+    this.#ndims = await this.generateEmbeddings(["hello"]).then(
+      (e) => e[0].length,
+    );
+  }
+
+  ndims() {
+    return this.#ndims;
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+    };
+  }
+  async generateEmbeddings(texts: string[]) {
+    const output = await this.extractor(texts, {
+      pooling: "mean",
+      normalize: true,
+    });
+    return output.tolist();
+  }
+}
+// -8<-- [end:embedding_impl]
+
+test("Registry examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:call_custom_function]
+    const registry = getRegistry();
+
+    const sentenceTransformer = await registry
+      .get<SentenceTransformersEmbeddings>("sentence-transformers")!
+      .create();
+
+    const schema = LanceSchema({
+      vector: sentenceTransformer.vectorField(),
+      text: sentenceTransformer.sourceField(),
+    });
+
+    const db = await lancedb.connect(databaseDir);
+    const table = await db.createEmptyTable("table", schema, {
+      mode: "overwrite",
+    });
+
+    await table.add([{ text: "hello" }, { text: "world" }]);
+
+    const results = await table.search("greeting").limit(1).toArray();
+    // -8<-- [end:call_custom_function]
+    expect(results.length).toBe(1);
+  });
+}, 100_000);
--- a/nodejs/examples/custom_embedding_function.ts
+++ b/nodejs/examples/custom_embedding_function.ts
@@ -1,64 +0,0 @@
-// --8<-- [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import {
-  LanceSchema,
-  TextEmbeddingFunction,
-  getRegistry,
-  register,
-} from "@lancedb/lancedb/embedding";
-import { pipeline } from "@xenova/transformers";
-// --8<-- [end:imports]
-
-// --8<-- [start:embedding_impl]
-@register("sentence-transformers")
-class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
-  name = "Xenova/all-miniLM-L6-v2";
-  #ndims!: number;
-  extractor: any;
-
-  async init() {
-    this.extractor = await pipeline("feature-extraction", this.name);
-    this.#ndims = await this.generateEmbeddings(["hello"]).then(
-      (e) => e[0].length,
-    );
-  }
-
-  ndims() {
-    return this.#ndims;
-  }
-
-  toJSON() {
-    return {
-      name: this.name,
-    };
-  }
-  async generateEmbeddings(texts: string[]) {
-    const output = await this.extractor(texts, {
-      pooling: "mean",
-      normalize: true,
-    });
-    return output.tolist();
-  }
-}
-// -8<-- [end:embedding_impl]
-
-// --8<-- [start:call_custom_function]
-const registry = getRegistry();
-
-const sentenceTransformer = await registry
-  .get<SentenceTransformersEmbeddings>("sentence-transformers")!
-  .create();
-
-const schema = LanceSchema({
-  vector: sentenceTransformer.vectorField(),
-  text: sentenceTransformer.sourceField(),
-});
-
-const db = await lancedb.connect("/tmp/db");
-const table = await db.createEmptyTable("table", schema, { mode: "overwrite" });
-
-await table.add([{ text: "hello" }, { text: "world" }]);
-
-const results = await table.search("greeting").limit(1).toArray();
-console.log(results[0].text);
-// -8<-- [end:call_custom_function]
--- a/nodejs/examples/embedding.test.ts
+++ b/nodejs/examples/embedding.test.ts
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import "@lancedb/lancedb/embedding/openai";
+import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { type Float, Float32, Utf8 } from "apache-arrow";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test;
+
+openAiTest("openai embeddings", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:openai_embeddings]
+    const db = await lancedb.connect(databaseDir);
+    const func = getRegistry()
+      .get("openai")
+      ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+    const wordsSchema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+    const tbl = await db.createEmptyTable("words", wordsSchema, {
+      mode: "overwrite",
+    });
+    await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+    const query = "greetings";
+    const actual = (await tbl.search(query).limit(1).toArray())[0];
+    // --8<-- [end:openai_embeddings]
+    expect(actual).toHaveProperty("text");
+  });
+});
+
+test("custom embedding function", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:embedding_function]
+    const db = await lancedb.connect(databaseDir);
+
+    @register("my_embedding")
+    class MyEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        // This is a placeholder for a real embedding function
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        // This is a placeholder for a real embedding function
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+
+    const func = new MyEmbeddingFunction();
+
+    const data = [{ text: "pepperoni" }, { text: "pineapple" }];
+
+    // Option 1: manually specify the embedding function
+    const table = await db.createTable("vectors", data, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+        vectorColumn: "vector",
+      },
+      mode: "overwrite",
+    });
+
+    // Option 2: provide the embedding function through a schema
+
+    const schema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const table2 = await db.createTable("vectors2", data, {
+      schema,
+      mode: "overwrite",
+    });
+    // --8<-- [end:embedding_function]
+    expect(await table.countRows()).toBe(2);
+    expect(await table2.countRows()).toBe(2);
+  });
+});
--- a/nodejs/examples/embedding.ts
+++ b/nodejs/examples/embedding.ts
@@ -1,83 +0,0 @@
-// --8<-- [start:imports]
-import * as lancedb from "@lancedb/lancedb";
-import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
-import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
-import { type Float, Float32, Utf8 } from "apache-arrow";
-// --8<-- [end:imports]
-
-{
-  // --8<-- [start:openai_embeddings]
-
-  const db = await lancedb.connect("/tmp/db");
-  const func = getRegistry()
-    .get("openai")
-    ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
-
-  const wordsSchema = LanceSchema({
-    text: func.sourceField(new Utf8()),
-    vector: func.vectorField(),
-  });
-  const tbl = await db.createEmptyTable("words", wordsSchema, {
-    mode: "overwrite",
-  });
-  await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
-
-  const query = "greetings";
-  const actual = (await (await tbl.search(query)).limit(1).toArray())[0];
-
-  // --8<-- [end:openai_embeddings]
-  console.log("result = ", actual.text);
-}
-
-{
-  // --8<-- [start:embedding_function]
-  const db = await lancedb.connect("/tmp/db");
-
-  @register("my_embedding")
-  class MyEmbeddingFunction extends EmbeddingFunction<string> {
-    toJSON(): object {
-      return {};
-    }
-    ndims() {
-      return 3;
-    }
-    embeddingDataType(): Float {
-      return new Float32();
-    }
-    async computeQueryEmbeddings(_data: string) {
-      // This is a placeholder for a real embedding function
-      return [1, 2, 3];
-    }
-    async computeSourceEmbeddings(data: string[]) {
-      // This is a placeholder for a real embedding function
-      return Array.from({ length: data.length }).fill([1, 2, 3]) as number[][];
-    }
-  }
-
-  const func = new MyEmbeddingFunction();
-
-  const data = [{ text: "pepperoni" }, { text: "pineapple" }];
-
-  // Option 1: manually specify the embedding function
-  const table = await db.createTable("vectors", data, {
-    embeddingFunction: {
-      function: func,
-      sourceColumn: "text",
-      vectorColumn: "vector",
-    },
-    mode: "overwrite",
-  });
-
-  // Option 2: provide the embedding function through a schema
-
-  const schema = LanceSchema({
-    text: func.sourceField(new Utf8()),
-    vector: func.vectorField(),
-  });
-
-  const table2 = await db.createTable("vectors2", data, {
-    schema,
-    mode: "overwrite",
-  });
-  // --8<-- [end:embedding_function]
-}
--- a/nodejs/examples/filtering.test.ts
+++ b/nodejs/examples/filtering.test.ts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("filtering examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    const data = Array.from({ length: 10_000 }, (_, i) => ({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+    }));
+
+    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+    // --8<-- [start:search]
+    const _result = await tbl
+      .search(Array(1536).fill(0.5))
+      .limit(1)
+      .where("id = 10")
+      .toArray();
+    // --8<-- [end:search]
+
+    // --8<-- [start:vec_search]
+    const result = await (
+      tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery
+    )
+      .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+      .postfilter()
+      .toArray();
+    // --8<-- [end:vec_search]
+    expect(result.length).toBe(0);
+
+    // --8<-- [start:sql_search]
+    await tbl.query().where("id = 10").limit(10).toArray();
+    // --8<-- [end:sql_search]
+  });
+});
--- a/nodejs/examples/filtering.ts
+++ b/nodejs/examples/filtering.ts
@@ -1,34 +0,0 @@
-import * as lancedb from "@lancedb/lancedb";
-
-const db = await lancedb.connect("data/sample-lancedb");
-
-const data = Array.from({ length: 10_000 }, (_, i) => ({
-  vector: Array(1536).fill(i),
-  id: i,
-  item: `item ${i}`,
-  strId: `${i}`,
-}));
-
-const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
-
-// --8<-- [start:search]
-const _result = await tbl
-  .search(Array(1536).fill(0.5))
-  .limit(1)
-  .where("id = 10")
-  .toArray();
-// --8<-- [end:search]
-
-// --8<-- [start:vec_search]
-await tbl
-  .search(Array(1536).fill(0))
-  .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-  .postfilter()
-  .toArray();
-// --8<-- [end:vec_search]
-
-// --8<-- [start:sql_search]
-await tbl.query().where("id = 10").limit(10).toArray();
-// --8<-- [end:sql_search]
-
-console.log("SQL search: done");
--- a/nodejs/examples/full_text_search.test.ts
+++ b/nodejs/examples/full_text_search.test.ts
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    const words = [
+      "apple",
+      "banana",
+      "cherry",
+      "date",
+      "elderberry",
+      "fig",
+      "grape",
+    ];
+
+    const data = Array.from({ length: 10_000 }, (_, i) => ({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+      doc: words[i % words.length],
+    }));
+
+    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+    await tbl.createIndex("doc", {
+      config: lancedb.Index.fts(),
+    });
+
+    // --8<-- [start:full_text_search]
+    const result = await tbl
+      .query()
+      .nearestToText("apple")
+      .select(["id", "doc"])
+      .limit(10)
+      .toArray();
+    expect(result.length).toBe(10);
+    // --8<-- [end:full_text_search]
+  });
+});
--- a/nodejs/examples/full_text_search.ts
+++ b/nodejs/examples/full_text_search.ts
@@ -1,52 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import * as lancedb from "@lancedb/lancedb";
-
-const db = await lancedb.connect("data/sample-lancedb");
-
-const words = [
-  "apple",
-  "banana",
-  "cherry",
-  "date",
-  "elderberry",
-  "fig",
-  "grape",
-];
-
-const data = Array.from({ length: 10_000 }, (_, i) => ({
-  vector: Array(1536).fill(i),
-  id: i,
-  item: `item ${i}`,
-  strId: `${i}`,
-  doc: words[i % words.length],
-}));
-
-const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
-
-await tbl.createIndex("doc", {
-  config: lancedb.Index.fts(),
-});
-
-// --8<-- [start:full_text_search]
-let result = await tbl
-  .search("apple")
-  .select(["id", "doc"])
-  .limit(10)
-  .toArray();
-console.log(result);
-// --8<-- [end:full_text_search]
-
-console.log("SQL search: done");
--- a/nodejs/examples/jest.config.cjs
+++ b/nodejs/examples/jest.config.cjs
@@ -0,0 +1,6 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  testPathIgnorePatterns: ["./dist"],
+};
--- a/nodejs/examples/jsconfig.json
+++ b/nodejs/examples/jsconfig.json
@@ -1,27 +0,0 @@
-{
-  "compilerOptions": {
-    // Enable latest features
-    "lib": ["ESNext", "DOM"],
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-
-    // Bundler mode
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
-    "noEmit": true,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false
-  }
-}
--- a/nodejs/examples/package-lock.json
+++ b/nodejs/examples/package-lock.json
--- a/nodejs/examples/package.json
+++ b/nodejs/examples/package.json
@@ -5,24 +5,29 @@
  "main": "index.js",
  "type": "module",
  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
+    "//2": "--testEnvironment is needed to run jest with sentence-transformers",
+    "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
+    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
+    "lint": "biome check *.ts && biome format *.ts",
+    "lint-ci": "biome ci .",
+    "lint-fix": "biome check --write *.ts && npm run format",
+    "format": "biome format --write *.ts"
  },
  "author": "Lance Devs",
  "license": "Apache-2.0",
  "dependencies": {
-    "@lancedb/lancedb": "file:../",
-    "@xenova/transformers": "^2.17.2"
+    "@huggingface/transformers": "^3.0.2",
+    "@lancedb/lancedb": "file:../dist",
+    "openai": "^4.29.2",
+    "sharp": "^0.33.5"
  },
  "devDependencies": {
+    "@biomejs/biome": "^1.7.3",
+    "@jest/globals": "^29.7.0",
+    "jest": "^29.7.0",
+    "jest-environment-node-single-context": "^29.4.0",
+    "ts-jest": "^29.2.5",
    "typescript": "^5.5.4"
-  },
-  "compilerOptions": {
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleResolution": "Node",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
  }
 }
--- a/nodejs/examples/search.test.ts
+++ b/nodejs/examples/search.test.ts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+import { withTempDirectory } from "./util.ts";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    {
+      const db = await lancedb.connect(databaseDir);
+
+      const data = Array.from({ length: 10_000 }, (_, i) => ({
+        vector: Array(128).fill(i),
+        id: `${i}`,
+        content: "",
+        longId: `${i}`,
+      }));
+
+      await db.createTable("my_vectors", data);
+    }
+
+    // --8<-- [start:search1]
+    const db = await lancedb.connect(databaseDir);
+    const tbl = await db.openTable("my_vectors");
+
+    const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();
+    // --8<-- [end:search1]
+    expect(results1.length).toBe(10);
+
+    // --8<-- [start:search2]
+    const results2 = await (
+      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
+    )
+      .distanceType("cosine")
+      .limit(10)
+      .toArray();
+    // --8<-- [end:search2]
+    expect(results2.length).toBe(10);
+  });
+});
--- a/nodejs/examples/search.ts
+++ b/nodejs/examples/search.ts
@@ -1,38 +0,0 @@
-// --8<-- [end:import]
-import * as fs from "node:fs";
-// --8<-- [start:import]
-import * as lancedb from "@lancedb/lancedb";
-
-async function setup() {
-  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
-  const db = await lancedb.connect("data/sample-lancedb");
-
-  const data = Array.from({ length: 10_000 }, (_, i) => ({
-    vector: Array(1536).fill(i),
-    id: `${i}`,
-    content: "",
-    longId: `${i}`,
-  }));
-
-  await db.createTable("my_vectors", data);
-}
-
-await setup();
-
-// --8<-- [start:search1]
-const db = await lancedb.connect("data/sample-lancedb");
-const tbl = await db.openTable("my_vectors");
-
-const _results1 = await tbl.search(Array(1536).fill(1.2)).limit(10).toArray();
-// --8<-- [end:search1]
-
-// --8<-- [start:search2]
-const _results2 = await tbl
-  .search(Array(1536).fill(1.2))
-  .distanceType("cosine")
-  .limit(10)
-  .toArray();
-console.log(_results2);
-// --8<-- [end:search2]
-
-console.log("search: done");
--- a/nodejs/examples/sentence-transformers.js
+++ b/nodejs/examples/sentence-transformers.js
@@ -1,50 +0,0 @@
-import * as lancedb from "@lancedb/lancedb";
-
-import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
-import { Utf8 } from "apache-arrow";
-
-const db = await lancedb.connect("/tmp/db");
-const func = await getRegistry().get("huggingface").create();
-
-const facts = [
-  "Albert Einstein was a theoretical physicist.",
-  "The capital of France is Paris.",
-  "The Great Wall of China is one of the Seven Wonders of the World.",
-  "Python is a popular programming language.",
-  "Mount Everest is the highest mountain in the world.",
-  "Leonardo da Vinci painted the Mona Lisa.",
-  "Shakespeare wrote Hamlet.",
-  "The human body has 206 bones.",
-  "The speed of light is approximately 299,792 kilometers per second.",
-  "Water boils at 100 degrees Celsius.",
-  "The Earth orbits the Sun.",
-  "The Pyramids of Giza are located in Egypt.",
-  "Coffee is one of the most popular beverages in the world.",
-  "Tokyo is the capital city of Japan.",
-  "Photosynthesis is the process by which plants make their food.",
-  "The Pacific Ocean is the largest ocean on Earth.",
-  "Mozart was a prolific composer of classical music.",
-  "The Internet is a global network of computers.",
-  "Basketball is a sport played with a ball and a hoop.",
-  "The first computer virus was created in 1983.",
-  "Artificial neural networks are inspired by the human brain.",
-  "Deep learning is a subset of machine learning.",
-  "IBM's Watson won Jeopardy! in 2011.",
-  "The first computer programmer was Ada Lovelace.",
-  "The first chatbot was ELIZA, created in the 1960s.",
-].map((text) => ({ text }));
-
-const factsSchema = LanceSchema({
-  text: func.sourceField(new Utf8()),
-  vector: func.vectorField(),
-});
-
-const tbl = await db.createTable("facts", facts, {
-  mode: "overwrite",
-  schema: factsSchema,
-});
-
-const query = "How many bones are in the human body?";
-const actual = await tbl.search(query).limit(1).toArray();
-
-console.log("Answer: ", actual[0]["text"]);
--- a/nodejs/examples/sentence-transformers.test.ts
+++ b/nodejs/examples/sentence-transformers.test.ts
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import { withTempDirectory } from "./util.ts";
+
+import * as lancedb from "@lancedb/lancedb";
+import "@lancedb/lancedb/embedding/transformers";
+import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
+import { Utf8 } from "apache-arrow";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+    const func = await getRegistry().get("huggingface").create();
+
+    const facts = [
+      "Albert Einstein was a theoretical physicist.",
+      "The capital of France is Paris.",
+      "The Great Wall of China is one of the Seven Wonders of the World.",
+      "Python is a popular programming language.",
+      "Mount Everest is the highest mountain in the world.",
+      "Leonardo da Vinci painted the Mona Lisa.",
+      "Shakespeare wrote Hamlet.",
+      "The human body has 206 bones.",
+      "The speed of light is approximately 299,792 kilometers per second.",
+      "Water boils at 100 degrees Celsius.",
+      "The Earth orbits the Sun.",
+      "The Pyramids of Giza are located in Egypt.",
+      "Coffee is one of the most popular beverages in the world.",
+      "Tokyo is the capital city of Japan.",
+      "Photosynthesis is the process by which plants make their food.",
+      "The Pacific Ocean is the largest ocean on Earth.",
+      "Mozart was a prolific composer of classical music.",
+      "The Internet is a global network of computers.",
+      "Basketball is a sport played with a ball and a hoop.",
+      "The first computer virus was created in 1983.",
+      "Artificial neural networks are inspired by the human brain.",
+      "Deep learning is a subset of machine learning.",
+      "IBM's Watson won Jeopardy! in 2011.",
+      "The first computer programmer was Ada Lovelace.",
+      "The first chatbot was ELIZA, created in the 1960s.",
+    ].map((text) => ({ text }));
+
+    const factsSchema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const tbl = await db.createTable("facts", facts, {
+      mode: "overwrite",
+      schema: factsSchema,
+    });
+
+    const query = "How many bones are in the human body?";
+    const actual = await tbl.search(query).limit(1).toArray();
+
+    expect(actual[0]["text"]).toBe("The human body has 206 bones.");
+  });
+});
--- a/nodejs/examples/tsconfig.json
+++ b/nodejs/examples/tsconfig.json
@@ -0,0 +1,17 @@
+{
+  "include": ["*.test.ts"],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "NodeNext",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "NodeNext",
+    "allowImportingTsExtensions": true,
+    "emitDeclarationOnly": true
+  }
+}
--- a/nodejs/examples/util.ts
+++ b/nodejs/examples/util.ts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import * as fs from "fs";
+import { tmpdir } from "os";
+import * as path from "path";
+
+export async function withTempDirectory(
+  fn: (tempDir: string) => Promise<void>,
+) {
+  const tmpDirPath = fs.mkdtempSync(path.join(tmpdir(), "temp-dir-"));
+  try {
+    await fn(tmpDirPath);
+  } finally {
+    fs.rmSync(tmpDirPath, { recursive: true });
+  }
+}
--- a/nodejs/jest.config.js
+++ b/nodejs/jest.config.js
@@ -4,4 +4,5 @@ module.exports = {
  testEnvironment: "node",
  moduleDirectories: ["node_modules", "./dist"],
  moduleFileExtensions: ["js", "ts"],
+  modulePathIgnorePatterns: ["<rootDir>/examples/"],
 };
--- a/nodejs/lancedb/embedding/transformers.ts
+++ b/nodejs/lancedb/embedding/transformers.ts
@@ -47,8 +47,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
  string,
  Partial<XenovaTransformerOptions>
 > {
-  #model?: import("@xenova/transformers").PreTrainedModel;
-  #tokenizer?: import("@xenova/transformers").PreTrainedTokenizer;
+  #model?: import("@huggingface/transformers").PreTrainedModel;
+  #tokenizer?: import("@huggingface/transformers").PreTrainedTokenizer;
  #modelName: XenovaTransformerOptions["model"];
  #initialized = false;
  #tokenizerOptions: XenovaTransformerOptions["tokenizerOptions"];
@@ -92,18 +92,19 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
    try {
      // SAFETY:
      // since typescript transpiles `import` to `require`, we need to do this in an unsafe way
-      // We can't use `require` because `@xenova/transformers` is an ESM module
+      // We can't use `require` because `@huggingface/transformers` is an ESM module
      // and we can't use `import` directly because typescript will transpile it to `require`.
      // and we want to remain compatible with both ESM and CJS modules
      // so we use `eval` to bypass typescript for this specific import.
-      transformers = await eval('import("@xenova/transformers")');
+      transformers = await eval('import("@huggingface/transformers")');
    } catch (e) {
-      throw new Error(`error loading @xenova/transformers\nReason: ${e}`);
+      throw new Error(`error loading @huggingface/transformers\nReason: ${e}`);
    }

    try {
      this.#model = await transformers.AutoModel.from_pretrained(
        this.#modelName,
+        { dtype: "fp32" },
      );
    } catch (e) {
      throw new Error(
@@ -128,7 +129,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
    } else {
      const config = this.#model!.config;

-      const ndims = config["hidden_size"];
+      // biome-ignore lint/style/useNamingConvention: we don't control this name.
+      const ndims = (config as unknown as { hidden_size: number }).hidden_size;
      if (!ndims) {
        throw new Error(
          "hidden_size not found in model config, you may need to manually specify the embedding dimensions. ",
@@ -183,7 +185,7 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
 }

 const tensorDiv = (
-  src: import("@xenova/transformers").Tensor,
+  src: import("@huggingface/transformers").Tensor,
  divBy: number,
 ) => {
  for (let i = 0; i < src.data.length; ++i) {
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -571,4 +571,9 @@ export class Query extends QueryBase<NativeQuery> {
      return new VectorQuery(vectorQuery);
    }
  }
+
+  nearestToText(query: string, columns?: string[]): Query {
+    this.doCall((inner) => inner.fullTextSearch(query, columns));
+    return this;
+  }
 }
--- a/nodejs/npm/darwin-arm64/package.json
+++ b/nodejs/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.12.0",
+	"version": "0.13.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
--- a/nodejs/npm/darwin-x64/package.json
+++ b/nodejs/npm/darwin-x64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.12.0",
+	"version": "0.13.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",
--- a/nodejs/npm/linux-arm64-gnu/package.json
+++ b/nodejs/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.12.0",
+	"version": "0.13.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
--- a/nodejs/npm/linux-x64-gnu/package.json
+++ b/nodejs/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.12.0",
+	"version": "0.13.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
--- a/nodejs/npm/win32-arm64-msvc/README.md
+++ b/nodejs/npm/win32-arm64-msvc/README.md
@@ -0,0 +1,3 @@
+# `@lancedb/lancedb-win32-arm64-msvc`
+
+This is the **aarch64-pc-windows-msvc** binary for `@lancedb/lancedb`
--- a/nodejs/npm/win32-arm64-msvc/package.json
+++ b/nodejs/npm/win32-arm64-msvc/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "@lancedb/lancedb-win32-arm64-msvc",
+  "version": "0.13.0-beta.1",
+  "os": [
+    "win32"
+  ],
+  "cpu": [
+    "arm64"
+  ],
+  "main": "lancedb.win32-arm64-msvc.node",
+  "files": [
+    "lancedb.win32-arm64-msvc.node"
+  ],
+  "license": "Apache 2.0",
+  "engines": {
+    "node": ">= 18"
+  }
+}
--- a/nodejs/npm/win32-x64-msvc/package.json
+++ b/nodejs/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.12.0",
+	"version": "0.13.0-beta.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
--- a/nodejs/package-lock.json
+++ b/nodejs/package-lock.json
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -10,7 +10,7 @@
    "vector database",
    "ann"
  ],
-  "version": "0.12.0",
+  "version": "0.13.0-beta.1",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -85,7 +85,7 @@
    "reflect-metadata": "^0.2.2"
  },
  "optionalDependencies": {
-    "@xenova/transformers": ">=2.17 < 3",
+    "@huggingface/transformers": "^3.0.2",
    "openai": "^4.29.2"
  },
  "peerDependencies": {
--- a/nodejs/src/lib.rs
+++ b/nodejs/src/lib.rs
@@ -82,7 +82,7 @@ pub struct OpenTableOptions {
 #[napi::module_init]
 fn init() {
    let env = Env::new()
-        .filter_or("LANCEDB_LOG", "trace")
+        .filter_or("LANCEDB_LOG", "warn")
        .write_style("LANCEDB_LOG_STYLE");
    env_logger::init_from_env(env);
 }
--- a/nodejs/tsconfig.json
+++ b/nodejs/tsconfig.json
@@ -12,7 +12,7 @@
    "experimentalDecorators": true,
    "moduleResolution": "Node"
  },
-  "exclude": ["./dist/*"],
+  "exclude": ["./dist/*", "./examples/*"],
  "typedocOptions": {
    "entryPoints": ["lancedb/index.ts"],
    "out": "../docs/src/javascript/",
--- a/python/python/lancedb/embeddings/init.py
+++ b/python/python/lancedb/embeddings/init.py
@@ -27,3 +27,4 @@ from .imagebind import ImageBindEmbeddings
 from .utils import with_embeddings
 from .jinaai import JinaEmbeddings
 from .watsonx import WatsonxEmbeddings
+from .voyageai import VoyageAIEmbeddingFunction
--- a/python/python/lancedb/embeddings/registry.py
+++ b/python/python/lancedb/embeddings/registry.py
@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import json
 from typing import Dict, Optional

@@ -170,7 +161,7 @@ def register(name):
    return __REGISTRY__.get_instance().register(name)


-def get_registry():
+def get_registry() -> EmbeddingFunctionRegistry:
    """
    Utility function to get the global instance of the registry

--- a/python/python/lancedb/embeddings/voyageai.py
+++ b/python/python/lancedb/embeddings/voyageai.py
@@ -0,0 +1,127 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+from typing import ClassVar, List, Union
+
+import numpy as np
+
+from ..util import attempt_import_or_raise
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import api_key_not_found_help, TEXT
+
+
+@register("voyageai")
+class VoyageAIEmbeddingFunction(TextEmbeddingFunction):
+    """
+    An embedding function that uses the VoyageAI API
+
+    https://docs.voyageai.com/docs/embeddings
+
+    Parameters
+    ----------
+    name: str
+        The name of the model to use. List of acceptable models:
+
+            * voyage-3
+            * voyage-3-lite
+            * voyage-finance-2
+            * voyage-multilingual-2
+            * voyage-law-2
+            * voyage-code-2
+
+
+    Examples
+    --------
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    voyageai = EmbeddingFunctionRegistry
+        .get_instance()
+        .get("voyageai")
+        .create(name="voyage-3")
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) =  voyageai.VectorField()
+
+    data = [ { "text": "hello world" },
+            { "text": "goodbye world" }]
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+
+    """
+
+    name: str
+    client: ClassVar = None
+
+    def ndims(self):
+        if self.name == "voyage-3-lite":
+            return 512
+        elif self.name == "voyage-code-2":
+            return 1536
+        elif self.name in [
+            "voyage-3",
+            "voyage-finance-2",
+            "voyage-multilingual-2",
+            "voyage-law-2",
+        ]:
+            return 1024
+        else:
+            raise ValueError(f"Model {self.name} not supported")
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.compute_source_embeddings(query, input_type="query")
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        input_type = (
+            kwargs.get("input_type") or "document"
+        )  # assume source input type if not passed by `compute_query_embeddings`
+        return self.generate_embeddings(texts, input_type=input_type)
+
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray], *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        input_type: Optional[str]
+
+        truncation: Optional[bool]
+        """
+        VoyageAIEmbeddingFunction._init_client()
+        rs = VoyageAIEmbeddingFunction.client.embed(
+            texts=texts, model=self.name, **kwargs
+        )
+
+        return [emb for emb in rs.embeddings]
+
+    @staticmethod
+    def _init_client():
+        if VoyageAIEmbeddingFunction.client is None:
+            voyageai = attempt_import_or_raise("voyageai")
+            if os.environ.get("VOYAGE_API_KEY") is None:
+                api_key_not_found_help("voyageai")
+            VoyageAIEmbeddingFunction.client = voyageai.Client(
+                os.environ["VOYAGE_API_KEY"]
+            )
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -11,6 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

+from datetime import timedelta
 import asyncio
 import logging
 from functools import cached_property
@@ -478,6 +479,19 @@ class RemoteTable(Table):
            "compact_files() is not supported on the LanceDB cloud"
        )

+    def optimize(
+        self,
+        *,
+        cleanup_older_than: Optional[timedelta] = None,
+        delete_unverified: bool = False,
+    ):
+        """optimize() is not supported on the LanceDB cloud.
+        Indices are optimized automatically."""
+        raise NotImplementedError(
+            "optimize() is not supported on the LanceDB cloud. "
+            "Indices are optimized automatically."
+        )
+
    def count_rows(self, filter: Optional[str] = None) -> int:
        return self._loop.run_until_complete(self._table.count_rows(filter))

--- a/python/python/lancedb/rerankers/init.py
+++ b/python/python/lancedb/rerankers/init.py
@@ -7,6 +7,7 @@ from .openai import OpenaiReranker
 from .jinaai import JinaReranker
 from .rrf import RRFReranker
 from .answerdotai import AnswerdotaiRerankers
+from .voyageai import VoyageAIReranker

 __all__ = [
    "Reranker",
@@ -18,4 +19,5 @@ __all__ = [
    "JinaReranker",
    "RRFReranker",
    "AnswerdotaiRerankers",
+    "VoyageAIReranker",
 ]
--- a/python/python/lancedb/rerankers/voyageai.py
+++ b/python/python/lancedb/rerankers/voyageai.py
@@ -0,0 +1,133 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+from functools import cached_property
+from typing import Optional
+
+import pyarrow as pa
+
+from ..util import attempt_import_or_raise
+from .base import Reranker
+
+
+class VoyageAIReranker(Reranker):
+    """
+    Reranks the results using the VoyageAI Rerank API.
+    https://docs.voyageai.com/docs/reranker
+
+    Parameters
+    ----------
+    model_name : str, default "rerank-english-v2.0"
+        The name of the cross encoder model to use. Available voyageai models are:
+        - rerank-2
+        - rerank-2-lite
+    column : str, default "text"
+        The name of the column to use as input to the cross encoder model.
+    top_n : int, default None
+        The number of results to return. If None, will return all results.
+    return_score : str, default "relevance"
+        options are "relevance" or "all". Only "relevance" is supported for now.
+    api_key : str, default None
+        The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+    truncation : Optional[bool], default None
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        column: str = "text",
+        top_n: Optional[int] = None,
+        return_score="relevance",
+        api_key: Optional[str] = None,
+        truncation: Optional[bool] = True,
+    ):
+        super().__init__(return_score)
+        self.model_name = model_name
+        self.column = column
+        self.top_n = top_n
+        self.api_key = api_key
+        self.truncation = truncation
+
+    @cached_property
+    def _client(self):
+        voyageai = attempt_import_or_raise("voyageai")
+        if os.environ.get("VOYAGE_API_KEY") is None and self.api_key is None:
+            raise ValueError(
+                "VOYAGE_API_KEY not set. Either set it in your environment or \
+                pass it as `api_key` argument to the VoyageAIReranker."
+            )
+        return voyageai.Client(
+            api_key=os.environ.get("VOYAGE_API_KEY") or self.api_key,
+        )
+
+    def _rerank(self, result_set: pa.Table, query: str):
+        docs = result_set[self.column].to_pylist()
+        response = self._client.rerank(
+            query=query,
+            documents=docs,
+            top_k=self.top_n,
+            model=self.model_name,
+            truncation=self.truncation,
+        )
+        results = (
+            response.results
+        )  # returns list (text, idx, relevance) attributes sorted descending by score
+        indices, scores = list(
+            zip(*[(result.index, result.relevance_score) for result in results])
+        )  # tuples
+        result_set = result_set.take(list(indices))
+        # add the scores
+        result_set = result_set.append_column(
+            "_relevance_score", pa.array(scores, type=pa.float32())
+        )
+
+        return result_set
+
+    def rerank_hybrid(
+        self,
+        query: str,
+        vector_results: pa.Table,
+        fts_results: pa.Table,
+    ):
+        combined_results = self.merge_results(vector_results, fts_results)
+        combined_results = self._rerank(combined_results, query)
+        if self.score == "relevance":
+            combined_results = self._keep_relevance_score(combined_results)
+        elif self.score == "all":
+            raise NotImplementedError(
+                "return_score='all' not implemented for voyageai reranker"
+            )
+        return combined_results
+
+    def rerank_vector(
+        self,
+        query: str,
+        vector_results: pa.Table,
+    ):
+        result_set = self._rerank(vector_results, query)
+        if self.score == "relevance":
+            result_set = result_set.drop_columns(["_distance"])
+
+        return result_set
+
+    def rerank_fts(
+        self,
+        query: str,
+        fts_results: pa.Table,
+    ):
+        result_set = self._rerank(fts_results, query)
+        if self.score == "relevance":
+            result_set = result_set.drop_columns(["_score"])
+
+        return result_set
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -3,6 +3,7 @@

 from __future__ import annotations

+import asyncio
 import inspect
 import time
 from abc import ABC, abstractmethod
@@ -32,7 +33,7 @@ import pyarrow.fs as pa_fs
 from lance import LanceDataset
 from lance.dependencies import _check_for_hugging_face

-from .common import DATA, VEC, VECTOR_COLUMN_NAME
+from .common import DATA, VEC, VECTOR_COLUMN_NAME, sanitize_uri
 from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
 from .merge import LanceMergeInsertBuilder
 from .pydantic import LanceModel, model_to_dict
@@ -57,6 +58,8 @@ from .util import (
 )
 from .index import lang_mapping

+from ._lancedb import connect as lancedb_connect
+
 if TYPE_CHECKING:
    import PIL
    from lance.dataset import CleanupStats, ReaderLike
@@ -70,6 +73,21 @@ pl = safe_import_polars()
 QueryType = Literal["vector", "fts", "hybrid", "auto"]


+def _pd_schema_without_embedding_funcs(
+    schema: Optional[pa.Schema], columns: List[str]
+) -> Optional[pa.Schema]:
+    """Return a schema without any embedding function columns"""
+    if schema is None:
+        return None
+    embedding_functions = EmbeddingFunctionRegistry.get_instance().parse_functions(
+        schema.metadata
+    )
+    if not embedding_functions:
+        return schema
+    columns = set(columns)
+    return pa.schema([field for field in schema if field.name in columns])
+
+
 def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
    if _check_for_hugging_face(data):
        # Huggingface datasets
@@ -100,10 +118,10 @@ def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
        elif isinstance(data[0], pa.RecordBatch):
            return pa.Table.from_batches(data, schema=schema)
        else:
-            return pa.Table.from_pylist(data)
+            return pa.Table.from_pylist(data, schema=schema)
    elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
-        # Do not add schema here, since schema may contains the vector column
-        table = pa.Table.from_pandas(data, preserve_index=False)
+        raw_schema = _pd_schema_without_embedding_funcs(schema, data.columns.to_list())
+        table = pa.Table.from_pandas(data, preserve_index=False, schema=raw_schema)
        # Do not serialize Pandas metadata
        meta = table.schema.metadata if table.schema.metadata is not None else {}
        meta = {k: v for k, v in meta.items() if k != b"pandas"}
@@ -169,6 +187,8 @@ def sanitize_create_table(
        schema = schema.to_arrow_schema()

    if data is not None:
+        if metadata is None and schema is not None:
+            metadata = schema.metadata
        data, schema = _sanitize_data(
            data,
            schema,
@@ -893,6 +913,55 @@ class Table(ABC):
        For most cases, the default should be fine.
        """

+    @abstractmethod
+    def optimize(
+        self,
+        *,
+        cleanup_older_than: Optional[timedelta] = None,
+        delete_unverified: bool = False,
+    ):
+        """
+        Optimize the on-disk data and indices for better performance.
+
+        Modeled after ``VACUUM`` in PostgreSQL.
+
+        Optimization covers three operations:
+
+         * Compaction: Merges small files into larger ones
+         * Prune: Removes old versions of the dataset
+         * Index: Optimizes the indices, adding new data to existing indices
+
+        Parameters
+        ----------
+        cleanup_older_than: timedelta, optional default 7 days
+            All files belonging to versions older than this will be removed.  Set
+            to 0 days to remove all versions except the latest.  The latest version
+            is never removed.
+        delete_unverified: bool, default False
+            Files leftover from a failed transaction may appear to be part of an
+            in-progress operation (e.g. appending new data) and these files will not
+            be deleted unless they are at least 7 days old. If delete_unverified is True
+            then these files will be deleted regardless of their age.
+
+        Experimental API
+        ----------------
+
+        The optimization process is undergoing active development and may change.
+        Our goal with these changes is to improve the performance of optimization and
+        reduce the complexity.
+
+        That being said, it is essential today to run optimize if you want the best
+        performance.  It should be stable and safe to use in production, but it our
+        hope that the API may be simplified (or not even need to be called) in the
+        future.
+
+        The frequency an application shoudl call optimize is based on the frequency of
+        data modifications.  If data is frequently added, deleted, or updated then
+        optimize should be run frequently.  A good rule of thumb is to run optimize if
+        you have added or modified 100,000 or more records or run more than 20 data
+        modification operations.
+        """
+
    @abstractmethod
    def add_columns(self, transforms: Dict[str, str]):
        """
@@ -1971,6 +2040,83 @@ class LanceTable(Table):
        """
        return self.to_lance().optimize.compact_files(*args, **kwargs)

+    def optimize(
+        self,
+        *,
+        cleanup_older_than: Optional[timedelta] = None,
+        delete_unverified: bool = False,
+    ):
+        """
+        Optimize the on-disk data and indices for better performance.
+
+        Modeled after ``VACUUM`` in PostgreSQL.
+
+        Optimization covers three operations:
+
+         * Compaction: Merges small files into larger ones
+         * Prune: Removes old versions of the dataset
+         * Index: Optimizes the indices, adding new data to existing indices
+
+        Parameters
+        ----------
+        cleanup_older_than: timedelta, optional default 7 days
+            All files belonging to versions older than this will be removed.  Set
+            to 0 days to remove all versions except the latest.  The latest version
+            is never removed.
+        delete_unverified: bool, default False
+            Files leftover from a failed transaction may appear to be part of an
+            in-progress operation (e.g. appending new data) and these files will not
+            be deleted unless they are at least 7 days old. If delete_unverified is True
+            then these files will be deleted regardless of their age.
+
+        Experimental API
+        ----------------
+
+        The optimization process is undergoing active development and may change.
+        Our goal with these changes is to improve the performance of optimization and
+        reduce the complexity.
+
+        That being said, it is essential today to run optimize if you want the best
+        performance.  It should be stable and safe to use in production, but it our
+        hope that the API may be simplified (or not even need to be called) in the
+        future.
+
+        The frequency an application shoudl call optimize is based on the frequency of
+        data modifications.  If data is frequently added, deleted, or updated then
+        optimize should be run frequently.  A good rule of thumb is to run optimize if
+        you have added or modified 100,000 or more records or run more than 20 data
+        modification operations.
+        """
+        try:
+            asyncio.get_running_loop()
+            raise AssertionError(
+                "Synchronous method called in asynchronous context. "
+                "If you are writing an asynchronous application "
+                "then please use the asynchronous APIs"
+            )
+
+        except RuntimeError:
+            asyncio.run(
+                self._async_optimize(
+                    cleanup_older_than=cleanup_older_than,
+                    delete_unverified=delete_unverified,
+                )
+            )
+            self.checkout_latest()
+
+    async def _async_optimize(
+        self,
+        cleanup_older_than: Optional[timedelta] = None,
+        delete_unverified: bool = False,
+    ):
+        conn = await lancedb_connect(
+            sanitize_uri(self._conn.uri),
+        )
+        table = AsyncTable(await conn.open_table(self.name))
+        return await table.optimize(
+            cleanup_older_than=cleanup_older_than, delete_unverified=delete_unverified
+        )
+
    def add_columns(self, transforms: Dict[str, str]):
        self._dataset_mut.add_columns(transforms)

--- a/python/python/tests/docs/test_cloud.py
+++ b/python/python/tests/docs/test_cloud.py
@@ -0,0 +1,293 @@
+# --8<-- [start:imports]
+# --8<-- [start:import-lancedb]
+# --8<-- [start:import-ingest-data]
+import lancedb
+import pyarrow as pa
+# --8<-- [end:import-ingest-data]
+import numpy as np
+
+# --8<-- [end:import-lancedb]
+# --8<-- [end:imports]
+# --8<-- [start:gen_data]
+def gen_data(total_rows: int, ndims: int = 1536):
+    return pa.RecordBatch.from_pylist(
+        [
+            {
+                "vector": np.random.rand(ndims).astype(np.float32).tolist(),
+                "id": i,
+                "name": "name_" + str(i),
+            }
+            for i in range(total_rows)
+        ],
+    ).to_pandas()
+
+
+# --8<-- [end:gen_data]
+
+
+def test_cloud_quickstart():
+    # --8<-- [start:connect]
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="your-cloud-region"
+    )
+    # --8<-- [end:connect]
+    # --8<-- [start:create_table]
+    table_name = "myTable"
+    table = db.create_table(table_name, data=gen_data(5000))
+    # --8<-- [end:create_table]
+    # --8<-- [start:create_index_search]
+    # create a vector index
+    table.create_index("cosine", vector_column_name="vector")
+    result = table.search([0.01, 0.02]).select(["vector", "item"]).limit(1).to_pandas()
+    print(result)
+    # --8<-- [end:create_index_search]
+    # --8<-- [start:drop_table]
+    db.drop_table(table_name)
+    # --8<-- [end:drop_table]
+
+
+def test_ingest_data():
+    # --8<-- [start:ingest_data]
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    # create an empty table with schema
+    table_name = "myTable"
+    data = [
+        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        {"vector": [10.2, 100.8], "item": "baz", "price": 30.0},
+        {"vector": [1.4, 9.5], "item": "fred", "price": 40.0},
+    ]
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.utf8()),
+            pa.field("price", pa.float32()),
+        ]
+    )
+    table = db.create_table(table_name, schema=schema)
+    table.add(data)
+    # --8<-- [end:ingest_data]
+    # --8<-- [start:ingest_data_in_batch]
+    def make_batches():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.utf8()),
+            pa.field("price", pa.float32()),
+        ]
+    )
+    db.create_table("table2", make_batches(), schema=schema)
+    # --8<-- [end:ingest_data_in_batch]
+
+
+def test_updates():
+    # --8<-- [start:update_data]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    table.update(where="price < 20.0", values={"vector": [2, 2], "item": "foo-updated"})
+    # --8<-- [end:update_data]
+    # --8<-- [start:merge_insert]
+    table = db.open_table(table_name)
+    # upsert
+    new_data = [{"vector": [1, 1], "item": "foo-updated", "price": 50.0}]
+    table.merge_insert(
+        "item"
+    ).when_matched_update_all().when_not_matched_insert_all().execute(new_data)
+    # --8<-- [end:merge_insert]
+    # --8<-- [start:delete_data]
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    # delete data
+    predicate = "price = 30.0"
+    table.delete(predicate)
+    # --8<-- [end:delete_data]
+
+
+def test_create_index():
+    # --8<-- [start:create_index]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    # the vector column only needs to be specified when there are
+    # multiple vector columns or the column is not named as "vector"
+    # L2 is used as the default distance metric
+    table.create_index(metric="cosine", vector_column_name="vector")
+    # --8<-- [end:create_index]
+
+
+def test_create_scalar_index():
+    # --8<-- [start:create_scalar_index]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    # default is BTree
+    table.create_scalar_index("item", index_type="BITMAP")
+    # --8<-- [end:create_scalar_index]
+
+
+def test_create_fts_index():
+    # --8<-- [start:create_fts_index]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    data = [
+        {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+        {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+    ]
+    table = db.create_table(table_name, data=data)
+    table.create_fts_index("text")
+    # --8<-- [end:create_fts_index]
+
+
+def test_search():
+    # --8<-- [start:vector_search]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    query = [0.4, 1.4]
+    result = (
+        table.search(query)
+        .where("price > 10.0", prefilter=True)
+        .select(["item", "vector"])
+        .limit(2)
+        .to_pandas()
+    )
+    print(result)
+    # --8<-- [end:vector_search]
+    # --8<-- [start:full_text_search]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+    table_name = "myTable"
+    table = db.create_table(
+        table_name,
+        data=[
+            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+        ],
+    )
+
+    table.create_fts_index("text")
+    table.search("puppy", query_type="fts").limit(10).select(["text"]).to_list()
+    # --8<-- [end:full_text_search]
+    # --8<-- [start:hybrid_search]
+    import os
+
+    import lancedb
+    import openai
+    from lancedb.embeddings import get_registry
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.rerankers import RRFReranker
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+
+    # Configuring the environment variable OPENAI_API_KEY
+    if "OPENAI_API_KEY" not in os.environ:
+        # OR set the key here as a variable
+        openai.api_key = "sk-..."
+    embeddings = get_registry().get("openai").create()
+
+    class Documents(LanceModel):
+        text: str = embeddings.SourceField()
+        vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+
+    table_name = "myTable"
+    table = db.create_table(table_name, schema=Documents)
+    data = [
+        {"text": "rebel spaceships striking from a hidden base"},
+        {"text": "have won their first victory against the evil Galactic Empire"},
+        {"text": "during the battle rebel spies managed to steal secret plans"},
+        {"text": "to the Empire's ultimate weapon the Death Star"},
+    ]
+    table.add(data=data)
+    table.create_index("L2", "vector")
+    table.create_fts_index("text")
+
+    # you can use table.list_indices() to make sure indices have been created
+    reranker = RRFReranker()
+    result = (
+        table.search(
+            "flower moon",
+            query_type="hybrid",
+            vector_column_name="vector",
+            fts_columns="text",
+        )
+        .rerank(reranker)
+        .limit(10)
+        .to_pandas()
+    )
+    print(result)
+    # --8<-- [end:hybrid_search]
+
+
+def test_filtering():
+    # --8<-- [start:filtering]
+    import lancedb
+
+    # connect to LanceDB
+    db = lancedb.connect(
+        uri="db://your-project-slug", api_key="your-api-key", region="us-east-1"
+    )
+    table_name = "myTable"
+    table = db.open_table(table_name)
+    result = (
+        table.search([100, 102])
+        .where("(item IN ('foo', 'bar')) AND (price > 10.0)")
+        .to_arrow()
+    )
+    print(result)
+    # --8<-- [end:filtering]
+    # --8<-- [start:sql_filtering]
+    table.search([100, 102]).where(
+        "(item IN ('foo', 'bar')) AND (price > 10.0)"
+    ).to_arrow()
+    # --8<-- [end:sql_filtering]
--- a/python/python/tests/test_embeddings.py
+++ b/python/python/tests/test_embeddings.py
@@ -1,15 +1,6 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 from typing import List, Union
 from unittest.mock import MagicMock, patch

@@ -18,6 +9,7 @@ import lancedb
 import numpy as np
 import pyarrow as pa
 import pytest
+import pandas as pd
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings import (
    EmbeddingFunctionConfig,
@@ -129,6 +121,142 @@ def test_embedding_with_bad_results(tmp_path):
    # assert tbl["vector"].null_count == 1


+def test_with_existing_vectors(tmp_path):
+    @register("mock-embedding")
+    class MockEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    registry = get_registry()
+    model = registry.get("mock-embedding").create()
+
+    class Schema(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=Schema, mode="overwrite")
+    tbl.add([{"text": "hello world", "vector": np.zeros(128).tolist()}])
+
+    embeddings = tbl.to_arrow()["vector"].to_pylist()
+    assert not np.any(embeddings), "all zeros"
+
+
+def test_embedding_function_with_pandas(tmp_path):
+    @register("mock-embedding")
+    class _MockEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    registery = get_registry()
+    func = registery.get("mock-embedding").create()
+
+    class TestSchema(LanceModel):
+        text: str = func.SourceField()
+        val: int
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    df = pd.DataFrame(
+        {
+            "text": ["hello world", "goodbye world"],
+            "val": [1, 2],
+            "not-used": ["s1", "s3"],
+        }
+    )
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TestSchema, mode="overwrite", data=df)
+    schema = tbl.schema
+    assert schema.field("text").type == pa.string()
+    assert schema.field("val").type == pa.int64()
+    assert schema.field("vector").type == pa.list_(pa.float32(), 128)
+
+    df = pd.DataFrame(
+        {
+            "text": ["extra", "more"],
+            "val": [4, 5],
+            "misc-col": ["s1", "s3"],
+        }
+    )
+    tbl.add(df)
+
+    assert tbl.count_rows() == 4
+    embeddings = tbl.to_arrow()["vector"]
+    assert embeddings.null_count == 0
+
+    df = pd.DataFrame(
+        {
+            "text": ["with", "embeddings"],
+            "val": [6, 7],
+            "vector": [np.zeros(128).tolist(), np.zeros(128).tolist()],
+        }
+    )
+    tbl.add(df)
+
+    embeddings = tbl.search().where("val > 5").to_arrow()["vector"].to_pylist()
+    assert not np.any(embeddings), "all zeros"
+
+
+def test_multiple_embeddings_for_pandas(tmp_path):
+    @register("mock-embedding")
+    class MockFunc1(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    @register("mock-embedding2")
+    class MockFunc2(TextEmbeddingFunction):
+        def ndims(self):
+            return 512
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> List[np.array]:
+            return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
+
+    registery = get_registry()
+    func1 = registery.get("mock-embedding").create()
+    func2 = registery.get("mock-embedding2").create()
+
+    class TestSchema(LanceModel):
+        text: str = func1.SourceField()
+        val: int
+        vec1: Vector(func1.ndims()) = func1.VectorField()
+        prompt: str = func2.SourceField()
+        vec2: Vector(func2.ndims()) = func2.VectorField()
+
+    df = pd.DataFrame(
+        {
+            "text": ["hello world", "goodbye world"],
+            "val": [1, 2],
+            "prompt": ["hello", "goodbye"],
+        }
+    )
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TestSchema, mode="overwrite", data=df)
+
+    schema = tbl.schema
+    assert schema.field("text").type == pa.string()
+    assert schema.field("val").type == pa.int64()
+    assert schema.field("vec1").type == pa.list_(pa.float32(), 128)
+    assert schema.field("prompt").type == pa.string()
+    assert schema.field("vec2").type == pa.list_(pa.float32(), 512)
+    assert tbl.count_rows() == 2
+
+
@pytest.mark.slow
 def test_embedding_function_rate_limit(tmp_path):
    def _get_schema_from_model(model):
@@ -196,6 +324,7 @@ def test_add_optional_vector(tmp_path):
        "ollama",
        "cohere",
        "instructor",
+        "voyageai",
    ],
 )
 def test_embedding_function_safe_model_dump(embedding_type):
--- a/python/python/tests/test_embeddings_slow.py
+++ b/python/python/tests/test_embeddings_slow.py
@@ -481,3 +481,22 @@ def test_ollama_embedding(tmp_path):
        json.dumps(dumped_model)
    except TypeError:
        pytest.fail("Failed to JSON serialize the dumped model")
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_embedding_function():
+    voyageai = get_registry().get("voyageai").create(name="voyage-3", max_retries=0)
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect("~/lancedb")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
--- a/python/python/tests/test_rerankers.py
+++ b/python/python/tests/test_rerankers.py
@@ -16,6 +16,7 @@ from lancedb.rerankers import (
    OpenaiReranker,
    JinaReranker,
    AnswerdotaiRerankers,
+    VoyageAIReranker,
 )
 from lancedb.table import LanceTable

@@ -344,3 +345,14 @@ def test_jina_reranker(tmp_path, use_tantivy):
    table, schema = get_test_table(tmp_path, use_tantivy)
    reranker = JinaReranker()
    _run_test_reranker(reranker, table, "single player experience", None, schema)
+
+
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+@pytest.mark.parametrize("use_tantivy", [True, False])
+def test_voyageai_reranker(tmp_path, use_tantivy):
+    pytest.importorskip("voyageai")
+    reranker = VoyageAIReranker(model_name="rerank-2")
+    table, schema = get_test_table(tmp_path, use_tantivy)
+    _run_test_reranker(reranker, table, "single player experience", None, schema)
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -1223,6 +1223,54 @@ async def test_time_travel(db_async: AsyncConnection):
        await table.restore()


+def test_sync_optimize(db):
+    table = LanceTable.create(
+        db,
+        "test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        ],
+    )
+
+    table.create_scalar_index("price", index_type="BTREE")
+    stats = table.to_lance().stats.index_stats("price_idx")
+    assert stats["num_indexed_rows"] == 2
+
+    table.add([{"vector": [2.0, 2.0], "item": "baz", "price": 30.0}])
+    assert table.count_rows() == 3
+    table.optimize()
+    stats = table.to_lance().stats.index_stats("price_idx")
+    assert stats["num_indexed_rows"] == 3
+
+
+@pytest.mark.asyncio
+async def test_sync_optimize_in_async(db):
+    table = LanceTable.create(
+        db,
+        "test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        ],
+    )
+
+    table.create_scalar_index("price", index_type="BTREE")
+    stats = table.to_lance().stats.index_stats("price_idx")
+    assert stats["num_indexed_rows"] == 2
+
+    table.add([{"vector": [2.0, 2.0], "item": "baz", "price": 30.0}])
+    assert table.count_rows() == 3
+    try:
+        table.optimize()
+    except Exception as e:
+        assert (
+            "Synchronous method called in asynchronous context. "
+            "If you are writing an asynchronous application "
+            "then please use the asynchronous APIs" in str(e)
+        )
+
+
@pytest.mark.asyncio
 async def test_optimize(db_async: AsyncConnection):
    table = await db_async.create_table(
--- a/rust/ffi/node/Cargo.toml
+++ b/rust/ffi/node/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.12.0"
+version = "0.13.0-beta.1"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true
--- a/rust/lancedb/Cargo.toml
+++ b/rust/lancedb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.12.0"
+version = "0.13.0-beta.1"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
--- a/rust/lancedb/src/index.rs
+++ b/rust/lancedb/src/index.rs
@@ -29,6 +29,7 @@ pub mod scalar;
 pub mod vector;

 /// Supported index types.
+#[derive(Debug, Clone)]
 pub enum Index {
    Auto,
    /// A `BTree` index is an sorted index on scalar columns.
Author	SHA1	Message	Date
qzhu	a503845c9f	more edit	2024-11-14 13:33:25 -08:00
qzhu	955a295026	code for cloud doc	2024-11-13 22:05:09 -08:00
qzhu	b70fa3892e	code for cloud doc	2024-11-13 22:03:53 -08:00
qzhu	31fb3b3b5c	first edit	2024-11-13 21:57:05 -08:00
Will Jones	0fd8a50bd7	ci(node): run examples in CI (#1796 ) This is done as setup for a PR that will fix the OpenAI dependency issue. * [x] FTS examples * [x] Setup mock openai * [x] Ran `npm audit fix` * [x] sentences embeddings test * [x] Double check formatting of docs examples	2024-11-13 11:10:56 -08:00
Umut Hope YILDIRIM	9f228feb0e	ci: remove cache to fix build issues on windows arm runner (#1820 )	2024-11-13 09:27:10 -08:00
Ayush Chaurasia	90e9c52d0a	docs: update hybrid search example to latest langchain (#1824 ) Co-authored-by: qzhu <qian@lancedb.com>	2024-11-12 20:06:25 -08:00
Will Jones	68974a4e06	ci: add index URL to fix failing docs build (#1823 )	2024-11-12 16:54:22 -08:00
Lei Xu	4c9bab0d92	fix: use pandas with pydantic embedding column (#1818 ) * Make Pandas `DataFrame` works with embedding function + Subset of columns * Make `lancedb.create_table()` work with embedding function	2024-11-11 14:48:56 -08:00
QianZhu	5117aecc38	docs: search param explanation for OSS doc (#1815 ) ![Screenshot 2024-11-09 at 11 09 14 AM](https://github.com/user-attachments/assets/2aeba016-aeff-4658-85c6-8640285ba0c9)	2024-11-11 11:57:17 -08:00
Umut Hope YILDIRIM	729718cb09	fix: arm64 runner proto already installed bug (#1810 ) https://github.com/lancedb/lancedb/actions/runs/11748512661/job/32732745458	2024-11-08 14:49:37 -08:00
Umut Hope YILDIRIM	b1c84e0bda	feat: added lancedb and vectordb release ci for win32-arm64-msvc npmjs only (#1805 )	2024-11-08 11:40:57 -08:00
fzowl	cbbc07d0f5	feat: voyageai support (#1799 ) Adding VoyageAI embedding and rerank support	2024-11-09 00:51:20 +05:30
Kursat Aktas	21021f94ca	docs: introducing LanceDB Guru on Gurubase.io (#1797 ) Hello team, I'm the maintainer of [Anteon](https://github.com/getanteon/anteon). We have created Gurubase.io with the mission of building a centralized, open-source tool-focused knowledge base. Essentially, each "guru" is equipped with custom knowledge to answer user questions based on collected data related to that tool. I wanted to update you that I've manually added the [LanceDB Guru](https://gurubase.io/g/lancedb) to Gurubase. LanceDB Guru uses the data from this repo and data from the [docs](https://lancedb.github.io/lancedb/) to answer questions by leveraging the LLM. In this PR, I showcased the "LanceDB Guru", which highlights that LanceDB now has an AI assistant available to help users with their questions. Please let me know your thoughts on this contribution. Additionally, if you want me to disable LanceDB Guru in Gurubase, just let me know that's totally fine. Signed-off-by: Kursat Aktas <kursat.ce@gmail.com>	2024-11-08 10:55:22 -08:00
BubbleCal	0ed77fa990	chore: impl Debug & Clone for Index params (#1808 ) we don't really need these trait in lancedb, but all fields in `Index` implement the 2 traits, so do it for possibility to use `Index` somewhere Signed-off-by: BubbleCal <bubble-cal@outlook.com>	2024-11-09 01:07:43 +08:00
BubbleCal	4372c231cd	feat: support optimize indices in sync API (#1769 ) Signed-off-by: BubbleCal <bubble-cal@outlook.com>	2024-11-08 08:48:07 -08:00
Umut Hope YILDIRIM	fa9ca8f7a6	ci: arm64 windows build support (#1770 ) Adds support for 'aarch64-pc-windows-msvc'.	2024-11-06 15:34:23 -08:00
Lance Release	2a35d24ee6	Updating package-lock.json	2024-11-06 17:26:36 +00:00
Lance Release	dd9ce337e2	Bump version: 0.13.0-beta.0 → 0.13.0-beta.1	2024-11-06 17:26:17 +00:00
Will Jones	b9921d56cc	fix(node): update default log level to warn (#1801 ) 🤦	2024-11-06 09:13:53 -08:00
Lance Release	0cfd9ed18e	Updating package-lock.json	2024-11-05 23:21:50 +00:00
Lance Release	975398c3a8	Bump version: 0.12.0 → 0.13.0-beta.0	2024-11-05 23:21:32 +00:00